Acknowledge Discord interactions before database work

Introduce an outbound Python host agent, Debian installer, durable remote
operations, immutable game-file libraries, host-aware deployments and ports,
and occupancy-safe campaigns and draining. Native hosts do not require Komodo.

Add administrator, operator and viewer accounts, protected management routes,
secure sessions, password recovery, audit history and separate telemetry tokens.
Document first-admin setup, reporter migration and live-host acceptance checks.

Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type
checking, the production build, compiled SSR/API smoke tests and administrator
recovery. Real Debian/Docker T5/T6 acceptance remains unrun.

.dockerignore

@@ -10,3 +10,7 @@ archive
 *.log
 test-fixtures
 data
+# Offline release signing material must never enter the build context.
+**/.gnupg
+**/private-keys-v1.d
+**/secring.gpg

.env.example

@@ -1,52 +1,56 @@
-# Filesystem paths — inside the Komodo container, these mirror the host layout
+# Controller URL: use HTTPS with a trusted certificate for enrolled hosts.
-SERVERS_ROOT=/root/zombies/t5
+EC_PUBLIC_URL=https://electric-cherry.example.com
-# T6 is separate; defaults to a sibling t6 directory beside SERVERS_ROOT.
+# Client addresses for login throttling behind a trusted reverse proxy.
-T6_SERVERS_ROOT=/root/zombies/t6
+# Leave ADDRESS_HEADER empty for direct connections. Enable only when all access
-REL_DEST=runtime/plutonium/storage/t5/raw/scripts/sp
+# goes through a proxy that replaces untrusted forwarding headers.
-# On-disk archive of active beta files (cleared per-beta on promote/rollback)
+# For one trusted proxy using X-Forwarded-For:
-BETA_ARCHIVE_DIR=/root/zombies/beta-archive
+# ADDRESS_HEADER=x-forwarded-for
+# XFF_DEPTH=1
+# Set XFF_DEPTH to the number of trusted proxy hops; see the access guide.
+ADDRESS_HEADER=
+XFF_DEPTH=1
+# Generate separate secrets with: openssl rand -hex 32
+# Setup token authorizes creation of the first administrator only. Remove it
+# after setup. Never reuse the telemetry token as a setup or host credential.
+EC_SETUP_TOKEN=
+# Game reporters send Authorization: Bearer <token> or X-EC-Telemetry-Token.
+# Telemetry ingestion is disabled until a token of at least 32 chars is set.
+EC_TELEMETRY_TOKEN=
+# Optional native Discord companion. Keep these in the controller environment;
+# enable the integration and optionally exclude servers under Admin → Discord.
+# The application ID and public key are in the Discord Developer Portal.
+# The bot token is a secret: never put it in channel messages or screenshots.
+EC_DISCORD_APPLICATION_ID=
+EC_DISCORD_PUBLIC_KEY=
+EC_DISCORD_BOT_TOKEN=
+# Bank credentials are generated and injected automatically per native instance.
+EC_BANK_MAX_BALANCE=250000
+# Persistent SQLite metadata, game-file storage, and beta archives.
+# For an existing deployment, retain its original data directory.
+EC_DATA_DIR=/root/zombies/.electric-cherry
+DB_PATH=./data/cherry.db
+GAME_FILES_ROOT=./data/game-files
+BETA_ARCHIVE_DIR=./data/beta-archive
+BODY_SIZE_LIMIT=Infinity
 
-# SQLite location
+# Prepare this runtime from Hosts before deploying. Image and game-file
-DB_PATH=/data/cherry.db
+# library versions are selected independently in the launch preview.
+PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest
 
-# Retention (days) for deployment rows and blob diffs
+# Optional shared RCON password. Native instances use independently generated
-DEPLOY_KEEP_DAYS=7
+# secrets when omitted; RCON packets are sent locally by each host agent.
+# RCON_PASSWORD=
 
-# Fleet-wide rolling restart limit, shared by deployments and betas.
+# Development tests only. Production host transport requires HTTPS.
-# A slot stays occupied until Komodo and RCON confirm recovery.
+# EC_AGENT_ALLOW_HTTP=true
-DEPLOYMENT_RESTART_CONCURRENCY=1
+# Browser login on localhost only (never for an exposed controller):
+# EC_AUTH_ALLOW_HTTP=true
 
-# New-server startup limit, shared by all collections and single-server launches.
+# Fleet concurrency and campaign retention are saved in Settings, not env vars.
-# A slot stays occupied through Komodo, RCON, and selected-map verification.
-DEPLOYMENT_CREATE_CONCURRENCY=1
 
-# Komodo
+# Optional request/report timings and event-loop delay diagnostics.
-KOMODO_ADDRESS=https://komodo.example.internal
+EC_PERF_LOG=false
-KOMODO_API_KEY=
+# Log statements taking at least this many milliseconds; 0 disables logging.
-KOMODO_API_SECRET=
+EC_SLOW_SQL_MS=0
-# Komodo server (host) that new deployments land on. Must match the exact
+# For a local SQLite disk, set to 256 for a 256 MiB memory map. Leave 0 for network mounts.
-# name shown on the Servers page in the Komodo UI. Required to deploy new
+EC_SQLITE_MMAP_MB=0
-# servers from the Fleet page.
-KOMODO_TARGET_SERVER=
-# Optional overrides for the plutainer defaults used when EC creates a new
-# server. Change these if you fork the image or use a different docker network.
-# PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest
-# PLUTAINER_NETWORK=komodo_default
-# PLUTAINER_GAMEFILES_VOLUME=t5_game_files
-# PLUTAINER_T6_GAMEFILES_VOLUME=t6_game_files
-# T6_REL_DEST_SCRIPT=runtime/plutonium/storage/t6/scripts/zm
-# T6_CONFIG_TARGET_FILE=dedicated_zm.cfg
-# Port range EC picks from when auto-assigning a game_port to a new server.
-# Must correspond to what's actually forwarded on the Komodo host firewall.
-# SERVER_PORT_MIN=28960
-# SERVER_PORT_MAX=28980
-
-# RCON — shared across the fleet (every server uses the same rcon_password).
-# Host defaults to host.docker.internal so the container can reach game
-# servers on the same Docker host; override if RCON lives elsewhere.
-RCON_PASSWORD=
-RCON_HOST=host.docker.internal
-RCON_TIMEOUT_MS=3000
-
-# HTTP (Komodo API)
-HTTP_TIMEOUT_MS=10000

.gitea/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+    checks:
+        runs-on: ubuntu-latest
+        container:
+            image: node:22-bookworm
+        steps:
+            - name: Check out repository
+              uses: https://github.com/actions/checkout@v4
+              with:
+                  persist-credentials: false
+
+            - name: Install Python and native build tools
+              run: |
+                  apt-get update
+                  apt-get install -y --no-install-recommends python3 make g++ gpg gpg-agent gpgv
+
+            - name: Install dependencies
+              run: npm ci
+
+            # Generates the SvelteKit config required by tests in a fresh checkout.
+            - name: Check styles and types
+              run: npm run check
+
+            - name: Check formatting and lint
+              run: npm run lint
+
+            - name: Run application tests
+              # Bound resource use for database-heavy fixtures on shared runners.
+              env:
+                  VITEST_MAX_WORKERS: '2'
+              run: npm test
+
+            - name: Run agent tests
+              run: npm run test:agent
+
+            - name: Build production bundles
+              run: npm run build

.gitignore

@@ -12,3 +12,5 @@ build
 data/
 test-fixtures/
 .gstack/
+__pycache__/
+*.pyc

.prettierignore

@@ -0,0 +1,5 @@
+# Prettier also respects .gitignore.
+package-lock.json
+
+# Generated by agent/package_release.py, including its checksum manifest.
+static/agent/

.prettierrc.json

@@ -0,0 +1,14 @@
+{
+    "useTabs": false,
+    "tabWidth": 4,
+    "singleQuote": true,
+    "trailingComma": "none",
+    "printWidth": 100,
+    "plugins": ["prettier-plugin-svelte"],
+    "overrides": [
+        {
+            "files": "*.svelte",
+            "options": { "parser": "svelte" }
+        }
+    ]
+}

AGENTS.md

@@ -0,0 +1,10 @@
+# Project instructions
+
+## Git workflow
+
+- Commit changes when the requested work is finished and relevant checks pass. Creating the commit is part of completing the task; do not wait for a separate request to commit.
+- Before starting a new feature, finish and commit the current work.
+- Keep unrelated features or fixes in separate commits. If a request includes two unrelated features, complete, validate, and commit each one before starting the next.
+- Stage only changes belonging to the current task. Leave unrelated pre-existing changes out of the commit.
+- Use a concise commit message that describes the change.
+- When the user links a repository issue, reference its issue number in the corresponding commit message (for example, `Refs #59`).

Dockerfile

@@ -23,12 +23,18 @@ COPY --from=build /app/build       ./build
 COPY --from=build /app/node_modules ./node_modules
 COPY --from=build /app/package.json ./package.json
 COPY --from=build /app/src/lib/server/db/schema.sql ./build/schema.sql
+COPY scripts/reset-admin.mjs ./scripts/reset-admin.mjs
+COPY scripts/database-diagnostics.mjs ./scripts/database-diagnostics.mjs
+COPY src/lib/server/auth/passwords.js ./src/lib/server/auth/passwords.js
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
 ENV NODE_ENV=production
 ENV PORT=3000
 ENV HOST=0.0.0.0
+# Legacy host imports stream whole game files; buffering routes enforce their own caps.
+ENV BODY_SIZE_LIMIT=Infinity
+ENV SQLITE_TMPDIR=/data/.sqlite-tmp
 EXPOSE 3000
 
 ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]

README.md

@@ -1,395 +1,83 @@
 # Electric Cherry
 
-Web app that manages a fleet of T5 and T6 Plutonium Zombies servers running under Komodo.
+A self-hosted dashboard for managing Plutonium T5 and T6 Zombies servers.
+Electric Cherry handles your fleet, game files, releases, and maintenance from
+one web interface. Game servers run on your hosts using Docker, Plutainer, and
+a small Python agent.
 
-Fleet membership lives in SQLite. EC registers servers during creation and removes
+![Fleet overview showing sample T5 and T6 lobbies, player counts, and match rounds](docs/assets/fleet.png)
-them during destruction; startup does not discover or import server folders.
-Existing database registrations are preserved, and missing mounts do not remove
-servers from the fleet. The legacy `seedServerRegistry` helper is an explicit
-import utility only.
 
-Status refreshes check Komodo container readiness, including nested Docker health
+_Fleet overview with sample data._
-status. A running container with `health: starting` is still **Starting**. Starting
-or restarting containers hide old game data and RCON errors; stopped, unhealthy,
-or unverifiable containers do not trigger routine RCON polling. Containers without
-a Docker health check use their running state as the readiness signal.
 
-With fresh container health, validated telemetry received in the last 45 seconds
+## Features
-supplies the current player count, roster, map, and game status. RCON is the fallback
-when telemetry is unavailable or stale. Fleet, server details, metrics, collection
-coverage, setup readiness, and beta health checks share this policy. Reports from
-before the latest start, restart, or rebuild cannot verify the new process.
-Historical scoreboards remain visible and are labeled separately from live counts;
-RCON diagnostics retain their own results. Telemetry does not report ping or player
-capacity, so those fields remain unknown in telemetry-only observations.
 
-## Fleet overview
+- Launch individual servers or saved collections of maps and lobbies.
+- Monitor players, host capacity, match history, and leaderboards.
+- Moderate players with scoped restrictions, live actions, preserved evidence and review history.
+- Manage scripts, plugins, and configuration with beta testing and rollback.
+- Queue maintenance around active games and scale spare lobbies with opt-in policies.
+- Control access with administrator, moderator, operator, and read-only accounts.
 
-Fleet searches server names, map names, and the full player roster. Filter occupied,
+## Getting started
-empty, unhealthy, testing, or pinned lobbies; switch between cards and a compact
-table; and group by game or map. Pins and display preferences are saved in this
-browser. Map groups preserve T6 variants and can preselect the exact map when
-launching another lobby.
 
-Health summaries exclude stale player counts. The match spotlight uses fresh round
+You'll need Docker Compose for the controller, an HTTPS reverse proxy, and
-telemetry from an occupied, ongoing match. Persisted campaign, beta, creation,
+Debian 12 or 13 amd64 game hosts. Bring your own game files and Plutonium server keys.
-removal, and restart-recovery state appears alongside affected servers. Fleet
-membership and operation progress refresh every ten seconds, with live server
-status between refreshes. Failed refreshes keep the last snapshot visible.
 
-## Stack
+```sh
-
+cp .env.example .env
-- SvelteKit (adapter-node) + TypeScript
+openssl rand -hex 32
-- SQLite via better-sqlite3 (replaces `script_metadata.json` as the source of
-  truth)
-- Server-sent events for live pipeline / server-status streams
-- Game telemetry for current player counts and rosters, with direct UDP RCON as fallback
-- Komodo API for restarts and per-server game-port discovery
-- Runs as a single Komodo-managed container on the same host as the game
-  servers, with `/root/zombies` bind-mounted in
-
-## Layout
-
-- `src/lib/server/db/` — schema, migrations, better-sqlite3 handle
-- `src/lib/server/fleet/` — file walking, push/beta engines
-- `src/lib/server/integrations/` — Komodo, RCON
-- `src/lib/server/pipeline/` — sweep loop, event bus, worker singleton
-- `src/lib/server/provisioning/` — lifecycle requests, queues, execution, recovery, and history
-- `src/routes/api/` — REST + SSE endpoints
-- `src/routes/` — UI pages (dashboard, campaigns, server deployments, betas, per-server)
-- `Dockerfile`, `docker-compose.yml`, `entrypoint.sh` — container build
-- `komodo/` — Komodo stack definition
-- `scripts/` — legacy JSON importer, fixture seeder
-- `tests/` — vitest unit tests
-
-## Plugin releases
-
-Plugins support named release versions, GitHub DLL imports and update discovery.
-See [Plugin releases](docs/plugin-releases.md) for the import and beta workflow.
-
-## Plutonium updates and scheduled restarts
-
-Open **Settings → Plutonium updates & restarts** to check the latest published
-revision, inspect each server's running revision, and queue a server or fleet
-restart. Release and running-version checks also run hourly. The published
-revision comes from the [official production updater manifest](https://cdn.plutoniummod.com/updater/prod/info.json),
-which exposes the game revision directly and avoids the changelog website's browser challenge;
-running revisions come from RCON `version`. Unsupported replies and failed
-queries appear as unknown rather than being treated as up to date. A failed
-manifest request retains the last release with a visible error.
-
-Daily fleet restarts are optional and disabled by default. Choose a queue time
-in your **selected timezone** and save the schedule. The timezone is saved with
-the schedule so the daily time follows daylight saving changes even when your
-browser is closed. Skipped times run at the first valid minute after the gap;
-repeated times run once on their first occurrence. Existing UTC schedules retain
-their original timing until saved again. The form displays their next occurrence
-in your timezone and identifies the saved zone if it differs. Changing a browser's
-display preference does not silently change a fleet schedule; save to apply it.
-Every due run queues the current fleet, regardless of whether an update exists.
-The schedule and jobs survive app restarts. Missed days become one run, and a
-server can have only one pending maintenance restart. Disabling the schedule
-does not cancel work already queued; use **Cancel** for those jobs.
-
-Maintenance waits for a fresh zero-player RCON or telemetry report and for existing
-deployment/beta work, then uses the shared rolling restart limit. Unreachable or
-occupied servers remain queued. After container and game-status recovery, Electric
-Cherry queries the running revision again. If a fresh published revision was
-known when queued, recovery on an older or unknown revision is reported as a
-failed verification, without an automatic restart loop. Ambiguous restart
-failures likewise require inspection before a new request. Activity is retained
-for 30 days, with pending jobs and the latest 100 records shown in Settings.
-
-Updates rely on Plutainer's startup updater (`PLUTAINER_AUTO_UPDATE` must be
-enabled); Electric Cherry does not download binaries or pull a container image.
-See [Plutainer's startup flow](https://github.com/Ayymoss/Plutainer/blob/main/CLAUDE.md)
-and [environment migration reference](https://github.com/Ayymoss/Plutainer/blob/main/MIGRATION.md).
-The worker starts on the first application request, so keep the app running and
-reachable for scheduled maintenance. Validate RCON revision replies and updater
-behavior on your live T5/T6 servers before relying on unattended maintenance.
-
-## How restarts stay safe
-
-**Operations → Watchdog** automatically detects unhealthy servers and file drift.
-Health incidents receive one restart followed by one rebuild if recovery fails;
-drift queues a rebuild with the promoted setup and a linked Campaign. Incidents,
-evidence, and attempts survive application restarts. Pause automation, inspect
-failures, and explicitly retry from Watchdog. Intentional stops and active beta or
-campaign work are respected. See [Watchdog](docs/watchdog.md) for timing and recovery behavior.
-
-Betas support configurable gameplay, participation, stability and manual-review
-requirements, saved presets, coverage rules and staged waves. See
-[Beta test plans](docs/beta-test-plans.md) for usage and the four-event game telemetry contract.
-
-File changes are staged sequentially. Their activation uses rolling restarts:
-by default only **one server across the fleet** can restart or recover at a
-time. Promotions, rollbacks, drift reconciliation, and beta restarts share this
-limit. Set `DEPLOYMENT_RESTART_CONCURRENCY` to a positive integer to raise it.
-
-The Provisioning worker runs every 15s. Campaigns, betas, and maintenance submit
-durable restart requests and read their results. Provisioning checks recovering
-servers first, then admits queued operations when there is capacity. Requests
-normally run in submission order; on a shared server, beta activation precedes
-campaign activation, and maintenance waits for both. Busy
-or unreachable servers stay queued without taking a slot, so other empty
-servers can proceed. A successful Komodo restart request retains its slot
-until the container is healthy and RCON or fresh telemetry verifies game status. Health checks start no sooner
-than 15 seconds after the request returns, avoiding an immediate check of the
-old process. An unhealthy server keeps its slot rather than letting failures
-spread across the fleet. A timed-out restart also keeps its slot because Komodo
-may have accepted the request.
-
-Slots persist in SQLite across app restarts, and overlapping timer/API triggers
-share the active run. Beta options to skip health or match gates still work;
-skipping a beta gate does not release the fleet's recovery slot early. New-server
-creation uses a separate shared startup queue, holding its slot through container,
-game-status, and selected-map verification.
-
-No token, no session, no auth freeze. If Komodo or RCON aren't configured,
-the sweep marks pending rows `manual` so nothing hangs forever.
-
-## Provisioning collections and launch previews
-
-Provisioning also offers **House Party Protocol**: review the entire fleet and queue
-destruction in one action. This includes queued creations and reuses pending
-destruction requests. Each server waits for a fresh zero-player reading; occupied
-or unreachable servers remain queued. Containers, server files, and registrations
-are removed, while saved keys and collections are retained. The preview must be
-reviewed again if fleet membership changes before confirmation. Servers created
-after confirmation are not included. Progress appears in deployment history.
-
-If deletion fails after Komodo removal, the operation detail page offers
-**Resume cleanup** to retry local file and database removal without RCON or
-repeating Komodo deletion. Requesting destruction again also resumes that cleanup.
-Recovery is blocked if a newer operation exists for the same server name.
-
-**Operations → Collections** includes a builder for whole server lineups: choose
-multiple T5/T6 maps and set the desired total lobbies for each. **Conform fleet**
-counts existing and queued servers and creates only the shortfall. The preview
-shows target/existing/missing counts and IDs, ports, and key requirements for new
-servers only. Already-satisfied targets create nothing; excess servers are kept.
-Choose **Match collection exactly** in the preview to also remove excess lobbies
-and maps outside the collection. Review the keep/create/remove lists and confirm
-removals. Matching servers stay untouched; unstarted excess creations are cancelled,
-and running servers wait for a fresh zero-player check before removal. New lobbies
-can reserve ports from these removals and wait until they are released. The collection
-page tracks both actions. **Fill missing servers** remains the default.
-All jobs and capacity reservations are committed together before provisioning;
-one collection page tracks each server's independent progress and failures.
-New servers start one at a time by default across all collections and individual
-launches. A startup slot stays occupied until Komodo is healthy and RCON or
-fresh telemetry reports the selected map; an unready server holds the queue until it recovers
-or is removed. The queue survives application restarts. The collection preview
-shows the startup limit, and its progress page shows active startups and each
-server's place in the shared queue. Set `DEPLOYMENT_CREATE_CONCURRENCY` to a
-positive integer to increase the limit. Each worker pass starts at most the
-available number of slots; immediate failures do not drain the remaining queue.
-Existing single-server collections can be expanded with **Edit**. Single-server
-launches and launching again from recorded recipes remain available.
-Collections save maps and target lobby counts. Every new server uses the latest
-promoted scripts, plugins, and configuration when provisioning begins, including
-promotions made after preview or while queued. Existing collections automatically
-follow this policy. Deployment history records the files selected for that launch;
-**Launch again** uses the current promoted setup for the same game and map.
-The progress page explains waiting conditions, shows elapsed time, and can resume
-health verification of an interrupted creation when its original server remains.
-See [Deployment collections](docs/deployment-blueprints.md) for the workflow,
-promotion policy, and API details.
-
-## Saved server keys
-
-Use **Keys** to add, replace, or remove a Plutonium key for an exact
-server ID such as `t5_zombie_theater_2`. IDs use a supported map slug and a
-lobby number from 1 to 20; the server does not need to be deployed first.
-During deployment, choose the map. The lowest available lobby is assigned
-automatically, with the exact server ID shown before deployment. Its saved key is reused automatically,
-and any newly entered key is saved for that ID, including if deployment fails.
-Destroying a server preserves its saved key. Updating or removing a saved key
-does not change a running server; changes apply to future deployments.
-
-Keys are stored in the application's SQLite database at `DB_PATH`, independently
-of server records. Preserve that database across container replacements. Key
-values are not returned by the management API, but are stored unencrypted in
-SQLite, so database files and backups contain credentials. Existing deployment
-keys are not automatically imported from Komodo; add them once to retain them.
-
-## Local dev
-
-Windows or macOS — no `/root/zombies` on disk, so seed a fixture tree first.
-
-```bash
-npm install
-npm run seed:fixture
-
-# Point env at the fixture (PowerShell)
-$env:SHARED_SCRIPTS = "$PWD/test-fixtures/zombies/shared/scripts"
-$env:SERVERS_ROOT   = "$PWD/test-fixtures/zombies/t5"
-$env:DB_PATH        = "$PWD/data/cherry.db"
-
-npm run dev
-# open http://localhost:3000
 ```
 
-Without RCON_PASSWORD / KOMODO_* set, the pipeline marks pending rows
+Edit `.env`: use the generated secret for `EC_SETUP_TOKEN`, set `EC_PUBLIC_URL`
-`manual` after a sweep — the UI still works, you just don't get real
+to your HTTPS address, and choose a persistent `EC_DATA_DIR`. Then start the
-restarts. Enough to develop the push/beta flow.
+controller and proxy your HTTPS address to port 3000:
 
-## Map selection and configs
+```sh
+docker compose up -d --build
+```
 
-The map selected when deploying a T5 zombies server controls its startup
+Open the dashboard and create your first administrator with the setup token.
-rotation automatically. No map-choice config overlay is required. New servers
+Remove the token from `.env` afterward and recreate the controller with
-use the promoted game baseline and matching overlays when available; otherwise
+`docker compose up -d`.
-Plutainer seeds its usual defaults and Electric Cherry applies the selected map
-through a generated `configs/electric_cherry_map.cfg`, loaded after the main
-config and before map rotation. This file is tracked and restored by drift
-reconciliation; operators do not need to create or promote it. The launch order
-uses Plutainer's [extra arguments and map-rotation controls](https://github.com/Ayymoss/Plutainer/blob/main/docs/configuration.md).
-Plutainer's
-default main config is preserved until a game baseline is promoted.
 
-Beta previews, promotion, and config drift repair all apply the server's stored
+1. Upload your T5 or T6 folder, including DLC, under **Releases → Game files**.
-catalog map after composing the baseline and overlays. An old `sv_maprotation`
+2. Add a machine under **Fleet → Hosts** and run its generated enrollment command.
-overlay cannot override that selection, but its other settings still apply.
+3. Save your Plutonium keys under **Admin → Keys**.
-Existing servers receive this behavior on their next config deployment or drift
+4. Wait for host setup to finish, then launch a server or collection.
-reconciliation; this change does not immediately rewrite or restart them.
-Servers with an unknown map retain their existing config behavior. Rollback
-restores the exact prior config bytes.
 
-## T6 Zombies (Black Ops II)
+See the [setup guide](docs/guides/getting-started.md) for telemetry, storage,
+and the first launch. Host agents connect outbound over HTTPS; the controller
+doesn't need a Docker socket or direct access to your game-server files.
 
-Choose **Black Ops II (T6)** in Deploy new server. T6 uses `t6_zombie_<slug>_<lobby>`
+## Documentation
-IDs, its own saved Plutonium keys, `PLUTAINER_GAME=t6zm`, and
-`configs/dedicated_zm.cfg`. Existing T5 deployments and environment settings
-keep their current paths. API clients can send `game: "t6"`; omitting it selects T5.
 
-Before the first deployment, populate the Docker volume `t6_game_files` on the
+- [Documentation index](docs/README.md) — setup, operations, releases, and analytics.
-Komodo host with your Black Ops II game files, including the DLC maps you intend
+- [Development guide](docs/development/README.md) — architecture, checks, and diagnostics.
-to host. Set `PLUTAINER_T6_GAMEFILES_VOLUME` to use another volume or an absolute
+- [Autoscaling plan](docs/plans/lobby-autoscaling-plan.md) — design and remaining pilot work.
-host directory. It is mounted read-only at `/home/plutainer/gamefiles`.
+- [Idle CPU research](docs/research/t5-t6-idle-cpu-research.md) — experiments and tradeoffs.
-Plutainer supplies its runtime and config seeds, not the base game files.
-Use a Plutonium server key created for **T6 Zombies**.
 
-Server state defaults to `/root/zombies/t6`, or the sibling `t6` directory beside
+## Development
-`SERVERS_ROOT`; override with `T6_SERVERS_ROOT`. Electric Cherry and the Komodo
-host must see this directory at the same absolute path. The supplied Compose
-mount already covers both games. GSC files go to
-`runtime/plutonium/storage/t6/scripts/zm` (no `raw/`), and game logs use
-`logs/games_zm.log`. Optional overrides are `T6_REL_DEST_SCRIPT` and
-`T6_CONFIG_TARGET_FILE`; a custom config filename needs a promoted baseline or
-a file supplied at that name, since Plutainer seeds `dedicated_zm.cfg`.
 
-The catalog includes the classic maps, Survival, Grief, and Turned selections.
+The controller uses SvelteKit, TypeScript, and SQLite. The host agent uses the
-Rotations use `execgts <ruleset>.cfg map <engine-map>`; Plutainer seeds the required
+Python 3 standard library. For local development, install Node.js 22.13+ and Python 3:
-`storage/t6/gamesettings` files. Keep seeding enabled, including with a promoted
-baseline. Town, Farm, and Bus Depot share `zm_transit`, so EC keeps their selection
-in `map_slug` through creation, beta, promotion, and drift repair. Map-scoped
-artifacts target the engine map and therefore apply to all its variants.
-Creation health checks verify the engine map reported by RCON or fresh telemetry; this does not
-independently verify the start location within that map.
 
-Both games share the fleet's port allocator (`SERVER_PORT_MIN`/`SERVER_PORT_MAX`)
+```sh
-to prevent collisions. EC explicitly sets `PLUTAINER_PORT`, so T6 need not use
+npm ci
-its upstream default of 4976. Forward the allocated UDP ports. Use a current
+export EC_SETUP_TOKEN=$(openssl rand -hex 32)
-Plutainer v2 image: EC relies on `PLUTAINER_RCON_PASSWORD`,
+export BETA_ARCHIVE_DIR=./data/beta-archive
-`PLUTAINER_MAP_ROTATE=false`, and extra arguments loaded after the main config.
+EC_AUTH_ALLOW_HTTP=true npm run dev
-Current Plutainer automatically whitelists the Docker gateway for T5/T6 RCON;
+```
-an admin client on another host needs its address allowed in the server config.
 
-Implementation references (checked September 2026):
+Open `http://localhost:3000` and use the setup token from your shell. The local
+database defaults to `./data/cherry.db`; keep it separate from production data.
+`npm run dev` packages the downloadable agent before starting the server.
 
-- [Plutainer game requirements](https://github.com/Ayymoss/Plutainer/blob/main/docs/games.md)
+```sh
-- [Plutainer launch, seeding, and RCON implementation](https://github.com/Ayymoss/Plutainer/blob/main/scripts/lib/cod.sh)
+npm run lint
-- [Plutainer configuration](https://github.com/Ayymoss/Plutainer/blob/main/docs/configuration.md)
+npm run check
-- [T6 map/ruleset combinations](https://github.com/xerxes-at/T6ServerConfigs/blob/master/localappdata/Plutonium/storage/t6/dedicated_zm.cfg)
-- [Plutonium GSC paths](https://plutonium.pw/docs/modding/loading-mods/)
-
-## Game telemetry
-
-The server script sends JSON to `POST /api/telemetry`:
-
-- `game_start`: server, map, and all players (name and GUID).
-- `round_change`: server, map, round, and all players with kills, headshots, downs, revives, and score.
-- `heartbeat`: the same fields as round change, sent periodically during a round to refresh player stats.
-- `game_end`: the same fields as round change, with final stats.
-
-These reports work with or without an active beta. Electric Cherry records
-the full player data, updates Fleet, and applies gameplay progress to active
-betas automatically. No discovery request or beta-specific fields are needed.
-
-See [Game reporter protocol](docs/beta-test-plans.md#game-reporter-protocol) for
-complete payload examples. Fresh telemetry can verify player count and game
-activity alongside RCON during provisioning and Watchdog recovery. Occupied
-reports prevent interruptions; stale reports cannot authorize them. Komodo still
-verifies container readiness.
-
-For a missing scoreboard, open the server's **Diagnostics → Telemetry API requests**.
-The log shows received requests for that exact server name, including accepted
-reports, rejected payloads, HTTP status, errors, and expandable submitted JSON.
-It refreshes every five seconds while visible and survives app restarts. Logging
-starts with this version; older requests cannot be reconstructed. See
-[Telemetry diagnostics](docs/beta-test-plans.md#telemetry-diagnostics) for retention
-and how to interpret missing or rejected reports.
-
-The **Leaderboard** records persistent player totals and map records from these
-same reports, including highest round, cumulative kills, rounds survived,
-headshots, revives, downs, and games played. See [Leaderboard](docs/leaderboard.md)
-for counting rules, history import, and the read API.
-
-## Running tests
-
-```bash
 npm test
+npm run test:agent
+npm run build
 ```
 
-Covers push (add/update/skip), beta lifecycle (open → rollback, open →
+Run `npm run format` to apply the repository's Prettier formatting.
-promote, conflict blocks), sweep (integrations-off manual branch), and
-RCON parsing (fake UDP socket).
-
-## Deploy to the Komodo host
-
-1. Build the image once locally or on the host:
-
-   ```bash
-   docker compose build
-   ```
-
-2. Import `komodo/stack.toml` into Komodo (or point Komodo at the
-   `docker-compose.yml` at repo root).
-
-3. Set the env vars in Komodo:
-
-   ```
-   KOMODO_ADDRESS, KOMODO_API_KEY, KOMODO_API_SECRET
-   RCON_PASSWORD              # single shared password across every T5/T6 server
-   RCON_HOST                  # optional; defaults to host.docker.internal
-   ```
-
-4. First boot triggers the legacy import: if
-   `/root/zombies/script_metadata.json` exists and `/data/cherry.db` is
-   empty, `entrypoint.sh` runs `scripts/import-legacy.ts` before starting
-   the server. The source JSON gets renamed to `.migrated`.
-
-5. Load the app on the LAN or over tailscale (no web-app auth by design).
-
-## Campaigns and provisioning
-
-Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility.
-
-Provisioning (`/provisioning`, `/api/provisioning`) owns Create, Start, Restart, Rebuild, Stop, and Destroy: request persistence, eligibility and occupancy checks, execution, recovery, and history. Campaigns, betas, and scheduled maintenance request lifecycle work through Provisioning and link to the resulting operation. They retain their file, test-evidence, and version-verification workflows. Pending automated restarts appear in Provisioning before execution. History survives server removal. New creation requests return HTTP 202 with an operation ID and open the provisioning detail page. Creation is complete only when Komodo reports healthy and RCON or fresh telemetry verifies game status with an engine map that exactly matches the selected map. Queued requests reserve their server name and port; queued creation and pending health checks survive application restarts. The detail page records startup progress and explains any health or map mismatch that is still blocking completion.
-
-Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh zero-player RCON or telemetry report, or a verified stopped container, before teardown. Occupied reports from either source block teardown; servers without current verification remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page.
-
-
-Use **Provisioning → Manage server**, or a server’s **Server actions**, to start,
-restart, stop, or rebuild an existing instance. **Rebuild destroys and creates in
-place**: the old container, Komodo deployment, server registration, and all server
-files are deleted. A fresh instance uses the same server ID, map, and port with
-the latest promoted setup. No old instance files are carried forward. The key
-stored separately in **Keys** must be available before teardown. Stop retains the
-instance so Start can bring it back. Disruptive operations wait for a fresh empty
-reading or a confirmed stopped container, and completion requires verification.
-See [Provisioning](docs/provisioning.md) for lifecycle behavior and recovery.
-
-Existing `/deployments` page links redirect to `/provisioning`, preserving filters
-and operation IDs. Legacy `/api/deployments` endpoints and stored history remain
-compatible.

agent/CHANGELOG.md

@@ -0,0 +1,106 @@
+# Agent changelog
+
+## 2026-10-17
+
+### 1.12.0 — Recover AFK reservations before teardown (EC #151)
+
+- Advertise `afk.settlement` and drain durable AFK decisions only after confirming
+  the game process has stopped. Removal, rebuild and creation cancellation retain
+  runtime storage until the controller acknowledges every recovery decision.
+- Authenticate recovery with the assigned host and unfinished teardown operation;
+  never follow a journal URL or use a retired game credential. Keep uncertain
+  crash durations distinct from confirmed consumption.
+- Deploy before enabling the matching AFK scripts/plugins. Package and offline-sign
+  the release before publication; no deployment or signing is performed here.
+
+## 2026-10-15
+
+### 1.11.0 — Private player moderation configuration
+
+- Advertise `moderation.configure` and accept separate generation-bound game
+  credentials through private create/rebuild configuration or the typed
+  `instance.configure-moderation` operation. Existing servers preserve startup
+  through a fixed include in their verified private bank configuration; servers
+  without a verified bootstrap require rebuilding.
+- Keep moderation credentials out of file reads, logs, manifests, staged changes,
+  shared recipes and operation receipts. Configure loads only the fixed private
+  file, never restarts gameplay, and does not replay RCON after an unknown result.
+- Controller coverage remains unavailable until the game moderation runtime
+  connects and reports its capabilities. Older agents retain normal launches.
+- Package release artifacts for offline signing before publication; do not
+  publish unsigned artifacts.
+
+## 2026-10-14
+
+### 1.10.1 — Retain verified rebuild artifacts (EC #154)
+
+- Retain the complete verified replacement file set before stopping the current
+  container, including inline files and private bank configuration, within the
+  existing 64 MiB operation limit. Staged-file comparison, replacement validation
+  and writing reuse those bytes without further artifact downloads.
+- Keep prepared bytes local to the operation. Failed and held attempts discard
+  their files; later operations prepare a fresh set. Immutable specifications,
+  journal requests and container identity labels remain unchanged.
+- Preserve checksum, path, per-file size, player-safety, bank-settlement and
+  generation checks. Add regressions for preflight failures, lost connectivity
+  after preflight, file-set limits and operation isolation.
+- Package release artifacts for offline signing before publication. No protocol
+  change or controller migration is required.
+
+## 2026-10-13
+
+### 1.10.0 — Host temperature telemetry
+
+- Read supported Linux hwmon CPU package/core, AMD physical die/CCD, NVMe and
+  Intel PCH temperatures with the Python standard library. Sensor failures,
+  faults and disabled inputs are skipped independently; unsupported hosts report
+  an empty list. AMD Tctl control offsets are excluded from physical readings.
+- Include labeled, bounded sensors in heartbeats with the `host.temperatures`
+  capability. Physical device/channel identities survive hwmon renumbering.
+- The controller shows current CPU readings and expandable sensor details, and
+  adds temperature history to Capacity. Fleet history has one CPU line per host;
+  individual hosts can add NVMe composite and chipset lines. History retains
+  minute peaks using the host-resource retention setting; missing data stays
+  unavailable and no thermal alerts or placement rules are introduced.
+- Adds database migration 23. Existing agents remain compatible. Release
+  artifacts are packaged and the stale signature is removed. Offline signing
+  and host rollout are still required; neither was performed here.
+
+## 2026-10-12
+
+### 1.9.3 — Protect private bank configuration (EC #152)
+
+- Exclude the generated bank config from file manifests and reject file reads,
+  staged changes, stale activations, rollback snapshots and log reads targeting
+  it. Creation and rebuilding still inject the private instance credential.
+- Package and offline-sign this release, then update agents before resuming
+  drift recovery. Old private-file stages fail safely and require a fresh plan.
+
+### 1.9.1 — Public game-bank endpoint
+
+- Companion to [TWS scripts #13](https://gitea.thewrightserver.net/josh/tws-zombies-scripts/issues/13).
+  In `agent/ec_agent.py`, stop generating `tws_bank_url` in
+  `electric_cherry_bank.cfg`; continue injecting the private per-instance token.
+  Launch configuration still supplies `tws_server_id`. The host's enrolled
+  controller address and authentication remain unchanged.
+- Recognize journals for `https://ec.thewrightserver.net/api/bank` and the legacy
+  bank URL derived from the enrolled controller. Reject other URLs and preserve
+  unresolved storage. Settlement still goes only through the enrolled controller's
+  authenticated, operation- and instance-scoped API. Original transaction IDs and
+  commit/cancel decisions are preserved across lost replies and retries.
+- Compatibility: pair with T5/T6 bank scripts `2.0.0` and existing TWS plugins
+  `1.1.0+`. No new Python dependency, controller migration or protocol change.
+  Old scripts still fall back to the public endpoint when the dvar is absent.
+- Deployment: package and offline-sign the release, update/restart host agents,
+  then rebuild through EC when a server's bank endpoint changes. Rebuild stops
+  the game and drains old decisions before starting the replacement or deleting
+  storage. An endpoint that already matches needs only a map restart for the
+  script update. Follow the [bank rollout procedure](../docs/guides/bank.md#upgrading-from-a-controller-derived-game-endpoint).
+  Packaging refreshes `static/agent/ec-agent.py` and `manifest.json` and removes
+  the old signature; a new offline signature is required before publishing.
+  No signing or deployment was performed.
+- Regression coverage in `agent/tests/test_agent.py`: generated credentials,
+  public and legacy journals for both games, rebuild/removal, lost replies,
+  agent restart, unchanged identities and rejection of untrusted endpoints.
+  Live Windows/Plutonium endpoint migration and signed host rollout remain to
+  be validated.

agent/ec_agent_update.py

@@ -0,0 +1,529 @@
+#!/usr/bin/env python3
+"""Stable Electric Cherry agent updater; only verified agent code is replaced.
+
+The installer owns this bootstrap and its systemd timer. Updates never execute
+downloaded code before installing it, alter game containers, or edit enrollment
+credentials. A durable journal makes an interrupted switch roll back safely.
+"""
+
+import ast
+import contextlib
+import fcntl
+import hashlib
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.parse
+import urllib.request
+import uuid
+
+UPDATER_PROTOCOL_VERSION = 2
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+# This repository is the trust authority, independent of controller configuration.
+RELEASE_KEY_URL = "https://gitea.thewrightserver.net/josh/electric-cherry/raw/branch/main/agent/release-signing.gpg"
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_LOCK = Path("/etc/electric-cherry/update.lock")
+UPDATE_DIR = Path("/etc/electric-cherry/update")
+UPDATE_JOURNAL = UPDATE_DIR / "journal.json"
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+AGENT_SERVICE = "electric-cherry-agent.service"
+MAX_MANIFEST = 16 * 1024
+MAX_SIGNATURE = 16 * 1024
+MAX_KEYRING = 256 * 1024
+MAX_SOURCE = 8 * 1024 * 1024
+HEALTH_TIMEOUT = 90
+VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class UpdateError(Exception):
+    pass
+
+
+class IntegrityError(UpdateError):
+    pass
+
+
+class ContactError(UpdateError):
+    pass
+
+
+class Deferred(UpdateError):
+    pass
+
+
+def trusted(path, missing=False, directory=False):
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise UpdateError("Agent update files are unavailable; repair the installation") from None
+        expected_type = stat.S_ISDIR if current != path or directory else stat.S_ISREG
+        if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise UpdateError("Agent update paths must be regular root-owned files and directories that other users cannot modify")
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def atomic_bytes(path, value, mode=0o600):
+    path = trusted(path, missing=True)
+    trusted(path.parent, directory=True)
+    # Code staging uses known names so recovery and host removal can also
+    # remove a file left behind by a power loss during the atomic write.
+    temporary = path.with_name(path.name + ".update") if path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE) else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
+    try:
+        if temporary.exists() or temporary.is_symlink():
+            trusted(temporary)
+            temporary.unlink()
+            sync_parent(temporary)
+        descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        trusted(path, missing=True)
+        os.replace(temporary, path)
+        sync_parent(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def atomic_json(path, value):
+    atomic_bytes(path, json.dumps(value, sort_keys=True, allow_nan=False).encode())
+
+
+def read_bytes(path, limit):
+    trusted(path)
+    with open(path, "rb") as stream:
+        value = stream.read(limit + 1)
+    if len(value) > limit:
+        raise UpdateError("An agent update file exceeds its size limit")
+    return value
+
+
+def read_json(path, limit=MAX_MANIFEST):
+    try:
+        value = json.loads(read_bytes(path, limit))
+    except (ValueError, UnicodeError):
+        raise UpdateError("Agent update metadata is invalid") from None
+    if not isinstance(value, dict):
+        raise UpdateError("Agent update metadata is invalid")
+    return value
+
+
+def sha256(value):
+    return hashlib.sha256(value).hexdigest()
+
+
+def version(value):
+    if not isinstance(value, str) or not VERSION_RE.fullmatch(value):
+        raise IntegrityError("Agent releases must use a numeric major.minor.patch version")
+    return tuple(int(part) for part in value.split("."))
+
+
+def source_version(source):
+    try:
+        tree = ast.parse(source, filename="electric-cherry-agent.py")
+        compile(tree, "electric-cherry-agent.py", "exec")
+    except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
+        raise IntegrityError("The agent release is not valid Python source") from None
+    declarations = [node.value.value for node in tree.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == "VERSION" for target in node.targets) and isinstance(node.value, ast.Constant)]
+    if len(declarations) != 1:
+        raise IntegrityError("The agent release must declare one explicit VERSION")
+    version(declarations[0])
+    return declarations[0]
+
+
+def fetch_url(url, limit, source):
+    request = urllib.request.Request(url, headers={"User-Agent": "electric-cherry-updater/2", "Cache-Control": "no-cache"})
+    class NoRedirect(urllib.request.HTTPRedirectHandler):
+        def redirect_request(self, req, fp, code, msg, headers, newurl):
+            raise ContactError("Agent release redirects are not allowed")
+    try:
+        with urllib.request.build_opener(NoRedirect()).open(request, timeout=25) as response:
+            if response.status != 200:
+                raise ContactError(f"The {source} did not return a complete download")
+            length = response.headers.get("Content-Length")
+            if length is not None and (not length.isdecimal() or int(length) > limit):
+                raise IntegrityError(f"The {source} exceeds its download size limit")
+            chunks, total = [], 0
+            deadline = time.monotonic() + 120
+            while True:
+                if time.monotonic() >= deadline:
+                    raise ContactError(f"The {source} download timed out")
+                # HTTPResponse.read1 performs one socket read. read(n) can
+                # wait indefinitely for n bytes from a trickling server.
+                block = response.read1(min(64 * 1024, limit + 1 - total))
+                if not block:
+                    break
+                total += len(block)
+                if total > limit:
+                    raise IntegrityError(f"The {source} exceeds its download size limit")
+                chunks.append(block)
+            if length is not None and total != int(length):
+                raise ContactError(f"The {source} download was interrupted")
+            return b"".join(chunks)
+    except (OSError, ValueError, http.client.HTTPException):
+        raise ContactError(f"The {source} is unavailable; the running agent was left unchanged") from None
+
+
+def fetch_release_key():
+    # Fetch afresh before every release check, including checks with no new
+    # version. There is deliberately no persistent keyring or offline fallback.
+    key = fetch_url(RELEASE_KEY_URL, MAX_KEYRING, "release-signing repository")
+    if not key:
+        raise IntegrityError("The release-signing repository returned an empty keyring")
+    return key
+
+
+def verify_release_signature(manifest, signature, public_key):
+    if not public_key or len(public_key) > MAX_KEYRING:
+        raise IntegrityError("The repository release-signing keyring is empty or too large")
+    if not signature:
+        raise IntegrityError("The controller release requires a detached manifest signature")
+    trusted(UPDATE_DIR, directory=True)
+    # An isolated home excludes any ambient GnuPG keyring or configuration.
+    # The exact downloaded bytes are verified before manifest JSON is parsed.
+    with tempfile.TemporaryDirectory(prefix="verify-", dir=UPDATE_DIR) as temporary:
+        home = Path(temporary)
+        keyring = home / "release-signing.gpg"
+        keyring.write_bytes(public_key)
+        manifest_path = home / "manifest.json"
+        signature_path = home / "manifest.json.sig"
+        manifest_path.write_bytes(manifest)
+        signature_path.write_bytes(signature)
+        try:
+            result = subprocess.run(["gpgv", "--homedir", str(home), "--keyring", str(keyring), str(signature_path), str(manifest_path)], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, check=False)
+        except (OSError, subprocess.TimeoutExpired):
+            raise IntegrityError("Release signature verification is unavailable; install or repair gpgv on this host") from None
+        if result.returncode:
+            raise IntegrityError("The release manifest signature is invalid or is not signed by a trusted release key")
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    trusted(path, missing=True)
+    trusted(Path(path).parent, directory=True)
+    descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+    with os.fdopen(descriptor, "r+") as stream:
+        try:
+            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        try:
+            yield stream
+        finally:
+            fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise UpdateError("Agent service management is unavailable") from None
+    if result.returncode:
+        raise UpdateError("Agent service management did not complete")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def service_state():
+    output = systemctl("show", AGENT_SERVICE, "--property=ActiveState", "--property=MainPID")
+    fields = dict(line.split("=", 1) for line in output.splitlines() if "=" in line)
+    try:
+        return fields.get("ActiveState") == "active", int(fields.get("MainPID", "0"))
+    except ValueError:
+        return False, 0
+
+
+class Updater:
+    def __init__(self):
+        self.installation = None
+        self.current_version = None
+        self.target = None
+
+    def load_installation(self):
+        config = read_json(AGENT_CONFIG, 64 * 1024)
+        controller = config.get("controller")
+        root_value = config.get("root")
+        host_id = config.get("hostId")
+        if not isinstance(controller, str) or not isinstance(root_value, str) or not isinstance(host_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", host_id):
+            raise UpdateError("Agent enrollment is incomplete")
+        parsed = urllib.parse.urlsplit(controller)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise UpdateError("Agent updates require an HTTPS controller without embedded credentials")
+        root = Path(root_value)
+        if not re.fullmatch(r"/[A-Za-z0-9/_.-]+", root_value) or str(root) != root_value or ".." in root.parts or len(root.parts) < 3:
+            raise UpdateError("The enrolled agent data directory is invalid")
+        trusted(root, directory=True)
+        trusted(root / "agent", directory=True)
+        marker = read_json(root / ".electric-cherry.json")
+        controller = controller.rstrip("/")
+        expected = {"version": 1, "root": root_value, "hostId": host_id, "controller": controller}
+        if marker != expected:
+            raise UpdateError("Agent data ownership does not match this enrollment")
+        info = root.stat()
+        installation = {"hostId": host_id, "controller": controller, "root": root_value, "rootIdentity": [info.st_dev, info.st_ino]}
+        if self.installation is not None and installation != self.installation:
+            raise UpdateError("Agent enrollment changed during the update; no code was replaced")
+        self.installation = installation
+        return installation
+
+    def ensure_directory(self):
+        trusted(UPDATE_DIR, missing=True, directory=True)
+        trusted(UPDATE_DIR.parent, directory=True)
+        if not UPDATE_DIR.exists():
+            UPDATE_DIR.mkdir(mode=0o700)
+            sync_parent(UPDATE_DIR)
+
+    def check_uninstall(self):
+        if UNINSTALL_REQUEST.exists() or UNINSTALL_REQUEST.is_symlink():
+            raise Deferred("Host deletion is pending; agent updates are paused")
+
+    def status(self, state, error=None, current=None):
+        value = {"state": state, "checkedAt": time.time()}
+        if current or self.current_version:
+            value["currentVersion"] = current or self.current_version
+        if self.target:
+            value["targetVersion"] = self.target["version"]
+        if error:
+            value["error"] = str(error)[:500]
+        atomic_json(UPDATE_STATUS, value)
+        return state
+
+    def fetch(self, path, limit):
+        return fetch_url(self.installation["controller"] + path, limit, "controller release")
+
+    def manifest(self):
+        public_key = fetch_release_key()
+        manifest = self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST)
+        try:
+            signature = self.fetch("/api/agent/release/manifest.json.sig", MAX_SIGNATURE)
+        except ContactError:
+            raise IntegrityError("The controller release signature is unavailable; a signed release is required") from None
+        self.ensure_directory()
+        verify_release_signature(manifest, signature, public_key)
+        try:
+            value = json.loads(manifest)
+        except (ValueError, UnicodeError):
+            raise IntegrityError("The controller release manifest is invalid") from None
+        if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64":
+            raise IntegrityError("The controller release is not compatible with this host agent")
+        if type(value.get("updaterProtocolVersion")) is not int or value["updaterProtocolVersion"] != UPDATER_PROTOCOL_VERSION:
+            raise Deferred("This release needs a newer updater bootstrap; rerun the host installer")
+        version(value.get("version"))
+        if not isinstance(value.get("sha256"), str) or not DIGEST_RE.fullmatch(value["sha256"]):
+            raise IntegrityError("The controller release manifest has an invalid checksum")
+        return {"version": value["version"], "sha256": value["sha256"]}
+
+    def hold(self, target, error):
+        self.ensure_directory()
+        atomic_json(UPDATE_DIR / "failed-release.json", {**target, "error": error, "failedAt": time.time()})
+
+    def release_held(self):
+        path = UPDATE_DIR / "failed-release.json"
+        if not path.exists():
+            return False
+        held = read_json(path)
+        return all(held.get(key) == self.target[key] for key in ("version", "sha256"))
+
+    def save_journal(self, journal, **changes):
+        journal.update(changes)
+        atomic_json(UPDATE_JOURNAL, journal)
+
+    def read_journal(self):
+        journal = read_json(UPDATE_JOURNAL)
+        if journal.get("installation") != self.installation or journal.get("phase") not in ("installing", "verifying", "rollback") or not isinstance(journal.get("id"), str) or not re.fullmatch(r"[a-f0-9]{32}", journal["id"]):
+            raise UpdateError("Pending agent update ownership or recovery metadata is invalid")
+        version(journal.get("previousVersion"))
+        version(journal.get("targetVersion"))
+        for field in ("targetSha256", "agentBackupSha256", "uninstallBackupSha256"):
+            if not isinstance(journal.get(field), str) or not DIGEST_RE.fullmatch(journal[field]):
+                raise UpdateError("Pending agent update recovery checksums are invalid")
+        return journal
+
+    def finish_journal(self):
+        trusted(UPDATE_JOURNAL)
+        UPDATE_JOURNAL.unlink()
+        sync_parent(UPDATE_JOURNAL)
+        # Removing the journal commits the decision. Backups are disposable
+        # only afterward, so a crash never leaves recovery without its inputs.
+        for name in ("agent.backup.py", "uninstall.backup.py", "candidate.py"):
+            path = trusted(UPDATE_DIR / name, missing=True)
+            path.unlink(missing_ok=True)
+        sync_parent(UPDATE_JOURNAL)
+
+    def activity_lock(self):
+        return exclusive_lock(Path(self.installation["root"]) / "agent" / "activity.lock")
+
+    def install(self, source):
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("The agent is executing an operation; update will retry when it is idle")
+            self.check_uninstall()
+            self.load_installation()
+            active, pid = service_state()
+            if not active or pid <= 0:
+                raise Deferred("Agent service is not running; automatic updates wait until it is started")
+            previous = read_bytes(AGENT_EXECUTABLE, MAX_SOURCE)
+            uninstall = read_bytes(UNINSTALL_EXECUTABLE, MAX_SOURCE)
+            if source_version(previous) != self.current_version:
+                raise Deferred("The installed agent changed while checking for updates")
+            source_version(uninstall)
+            self.ensure_directory()
+            atomic_bytes(UPDATE_DIR / "agent.backup.py", previous)
+            atomic_bytes(UPDATE_DIR / "uninstall.backup.py", uninstall)
+            atomic_bytes(UPDATE_DIR / "candidate.py", source)
+            journal = {"id": uuid.uuid4().hex, "phase": "installing", "installation": self.installation, "previousVersion": self.current_version, "targetVersion": self.target["version"], "targetSha256": self.target["sha256"], "agentBackupSha256": sha256(previous), "uninstallBackupSha256": sha256(uninstall), "startedAt": time.time()}
+            self.save_journal(journal)
+            self.status("updating")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            atomic_bytes(AGENT_EXECUTABLE, source, 0o755)
+            atomic_bytes(UNINSTALL_EXECUTABLE, source, 0o755)
+            self.save_journal(journal, phase="verifying", switchedAt=time.time())
+        # The candidate takes the same lock during startup credential recovery.
+        # Never wait for its start or authenticated heartbeat while holding it.
+        systemctl("start", AGENT_SERVICE)
+        if not self.wait_healthy(journal):
+            raise UpdateError("The new agent did not report a healthy authenticated heartbeat; the previous release will be restored")
+        self.check_uninstall()
+        self.load_installation()
+        if any(sha256(read_bytes(path, MAX_SOURCE)) != journal["targetSha256"] for path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE)):
+            raise UpdateError("Installed agent code changed before update verification finished")
+        self.status("updated", current=journal["targetVersion"])
+        self.current_version = journal["targetVersion"]
+        self.finish_journal()
+        return "updated"
+
+    def wait_healthy(self, journal):
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            self.check_uninstall()
+            try:
+                active, pid = service_state()
+                health = read_json(AGENT_HEALTH, 4096)
+                checked = health.get("checkedAt")
+                if active and pid > 0 and type(health.get("pid")) is int and health["pid"] == pid and health.get("version") == journal["targetVersion"] and health.get("updateId") == journal["id"] and type(checked) in (float, int) and math.isfinite(checked) and journal["switchedAt"] <= checked <= time.time() + 5:
+                    return True
+            except (UpdateError, OSError, ValueError):
+                pass
+            time.sleep(2)
+        return False
+
+    def rollback(self, journal, error):
+        self.target = {"version": journal["targetVersion"], "sha256": journal["targetSha256"]}
+        self.check_uninstall()
+        self.load_installation()
+        if journal["installation"] != self.installation:
+            raise UpdateError("Agent enrollment changed; rollback needs operator review")
+        backups = [(AGENT_EXECUTABLE, "agent.backup.py", "agentBackupSha256"), (UNINSTALL_EXECUTABLE, "uninstall.backup.py", "uninstallBackupSha256")]
+        sources = []
+        for destination, name, checksum in backups:
+            source = read_bytes(UPDATE_DIR / name, MAX_SOURCE)
+            if sha256(source) != journal[checksum]:
+                raise UpdateError("Agent rollback backup failed integrity verification; repair the installation")
+            source_version(source)
+            sources.append((destination, source))
+        if source_version(sources[0][1]) != journal["previousVersion"]:
+            raise UpdateError("Agent rollback version does not match its recovery journal")
+        self.save_journal(journal, phase="rollback")
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("Agent rollback is waiting for the active operation to finish")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            for destination, source in sources:
+                atomic_bytes(destination, source, 0o755)
+            self.save_journal(journal, rollbackSwitchedAt=time.time())
+        systemctl("start", AGENT_SERVICE)
+        rollback_health = {**journal, "targetVersion": journal["previousVersion"], "switchedAt": journal["rollbackSwitchedAt"]}
+        if not self.wait_healthy(rollback_health):
+            raise UpdateError("The previous agent release was restored but its authenticated health is unconfirmed; recovery will retry")
+        self.hold(self.target, error)
+        self.status("failed", error, current=journal["previousVersion"])
+        self.current_version = journal["previousVersion"]
+        self.finish_journal()
+        return "failed"
+
+    def run(self):
+        with exclusive_lock(UPDATE_LOCK) as updater:
+            if updater is None:
+                return "deferred"  # Preserve the active worker's status.
+            recovering = False
+            try:
+                self.check_uninstall()
+                self.load_installation()
+                if UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink():
+                    journal = self.read_journal()
+                    self.current_version = journal["previousVersion"]
+                    recovering = True
+                    return self.rollback(journal, "An interrupted agent update was rolled back to the previous release")
+                self.current_version = source_version(read_bytes(AGENT_EXECUTABLE, MAX_SOURCE))
+                self.status("checking")
+                self.target = self.manifest()
+                if version(self.target["version"]) <= version(self.current_version):
+                    return self.status("current")
+                if self.release_held():
+                    return self.status("failed", "This release previously failed verification and is held until the controller publishes a different release")
+                source = self.fetch("/api/agent/release/ec-agent.py", MAX_SOURCE)
+                if sha256(source) != self.target["sha256"]:
+                    raise IntegrityError("The agent release checksum does not match its manifest")
+                if source_version(source) != self.target["version"]:
+                    raise IntegrityError("The agent source VERSION does not match its release manifest")
+                return self.install(source)
+            except Deferred as error:
+                return self.status("deferred", error)
+            except ContactError as error:
+                return self.status("deferred", error)
+            except (UpdateError, OSError, ValueError, KeyError) as error:
+                message = str(error) if isinstance(error, UpdateError) else "Agent update could not finish; inspect the updater service"
+                if UPDATE_JOURNAL.exists():
+                    if recovering:
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                    try:
+                        return self.rollback(self.read_journal(), message)
+                    except Deferred as deferred:
+                        return self.status("deferred", deferred)
+                    except (UpdateError, OSError, ValueError, KeyError):
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                if isinstance(error, IntegrityError) and self.target:
+                    self.hold(self.target, message)
+                return self.status("failed", message)
+
+
+def main():
+    if os.geteuid() != 0:
+        raise UpdateError("Run the agent updater through its installed root service")
+    Updater().run()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (UpdateError, OSError, ValueError, KeyError):
+        print("Electric Cherry agent updater failed; inspect its installation and update status", file=sys.stderr)
+        sys.exit(1)

agent/install.sh

@@ -0,0 +1,395 @@
+#!/bin/bash
+# Electric Cherry host installer. Downloaded release integrity is checked before
+# execution. Existing Docker data and unrelated containers are never changed.
+set -euo pipefail
+umask 077
+
+controller=''
+enrollment_token=''
+data_dir='/var/lib/electric-cherry'
+reenroll=''
+existing_enrollment=false
+existing_data_root=false
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        --controller) controller="${2:?Missing controller URL}"; shift 2 ;;
+        --token) enrollment_token="${2:?Missing enrollment token}"; shift 2 ;;
+        --data-dir) data_dir="${2:?Missing data directory}"; shift 2 ;;
+        --reenroll) reenroll='--force-enroll'; shift ;;
+        *) echo "Unknown installer option" >&2; exit 2 ;;
+    esac
+done
+[ "$(id -u)" = 0 ] || { echo 'Run the installer as root.' >&2; exit 1; }
+case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
+[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
+case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
+case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
+[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
+. /etc/os-release
+[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
+case "${VERSION_ID:-}" in 12|13) ;; *) echo 'Supported platform: Debian 12 or 13.' >&2; exit 1 ;; esac
+[ "$(dpkg --print-architecture)" = amd64 ] || { echo 'Supported architecture: amd64.' >&2; exit 1; }
+command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux gpgv
+installation_preflight() {
+    if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
+        echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+    if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
+        echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
+import json, pathlib, re, stat, sys
+
+controller, raw_root, config_name = sys.argv[1:]
+root = pathlib.Path(raw_root)
+config = pathlib.Path(config_name)
+reserved = {
+    '/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
+    '/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
+    '/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
+    '/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
+}
+if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
+        or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
+        or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
+    raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
+installed_paths = [config,
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
+if any(root == path or root in path.parents for path in installed_paths):
+    raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
+
+def trusted_directories(path):
+    for current in [*reversed(path.parents), path]:
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            continue
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
+
+def trusted_file(path):
+    info = path.lstat()
+    if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+        raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+
+def trusted_json(path):
+    trusted_file(path)
+    value = json.loads(path.read_text())
+    if not isinstance(value, dict):
+        raise SystemExit('Existing installation metadata is invalid.')
+    return value
+
+trusted_directories(root)
+trusted_directories(config.parent)
+mountinfo = pathlib.Path('/proc/1/mountinfo')
+if mountinfo.exists():
+    for line in mountinfo.read_text().splitlines():
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == raw_root or mount.startswith(raw_root + '/'):
+            raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
+request = config.with_name('decommission.json')
+if request.exists() or request.is_symlink():
+    raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+trusted_directories(config.parent / 'update')
+journal = config.parent / 'update' / 'journal.json'
+if journal.exists() or journal.is_symlink():
+    raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
+update_lock = config.with_name('update.lock')
+if update_lock.exists() or update_lock.is_symlink():
+    trusted_file(update_lock)
+saved = None
+if config.exists() or config.is_symlink():
+    saved = trusted_json(config)
+    if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
+        raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
+    if not saved.get('hostId') or not saved.get('credential'):
+        raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
+marker = root / '.electric-cherry.json'
+if marker.exists() or marker.is_symlink():
+    owned = trusted_json(marker)
+    if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
+        raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
+elif root.exists() and any(root.iterdir()):
+    # Upgrade installations predating ownership markers only when the entire
+    # root has the known layout. Never adopt a directory of unrelated files.
+    if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
+        raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
+for directory in ('agent', 'instances', 'libraries'):
+    trusted_directories(root / directory)
+activity_lock = root / 'agent' / 'activity.lock'
+if activity_lock.exists() or activity_lock.is_symlink():
+    trusted_file(activity_lock)
+PY
+}
+if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
+if [ -d "$data_dir" ]; then existing_data_root=true; fi
+installation_preflight
+temporary_dir=$(mktemp -d /tmp/electric-cherry-install.XXXXXXXXXX)
+updater_timer_was_active=false
+update_lock_held=false
+activity_lock_held=false
+installer_finished=false
+installation_cleanup() {
+    local ec_install_status=$?
+    trap - EXIT
+    if [ "$activity_lock_held" = true ]; then
+        flock --unlock 8 || true
+        exec 8>&-
+    fi
+    if [ "$update_lock_held" = true ]; then
+        flock --unlock 9 || true
+        exec 9>&-
+    fi
+    # Recovery must remain scheduled when an upgrade or its second preflight
+    # fails. Release the worker lock before restarting the prior active timer.
+    if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
+        systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
+    fi
+    rm -rf "$temporary_dir" || true
+    exit "$ec_install_status"
+}
+trap installation_cleanup EXIT
+# INSTALLATION_TRUST: always fetch directly from the repository, never the controller.
+key_status=$(curl --proto '=https' --tlsv1.2 --fail --silent --show-error \
+    --connect-timeout 25 --max-time 120 --max-filesize 262144 \
+    --header 'Cache-Control: no-cache' --write-out '%{http_code}' \
+    'https://gitea.thewrightserver.net/josh/electric-cherry/raw/branch/main/agent/release-signing.gpg' \
+    -o "$temporary_dir/release-signing.gpg")
+[ "$key_status" = 200 ] && [ -s "$temporary_dir/release-signing.gpg" ] || {
+    echo 'The release-signing repository did not return a public key. Installation stopped.' >&2
+    exit 1
+}
+# INSTALLATION_TRUST_END
+controller="${controller%/}"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json.sig" -o "$temporary_dir/manifest.json.sig"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
+python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" "$temporary_dir/release-signing.gpg" <<'PY'
+# INSTALLATION_RELEASE: check both downloads before installing or executing either.
+import ast, hashlib, json, pathlib, re, stat, subprocess, sys, tempfile
+manifest_path = pathlib.Path(sys.argv[1])
+manifest_bytes = manifest_path.read_bytes()
+signature = manifest_path.with_name('manifest.json.sig')
+keyring = pathlib.Path(sys.argv[4])
+# The key lives in our private root-owned mktemp directory under sticky /tmp.
+# /tmp itself is intentionally writable; require protection of the directory
+# and file we created, rather than accepting a pre-existing host keyring.
+for path in (keyring.parent, keyring):
+    info = path.lstat()
+    expected_type = stat.S_ISREG if path == keyring else stat.S_ISDIR
+    if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+        raise SystemExit('Release signing key must be a regular root-owned file in protected directories.')
+if not 0 < keyring.stat().st_size <= 256 * 1024:
+    raise SystemExit('Repository public key is empty or exceeds its size limit.')
+if len(manifest_bytes) > 16 * 1024 or signature.stat().st_size > 16 * 1024:
+    raise SystemExit('Release signing metadata exceeds its size limit.')
+try:
+    with tempfile.TemporaryDirectory(prefix='gpgv-', dir=manifest_path.parent) as home:
+        verified = subprocess.run(['gpgv', '--homedir', home, '--keyring', str(keyring),
+                                   str(signature), '-'], input=manifest_bytes,
+                                  stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                                  timeout=30, check=False)
+except (OSError, subprocess.TimeoutExpired):
+    raise SystemExit('Release signature verification requires a working gpgv installation.') from None
+if verified.returncode:
+    raise SystemExit('Release signature verification failed. The installed agent was left unchanged.')
+manifest = json.loads(manifest_bytes)
+if not isinstance(manifest, dict):
+    raise SystemExit('Unsupported agent or updater release manifest.')
+agent = pathlib.Path(sys.argv[2]).read_bytes()
+updater = pathlib.Path(sys.argv[3]).read_bytes()
+if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
+    raise SystemExit('Agent release checksum verification failed.')
+if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
+    raise SystemExit('Updater release checksum verification failed.')
+if manifest.get('platform') != 'debian-amd64' or type(manifest.get('protocolVersion')) is not int or manifest['protocolVersion'] != 1 or type(manifest.get('updaterProtocolVersion')) is not int or manifest['updaterProtocolVersion'] != 2:
+    raise SystemExit('Unsupported agent or updater release manifest.')
+versions = [node.value.value for node in ast.parse(agent).body
+            if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
+            and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
+    raise SystemExit('Agent release version does not match its manifest.')
+compile(agent, 'ec-agent.py', 'exec')
+compile(updater, 'ec-agent-update.py', 'exec')
+PY
+# INSTALLATION_DOCKER: Debian 13 splits the client out of docker.io.
+if ! command -v docker >/dev/null; then
+    if [ "$VERSION_ID" = 13 ] && [ "$(dpkg-query -W -f='${Status}' docker.io 2>/dev/null)" = 'install ok installed' ]; then
+        # Recover an earlier daemon-only install, even if it created Docker data.
+        # Add only the client; do not replace or restart the existing engine.
+        apt-get install -y --no-install-recommends --no-remove docker-cli
+    elif [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
+        echo 'Existing Docker data/configuration detected without a Docker CLI. Repair Docker, then rerun.' >&2
+        exit 1
+    else
+        docker_packages=(docker.io)
+        if [ "$VERSION_ID" = 13 ]; then docker_packages+=(docker-cli); fi
+        apt-get install -y --no-install-recommends "${docker_packages[@]}"
+        systemctl enable --now docker
+    fi
+fi
+command -v docker >/dev/null || { echo 'Docker CLI is missing after installation. Repair the Docker client package, then rerun.' >&2; exit 1; }
+docker info >/dev/null || { echo 'Docker is installed but unavailable. Inspect systemctl status docker and journalctl -u docker, then rerun.' >&2; exit 1; }
+install -d -m 0700 /etc/electric-cherry
+# Wait for an already-running updater to finish; never interrupt its rollback.
+# The same lock also prevents a timer firing while files are being installed.
+if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
+    if systemctl is-active --quiet electric-cherry-update.timer; then
+        updater_timer_was_active=true
+    fi
+    systemctl stop electric-cherry-update.timer
+fi
+exec 9>/etc/electric-cherry/update.lock
+flock --exclusive 9
+update_lock_held=true
+installation_preflight
+# Serialize with running operations, especially a deletion request that could
+# otherwise start between preflight and stopping the main agent.
+if [ -d "$data_dir/agent" ]; then
+    exec 8>"$data_dir/agent/activity.lock"
+    flock --exclusive 8
+    activity_lock_held=true
+    installation_preflight
+fi
+# Stop only the agent while replacing its own executable/journal owner. Existing
+# game containers keep running under Docker throughout installation and updates.
+if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
+    systemctl stop electric-cherry-agent.service
+fi
+installation_preflight
+install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
+install -d -m 0700 "$data_dir/agent"
+available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
+[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
+install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
+    install -d -m 0755 /usr/local/libexec
+    install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
+}
+# A separate worker survives stopping/removing the agent service. Its only
+# entry point is a fixed, authenticated deletion request prepared by the agent.
+install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
+install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
+printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
+unset enrollment_token
+python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
+python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
+import json, os, pathlib, sys, uuid
+root = pathlib.Path(sys.argv[1])
+config = json.loads(pathlib.Path(sys.argv[2]).read_text())
+if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
+    raise SystemExit('Enrollment did not establish ownership of this data directory.')
+marker = root / '.electric-cherry.json'
+temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
+try:
+    with temporary.open('x') as stream:
+        os.chmod(temporary, 0o600)
+        json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
+        stream.flush()
+        os.fsync(stream.fileno())
+    os.replace(temporary, marker)
+    directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory_fd)
+    finally:
+        os.close(directory_fd)
+finally:
+    temporary.unlink(missing_ok=True)
+PY
+cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
+[Unit]
+Description=Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-agent.py --config /etc/electric-cherry/agent.json
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+UMask=0077
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=$data_dir /etc/electric-cherry
+
+[Install]
+WantedBy=multi-user.target
+EOF
+cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
+[Unit]
+Description=Remove an Electric Cherry Host Installation
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
+Restart=on-failure
+RestartSec=15
+TimeoutStartSec=0
+UMask=0077
+
+[Install]
+WantedBy=multi-user.target
+EOF
+cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
+[Unit]
+Description=Update the Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
+TimeoutStartSec=0
+UMask=0077
+EOF
+cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
+[Unit]
+Description=Check for Electric Cherry Host Agent Updates
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=30s
+AccuracySec=1s
+Unit=electric-cherry-update.service
+
+[Install]
+WantedBy=timers.target
+EOF
+systemctl daemon-reload
+systemctl enable electric-cherry-agent
+if [ "$activity_lock_held" = true ]; then
+    flock --unlock 8
+    exec 8>&-
+    activity_lock_held=false
+fi
+systemctl restart electric-cherry-agent
+systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
+systemctl enable --now electric-cherry-update.timer
+installer_finished=true
+echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'

agent/package_release.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Publish the reviewed local agent source into the controller static release."""
+import ast
+import hashlib
+import json
+from pathlib import Path
+import re
+
+
+def manifest_bytes(agent, updater, installer):
+    """Return the deterministic bytes signed by the offline release signer."""
+    # Inspect source without importing or executing release code.
+    versions = [node.value.value for node in ast.parse(agent).body
+                if isinstance(node, ast.Assign) and any(isinstance(name, ast.Name) and name.id == "VERSION" for name in node.targets)
+                and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+    if len(versions) != 1 or not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", versions[0]):
+        raise ValueError("Agent source must declare exactly one literal semantic VERSION")
+    compile(agent, "ec_agent.py", "exec")
+    compile(updater, "ec_agent_update.py", "exec")
+    manifest = {"version": versions[0], "protocolVersion": 1, "platform": "debian-amd64",
+                "sha256": hashlib.sha256(agent).hexdigest(), "updaterProtocolVersion": 2,
+                "updaterSha256": hashlib.sha256(updater).hexdigest(),
+                "installerSha256": hashlib.sha256(installer).hexdigest()}
+    return (json.dumps(manifest, indent=2) + "\n").encode("utf-8")
+
+
+def package(source, target):
+    """Read version metadata without importing or executing release code."""
+    source, target = Path(source), Path(target)
+    agent = (source / "ec_agent.py").read_bytes()
+    updater = (source / "ec_agent_update.py").read_bytes()
+    installer = (source / "install.sh").read_bytes()
+    manifest = manifest_bytes(agent, updater, installer)
+    target.mkdir(parents=True, exist_ok=True)
+    try:
+        previous_manifest = (target / "manifest.json").read_bytes()
+    except FileNotFoundError:
+        previous_manifest = None
+    if previous_manifest != manifest:
+        # Builds never sign. Keep an offline signature only across identical
+        # rebuilds; changed releases remain unsigned until reviewed and signed.
+        (target / "manifest.json.sig").unlink(missing_ok=True)
+    (target / "ec-agent.py").write_bytes(agent)
+    (target / "ec-agent-update.py").write_bytes(updater)
+    (target / "install.sh").write_bytes(installer)
+    temporary_manifest = target / ".manifest.json.tmp"
+    temporary_manifest.write_bytes(manifest)
+    temporary_manifest.replace(target / "manifest.json")
+
+
+if __name__ == "__main__":
+    source = Path(__file__).resolve().parent
+    package(source, source.parent / "static" / "agent")

agent/sign_release.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Sign a reviewed, packaged release using a separate offline GnuPG home.
+
+This command is never invoked by controller builds or runtime. Keep
+the signing key and its GnuPG home outside this checkout and Docker context.
+"""
+import argparse
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+
+from package_release import manifest_bytes
+
+
+def reviewed_manifest(release):
+    manifest = (release / "manifest.json").read_bytes()
+    expected = manifest_bytes((release / "ec-agent.py").read_bytes(),
+                              (release / "ec-agent-update.py").read_bytes(),
+                              (release / "install.sh").read_bytes())
+    if manifest != expected:
+        raise ValueError("Release manifest does not match packaged artifacts; package and review the release again")
+    return manifest
+
+
+def sign_release(release, gnupghome, fingerprint):
+    """Sign exact manifest bytes, without importing any private key material."""
+    release, gnupghome = Path(release).resolve(), Path(gnupghome).resolve()
+    repository = Path(__file__).resolve().parent.parent
+    if not re.fullmatch(r"(?:[A-Fa-f0-9]{40}|[A-Fa-f0-9]{64})", fingerprint):
+        raise ValueError("Signing requires a full OpenPGP key fingerprint (40 or 64 hexadecimal characters)")
+    if gnupghome.is_relative_to(repository) or gnupghome.is_relative_to(release):
+        raise ValueError("Keep the signing GnuPG home outside the repository and release directory")
+    if not gnupghome.is_dir():
+        raise ValueError("The offline signing GnuPG home must already exist")
+    manifest = reviewed_manifest(release)
+    with tempfile.TemporaryDirectory(prefix=".sign-release-", dir=release) as directory:
+        staged_manifest = Path(directory) / "manifest.json"
+        signature = Path(directory) / "manifest.json.sig"
+        staged_manifest.write_bytes(manifest)
+        subprocess.run([
+            "gpg", "--no-options", "--homedir", str(gnupghome), "--batch", "--yes",
+            "--armor", "--detach-sign", "--digest-algo", "SHA512",
+            "--local-user", fingerprint.upper() + "!", "--output", str(signature),
+            str(staged_manifest),
+        ], check=True)
+        if not signature.read_bytes().startswith(b"-----BEGIN PGP SIGNATURE-----\n"):
+            raise ValueError("GnuPG did not produce an ASCII-armored detached signature")
+        if reviewed_manifest(release) != manifest:
+            raise ValueError("Release changed during signing; review and sign the release again")
+        signature.chmod(0o644)
+        signature.replace(release / "manifest.json.sig")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--release-dir", type=Path,
+                        default=Path(__file__).resolve().parent.parent / "static" / "agent")
+    parser.add_argument("--gnupghome", required=True, type=Path,
+                        help="Existing offline GnuPG home outside the checkout/build context")
+    parser.add_argument("--fingerprint", required=True,
+                        help="Full fingerprint of the exact signing key (not a key ID or email)")
+    args = parser.parse_args()
+    try:
+        sign_release(args.release_dir, args.gnupghome, args.fingerprint)
+    except (OSError, ValueError, SyntaxError, subprocess.CalledProcessError) as cause:
+        parser.exit(1, f"Release signing failed: {cause}\n")
+    print(f"Signed {args.release_dir / 'manifest.json'}")
+
+
+if __name__ == "__main__":
+    main()

agent/tests/test_installer.py

@@ -0,0 +1,549 @@
+"""Exercise installer guards without installing software or touching services."""
+
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+
+SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
+BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
+PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
+OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
+RELEASE = next(block for block in BLOCKS if "# INSTALLATION_RELEASE:" in block)
+CLEANUP = re.search(r"installation_cleanup\(\) \{.*?\n\}", SOURCE, re.DOTALL).group()
+DOCKER = SOURCE[SOURCE.index("# INSTALLATION_DOCKER:"):SOURCE.index("install -d -m 0700 /etc/electric-cherry")]
+SPEC = importlib.util.spec_from_file_location("package_release", Path(__file__).resolve().parents[1] / "package_release.py")
+PACKAGER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(PACKAGER)
+
+
+class InstallerTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name).resolve()
+        self.root = self.base / "managed"
+        self.config = self.base / "configuration" / "agent.json"
+        self.config.parent.mkdir()
+        self.keyring = self.config.with_name('release-signing.gpg')
+        self.keyring.write_bytes(b'publisher public key')
+        self.controller = "https://controller.example"
+        self.saved = {"root": str(self.root), "controller": self.controller,
+                      "hostId": "host-one", "credential": "private"}
+        self.overrides = {}
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def preflight(self, root=None):
+        original_lstat = Path.lstat
+
+        def root_owned(path):
+            # Simulate Linux root-owned parents while keeping the actual inode
+            # types. No filesystem mutations or elevated privileges are needed.
+            info = original_lstat(path)
+            fields = list(info)
+            fields[4] = 0
+            if stat.S_ISDIR(info.st_mode):
+                fields[0] &= ~0o022
+            override = self.overrides.get(str(path), {})
+            fields[0] = override.get("mode", fields[0])
+            fields[4] = override.get("uid", fields[4])
+            return os.stat_result(fields)
+
+        argv = ["-", self.controller, str(root or self.root), str(self.config)]
+        with patch.object(sys, "argv", argv), patch.object(Path, "lstat", root_owned):
+            exec(compile(PREFLIGHT, "installer-preflight", "exec"), {})
+
+    def legacy_installation(self):
+        for directory in ("agent", "instances", "libraries"):
+            (self.root / directory).mkdir(parents=True, exist_ok=True)
+        self.config.write_text(json.dumps(self.saved))
+
+    def ownership(self):
+        with patch.object(sys, "argv", ["-", str(self.root), str(self.config)]):
+            exec(compile(OWNERSHIP, "installer-ownership", "exec"), {})
+
+    def test_new_directory_and_empty_directory_are_allowed_without_adopting_files(self):
+        self.preflight()
+        self.assertFalse(self.root.exists())
+        self.root.mkdir()
+        self.preflight()
+        (self.root / "unrelated.txt").write_text("keep")
+        with self.assertRaisesRegex(SystemExit, "not owned"):
+            self.preflight()
+        self.assertEqual((self.root / "unrelated.txt").read_text(), "keep")
+
+    def test_no_preprovisioned_signing_key_is_required(self):
+        self.keyring.unlink()
+        self.preflight()
+
+    def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self):
+        self.legacy_installation()
+        original_config = self.config.read_bytes()
+        (self.root / "libraries" / "game.ff").write_bytes(b"game data")
+        self.preflight()
+        self.ownership()
+        marker = self.root / ".electric-cherry.json"
+        self.assertEqual(json.loads(marker.read_text()), {
+            "version": 1, "root": str(self.root), "hostId": "host-one", "controller": self.controller})
+        self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600)
+        self.assertEqual(self.config.read_bytes(), original_config)
+        self.assertEqual((self.root / "libraries" / "game.ff").read_bytes(), b"game data")
+        self.preflight()
+
+    def test_legacy_upgrade_never_adopts_unrelated_top_level_files(self):
+        self.legacy_installation()
+        (self.root / "personal-backup.tar").write_bytes(b"keep")
+        with self.assertRaisesRegex(SystemExit, "not owned"):
+            self.preflight()
+        self.assertFalse((self.root / ".electric-cherry.json").exists())
+
+    def test_reenrollment_cannot_silently_change_controller_or_root(self):
+        self.legacy_installation()
+        for field, value in (("controller", "https://another.example"), ("root", str(self.base / "other"))):
+            with self.subTest(field=field):
+                self.config.write_text(json.dumps({**self.saved, field: value}))
+                with self.assertRaisesRegex(SystemExit, "different controller or data directory"):
+                    self.preflight()
+
+    def test_existing_marker_must_match_controller_host_and_root(self):
+        self.legacy_installation()
+        self.ownership()
+        marker = self.root / ".electric-cherry.json"
+        valid = json.loads(marker.read_text())
+        for field, value in (("hostId", "host-two"), ("root", str(self.base)), ("controller", "https://another.example")):
+            with self.subTest(field=field):
+                marker.write_text(json.dumps({**valid, field: value}))
+                with self.assertRaisesRegex(SystemExit, "ownership does not match"):
+                    self.preflight()
+
+    def test_lexical_aliases_and_system_roots_are_rejected_before_mutation(self):
+        for root in ("/var/lib/..", "/var/lib/../lib", "/var//lib/managed", "/var/lib/./managed", "/var/lib/managed/", "/var/lib", "/usr/local/libexec", "/", "/games", str(self.config.parent), str(self.config.parent / "data")):
+            with self.subTest(root=root), self.assertRaisesRegex(SystemExit, "canonical, dedicated"):
+                self.preflight(root)
+        self.assertFalse(self.root.exists())
+
+    def test_symlink_roots_ancestors_metadata_and_managed_subdirectories_are_rejected(self):
+        elsewhere = self.base / "elsewhere"
+        elsewhere.mkdir()
+        self.root.symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight(str(self.root / "nested"))
+        self.root.unlink()
+        self.legacy_installation()
+        (self.root / "libraries").rmdir()
+        (self.root / "libraries").symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        (self.root / "libraries").unlink()
+        (self.root / "libraries").mkdir()
+        self.config.unlink()
+        real_config = self.base / "real-config.json"
+        real_config.write_text(json.dumps(self.saved))
+        self.config.symlink_to(real_config)
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+
+    def test_untrusted_parents_and_metadata_cannot_authorize_cleanup(self):
+        self.legacy_installation()
+        self.overrides[str(self.base)] = {"uid": 1000}
+        with self.assertRaisesRegex(SystemExit, "root-owned"):
+            self.preflight()
+        self.overrides = {str(self.base): {"mode": stat.S_IFDIR | 0o777}}
+        with self.assertRaisesRegex(SystemExit, "root-owned"):
+            self.preflight()
+        self.overrides = {str(self.config): {"uid": 1000}}
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+
+    def test_pending_deletion_blocks_upgrade_even_with_dangling_request_symlink(self):
+        self.legacy_installation()
+        request = self.config.with_name("decommission.json")
+        request.write_text("{}")
+        with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
+            self.preflight()
+        request.unlink()
+        request.symlink_to(self.base / "missing")
+        with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
+            self.preflight()
+
+    def test_pending_update_recovery_is_preserved_and_blocks_manual_replacement(self):
+        self.legacy_installation()
+        update = self.config.parent / "update"
+        update.mkdir()
+        journal = update / "journal.json"
+        saved = '{"phase":"swapping","targetVersion":"1.2.0"}'
+        journal.write_text(saved)
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+        self.assertEqual(journal.read_text(), saved)
+        journal.unlink()
+        journal.symlink_to(self.base / "missing")
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+
+    def test_update_directory_and_lock_cannot_follow_symlinks(self):
+        self.legacy_installation()
+        elsewhere = self.base / "unrelated"
+        elsewhere.mkdir()
+        update = self.config.parent / "update"
+        update.symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        update.unlink()
+        lock = self.config.with_name("update.lock")
+        lock.symlink_to(elsewhere / "sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "sentinel").exists())
+        lock.unlink()
+        activity = self.root / "agent" / "activity.lock"
+        activity.symlink_to(elsewhere / "activity-sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "activity-sentinel").exists())
+
+    def test_data_root_cannot_contain_the_updater_or_service_installation(self):
+        for path in ("/etc/systemd", "/etc/systemd/system", "/usr/local/libexec/electric-cherry-update.py", "/usr/local/libexec/electric-cherry-agent.py.update", "/usr/local/libexec/electric-cherry-uninstall.py.update"):
+            with self.subTest(path=path), self.assertRaisesRegex(SystemExit, "outside the agent executable and service"):
+                self.preflight(path)
+
+    def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
+        mountinfo = Path('/proc/1/mountinfo')
+        original_exists, original_read = Path.exists, Path.read_text
+        for mount, allowed in ((self.base, True), (self.root, False), (self.root / 'libraries', False)):
+            def exists(path):
+                return True if path == mountinfo else original_exists(path)
+
+            def read(path, *args, **kwargs):
+                return f'29 24 8:1 / {mount} rw - ext4 /dev/sda1 rw' if path == mountinfo else original_read(path, *args, **kwargs)
+
+            with self.subTest(mount=mount), patch.object(Path, 'exists', exists), patch.object(Path, 'read_text', read):
+                if allowed:
+                    self.preflight()
+                else:
+                    with self.assertRaisesRegex(SystemExit, "dedicated subdirectory of the mounted disk"):
+                        self.preflight()
+
+
+class DockerInstallerTests(unittest.TestCase):
+    def run_docker_setup(self, version="13", cli=False, engine=False, available=True,
+                         data=False, config=False, apt_status=0, supply_cli=True):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            docker_data = base / "docker-data"
+            docker_config = base / "daemon.json"
+            if data:
+                docker_data.mkdir()
+                (docker_data / "sentinel").write_text("existing containers")
+            if config:
+                docker_config.write_text("existing configuration")
+            block = DOCKER.replace("/var/lib/docker", str(docker_data)).replace("/etc/docker/daemon.json", str(docker_config))
+            # Simulate Debian's package split and Docker state. All commands that
+            # could install packages or touch services are shell stubs.
+            script = r'''
+set -euo pipefail
+command() { [ "$*" = '-v docker' ] && [ "$EC_TEST_CLI" = true ]; }
+dpkg-query() {
+    [ "$*" = '-W -f=${Status} docker.io' ] || exit 90
+    [ "$EC_TEST_ENGINE" = true ] || return 1
+    printf 'install ok installed'
+}
+apt-get() {
+    printf 'apt-get %s\n' "$*"
+    [ "$EC_TEST_APT_STATUS" = 0 ] || return "$EC_TEST_APT_STATUS"
+    if [ "$EC_TEST_SUPPLY_CLI" = true ]; then
+        case " $* " in
+            *' docker-cli '*) EC_TEST_CLI=true ;;
+            *' docker.io '*) if [ "$VERSION_ID" = 12 ]; then EC_TEST_CLI=true; fi ;;
+        esac
+    fi
+}
+systemctl() { printf 'systemctl %s\n' "$*"; }
+docker() {
+    [ "$EC_TEST_CLI" = true ] || return 127
+    [ "$*" = info ] || exit 91
+    if [ "$EC_TEST_AVAILABLE" != true ]; then
+        echo 'Cannot connect to the Docker daemon' >&2
+        return 1
+    fi
+}
+'''
+            environment = {**os.environ, "VERSION_ID": version,
+                           "EC_TEST_CLI": str(cli).lower(), "EC_TEST_ENGINE": str(engine).lower(),
+                           "EC_TEST_AVAILABLE": str(available).lower(), "EC_TEST_APT_STATUS": str(apt_status),
+                           "EC_TEST_SUPPLY_CLI": str(supply_cli).lower()}
+            result = subprocess.run(["bash", "-c", script + block], env=environment, capture_output=True, text=True)
+            if data:
+                self.assertEqual((docker_data / "sentinel").read_text(), "existing containers")
+            if config:
+                self.assertEqual(docker_config.read_text(), "existing configuration")
+            return result
+
+    def test_fresh_debian_12_and_13_install_both_engine_and_client(self):
+        for version, packages in (("12", "docker.io"), ("13", "docker.io docker-cli")):
+            with self.subTest(version=version):
+                result = self.run_docker_setup(version=version)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(result.stdout.splitlines(), [
+                    f"apt-get install -y --no-install-recommends {packages}",
+                    "systemctl enable --now docker"])
+
+    def test_rerun_repairs_debian_13_missing_client_without_touching_engine(self):
+        result = self.run_docker_setup(engine=True, data=True, config=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stdout.splitlines(), [
+            "apt-get install -y --no-install-recommends --no-remove docker-cli"])
+
+    def test_existing_docker_including_ce_is_reused_without_package_or_service_changes(self):
+        for engine in (False, True):
+            with self.subTest(debian_engine=engine):
+                result = self.run_docker_setup(cli=True, engine=engine, data=True, config=True)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_unknown_existing_data_or_configuration_still_blocks_engine_installation(self):
+        for data, config in ((True, False), (False, True)):
+            with self.subTest(data=data, config=config):
+                result = self.run_docker_setup(data=data, config=config)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("Existing Docker data/configuration", result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_unavailable_existing_engine_exposes_docker_error_without_restarting_it(self):
+        result = self.run_docker_setup(cli=True, available=False)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("Cannot connect to the Docker daemon", result.stderr)
+        self.assertIn("journalctl -u docker", result.stderr)
+        self.assertEqual(result.stdout, "")
+
+    def test_package_failure_stops_before_starting_docker(self):
+        result = self.run_docker_setup(apt_status=42)
+        self.assertEqual(result.returncode, 42)
+        self.assertNotIn("systemctl", result.stdout)
+
+    def test_missing_client_is_distinguished_from_an_unavailable_daemon(self):
+        result = self.run_docker_setup(supply_cli=False)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("Docker CLI is missing", result.stderr)
+        self.assertNotIn("Docker is installed but unavailable", result.stderr)
+
+
+class RepositoryTrustTests(unittest.TestCase):
+    def fetch(self, status="200", body="public key", curl_exit=0):
+        block = SOURCE[SOURCE.index("# INSTALLATION_TRUST:"):SOURCE.index("# INSTALLATION_TRUST_END")]
+        with tempfile.TemporaryDirectory() as temporary:
+            script = r'''set -euo pipefail
+curl() {
+    printf '%s\n' "$@" > "$temporary_dir/curl-args"
+    printf '%s' "$EC_TEST_BODY" > "$temporary_dir/release-signing.gpg"
+    printf '%s' "$EC_TEST_STATUS"
+    return "$EC_TEST_EXIT"
+}
+''' + block + "\nprintf 'verified-fetch'"
+            result = subprocess.run(["bash", "-c", script], capture_output=True, text=True,
+                                    env={**os.environ, "temporary_dir": temporary, "EC_TEST_BODY": body,
+                                         "EC_TEST_STATUS": status, "EC_TEST_EXIT": str(curl_exit)})
+            args = (Path(temporary) / "curl-args").read_text().splitlines()
+            return result, args
+
+    def test_installer_fetches_fixed_repository_key_with_bounds_and_tls(self):
+        result, args = self.fetch()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stdout, "verified-fetch")
+        self.assertIn("https://gitea.thewrightserver.net/josh/electric-cherry/raw/branch/main/agent/release-signing.gpg", args)
+        for option, value in (("--proto", "=https"), ("--max-filesize", "262144"),
+                              ("--max-time", "120"), ("--connect-timeout", "25"),
+                              ("--header", "Cache-Control: no-cache")):
+            self.assertEqual(args[args.index(option) + 1], value)
+        self.assertNotIn("--location", args)
+        self.assertNotIn("--insecure", args)
+        self.assertLess(SOURCE.index("# INSTALLATION_TRUST:"), SOURCE.index('$controller/api/agent/release/manifest.json'))
+
+    def test_repository_failure_redirect_or_empty_key_stops_installation(self):
+        for status, body, code in (("302", "redirect", 0), ("404", "missing", 22),
+                                    ("200", "", 0), ("200", "partial", 28), ("200", "oversized", 63)):
+            with self.subTest(status=status, body=body):
+                result, _ = self.fetch(status, body, code)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertNotIn("verified-fetch", result.stdout)
+
+
+class ReleaseTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.source = Path(self.temporary.name).resolve() / "source"
+        self.target = Path(self.temporary.name).resolve() / "release"
+        self.source.mkdir()
+        (self.source / "ec_agent.py").write_text('VERSION = "3.12.7"\nraise RuntimeError("must never execute during packaging")\n')
+        (self.source / "ec_agent_update.py").write_text('raise RuntimeError("must never execute during verification")\n')
+        (self.source / "install.sh").write_text("#!/bin/bash\n")
+        PACKAGER.package(self.source, self.target)
+        self.keyring = self.source.parent / 'release-signing.gpg'
+        self.keyring.write_bytes(b'publisher public key')
+        (self.target / 'manifest.json.sig').write_bytes(b'detached signature')
+        self.gpgv = Mock(return_value=subprocess.CompletedProcess([], 0))
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def verify(self):
+        argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py"), str(self.keyring)]
+        original_lstat = Path.lstat
+        def root_owned(path):
+            fields = list(original_lstat(path))
+            fields[4] = 0
+            if stat.S_ISDIR(fields[0]):
+                fields[0] &= ~0o022
+            return os.stat_result(fields)
+        with patch.object(sys, "argv", argv), patch.object(Path, 'lstat', root_owned), patch.object(subprocess, 'run', self.gpgv):
+            exec(compile(RELEASE, "installer-release", "exec"), {})
+
+    def manifest(self, **changes):
+        path = self.target / "manifest.json"
+        value = json.loads(path.read_text())
+        value.update(changes)
+        path.write_text(json.dumps(value))
+        return value
+
+    def test_package_derives_version_and_hashes_both_sources_without_executing_them(self):
+        manifest = self.manifest()
+        self.assertEqual(manifest["version"], "3.12.7")
+        self.assertEqual(manifest["updaterProtocolVersion"], 2)
+        self.assertEqual(manifest["sha256"], hashlib.sha256((self.source / "ec_agent.py").read_bytes()).hexdigest())
+        self.assertEqual(manifest["updaterSha256"], hashlib.sha256((self.source / "ec_agent_update.py").read_bytes()).hexdigest())
+        self.assertEqual((self.target / "ec-agent-update.py").read_bytes(), (self.source / "ec_agent_update.py").read_bytes())
+        self.verify()
+
+    def test_changed_agent_and_updater_downloads_are_rejected(self):
+        for filename, error in (("ec-agent.py", "Agent release checksum"), ("ec-agent-update.py", "Updater release checksum")):
+            with self.subTest(filename=filename):
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"# changed\n")
+                with self.assertRaisesRegex(SystemExit, error):
+                    self.verify()
+                path.write_bytes(original)
+
+    def test_installer_rejects_missing_or_unsupported_updater_protocol(self):
+        for protocol in (None, 0, 1, True, "2"):
+            self.manifest(updaterProtocolVersion=protocol)
+            with self.subTest(protocol=protocol), self.assertRaisesRegex(SystemExit, "Unsupported agent or updater"):
+                self.verify()
+
+    def test_empty_or_oversized_downloaded_key_cannot_authorize_installation(self):
+        for value in (b"", b"x" * (256 * 1024 + 1)):
+            self.keyring.write_bytes(value)
+            with self.subTest(size=len(value)), self.assertRaisesRegex(SystemExit, "public key is empty or exceeds"):
+                self.verify()
+        self.gpgv.assert_not_called()
+
+    def test_signature_is_verified_against_downloaded_key_before_manifest_parsing(self):
+        self.verify()
+        args, kwargs = self.gpgv.call_args
+        self.assertEqual(args[0][0], 'gpgv')
+        self.assertEqual(args[0][-4:], ['--keyring', str(self.keyring), str(self.target / 'manifest.json.sig'), '-'])
+        self.assertEqual(kwargs['input'], (self.target / 'manifest.json').read_bytes())
+        self.assertEqual(kwargs['timeout'], 30)
+        self.assertFalse(Path(args[0][2]).exists())
+        (self.target / 'manifest.json').write_bytes(b'not even JSON')
+        self.gpgv.return_value = subprocess.CompletedProcess([], 1)
+        with self.assertRaisesRegex(SystemExit, 'signature verification failed'):
+            self.verify()
+
+    def test_missing_signature_key_or_gpgv_prevents_installation(self):
+        for path in (self.keyring, self.target / 'manifest.json.sig'):
+            original = path.read_bytes()
+            path.unlink()
+            self.gpgv.reset_mock()
+            with self.subTest(path=path), self.assertRaises(FileNotFoundError):
+                self.verify()
+            self.gpgv.assert_not_called()
+            path.write_bytes(original)
+        for failure in (FileNotFoundError(), subprocess.TimeoutExpired('gpgv', 30)):
+            self.gpgv.side_effect = failure
+            with self.subTest(failure=failure), self.assertRaisesRegex(SystemExit, 'working gpgv'):
+                self.verify()
+
+    def test_installer_rejects_manifest_version_mismatch_and_invalid_code(self):
+        self.manifest(version="3.12.8")
+        with self.assertRaisesRegex(SystemExit, "version does not match"):
+            self.verify()
+        malformed = b"def malformed(:\n"
+        (self.target / "ec-agent-update.py").write_bytes(malformed)
+        self.manifest(version="3.12.7", updaterSha256=hashlib.sha256(malformed).hexdigest())
+        with self.assertRaises(SyntaxError):
+            self.verify()
+
+    def test_packaging_requires_one_literal_semantic_agent_version(self):
+        for source in ('VERSION = "bad"', 'VERSION = "01.2.3"', 'VERSION = str(3)', 'VERSION = "1.2.3"\nVERSION = "1.2.4"'):
+            (self.source / "ec_agent.py").write_text(source)
+            with self.subTest(source=source), self.assertRaisesRegex(ValueError, "literal semantic VERSION"):
+                PACKAGER.package(self.source, self.target)
+
+    def test_installer_serializes_changes_and_enables_update_timer_only_after_agent_starts(self):
+        locked = SOURCE.index("flock --exclusive 9")
+        self.assertLess(SOURCE.index("systemctl stop electric-cherry-update.timer"), locked)
+        self.assertLess(locked, SOURCE.index("installation_preflight", locked))
+        self.assertLess(SOURCE.index("installation_preflight", locked), SOURCE.index('install -m 0755 "$temporary_dir/ec-agent.py"'))
+        self.assertLess(locked, SOURCE.index("flock --exclusive 8"))
+        self.assertLess(SOURCE.index("flock --exclusive 8"), SOURCE.index("systemctl stop electric-cherry-agent.service"))
+        self.assertLess(SOURCE.index("flock --unlock 8", SOURCE.index("systemctl daemon-reload")), SOURCE.index("systemctl restart electric-cherry-agent"))
+        self.assertLess(SOURCE.index("systemctl is-active --quiet electric-cherry-agent"), SOURCE.index("systemctl enable --now electric-cherry-update.timer"))
+        self.assertIn("ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py", SOURCE)
+        self.assertIn("OnUnitInactiveSec=5min", SOURCE)
+        self.assertIn("RandomizedDelaySec=30s", SOURCE)
+        self.assertIn("AccuracySec=1s", SOURCE)
+        self.assertNotIn("Persistent=true", SOURCE)
+
+    def test_failed_install_restores_only_previously_active_timer_after_releasing_locks(self):
+        for previous, finished, code in ((True, False, 37), (False, False, 37), (True, True, 0)):
+            with self.subTest(previous=previous, finished=finished):
+                scratch = Path(self.temporary.name) / "scratch"
+                scratch.mkdir(exist_ok=True)
+                log = Path(self.temporary.name) / "cleanup.log"
+                log.unlink(missing_ok=True)
+                script = CLEANUP + r'''
+flock() { printf 'unlock:%s\n' "$2" >> "$EC_TEST_LOG"; }
+systemctl() {
+    if { : >&8; } 2>/dev/null || { : >&9; } 2>/dev/null; then
+        printf 'lock-still-open\n' >> "$EC_TEST_LOG"
+        return 1
+    fi
+    printf '%s\n' "$*" >> "$EC_TEST_LOG"
+}
+temporary_dir="$EC_TEST_TEMP"
+updater_timer_was_active="$EC_TEST_PREVIOUS"
+installer_finished="$EC_TEST_FINISHED"
+update_lock_held=true
+activity_lock_held=true
+exec 8>"$EC_TEST_LOCK"
+exec 9>"$EC_TEST_LOCK"
+trap installation_cleanup EXIT
+exit "$EC_TEST_EXIT"
+'''
+                environment = {**os.environ, "EC_TEST_LOG": str(log), "EC_TEST_TEMP": str(scratch),
+                               "EC_TEST_PREVIOUS": str(previous).lower(), "EC_TEST_FINISHED": str(finished).lower(),
+                               "EC_TEST_LOCK": str(Path(self.temporary.name) / "test.lock"), "EC_TEST_EXIT": str(code)}
+                result = subprocess.run(["bash", "-c", script], env=environment, capture_output=True)
+                self.assertEqual(result.returncode, code, result.stderr)
+                expected = ["unlock:8", "unlock:9"]
+                if previous and not finished:
+                    expected.append("start electric-cherry-update.timer")
+                self.assertEqual(log.read_text().splitlines(), expected)
+                self.assertFalse(scratch.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_plutonium_cache.py

@@ -0,0 +1,506 @@
+import base64
+import copy
+import hashlib
+import json
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+import urllib.error
+
+import test_agent as fixtures
+
+
+module = fixtures.module
+AgentError = fixtures.AgentError
+IMAGE = "ghcr.io/ayymoss/plutainer:latest"
+BOOTSTRAPPER = "bin/plutonium-bootstrapper-win32.exe"
+BASE_URL = "https://cdn.plutoniummod.com/updater/prod/files/"
+
+
+class PlutoniumCacheTests(unittest.TestCase):
+    tearDown = fixtures.AgentTests.tearDown
+    operation = fixtures.AgentTests.operation
+    create = fixtures.AgentTests.create
+
+    def setUp(self):
+        fixtures.AgentTests.setUp(self)
+        network = patch.object(self.agent, "plutonium_open", side_effect=AssertionError("Unexpected vendor request"), create=True)
+        network.start()
+        self.addCleanup(network.stop)
+
+    def release(self, revision=5354, contents=None):
+        contents = contents or {
+            BOOTSTRAPPER: b"bootstrapper",
+            "bin/plutonium.dll": b"runtime",
+            "storage/t5/zone/plutonium.ff": b"zone-data",
+            "storage/iw5/images/~weapon_ak47_iw5_spc-rgb&weap~fa48e615.iwi": b"texture-data",
+            "launcher/plutonium.exe": b"launcher-not-needed",
+        }
+        manifest = {
+            "product": "plutonium-core-prod",
+            "revision": revision,
+            "baseUrl": BASE_URL,
+            "files": [
+                {"name": name, "size": len(content), "hash": hashlib.sha1(content).hexdigest()}
+                for name, content in contents.items()
+            ],
+        }
+        return manifest, contents
+
+    def opener(self, manifest, contents, calls=None):
+        calls = calls if calls is not None else []
+
+        def open_url(url):
+            calls.append(url)
+            if url == module.PLUTONIUM_MANIFEST_URL:
+                return fixtures.Response(json.dumps(manifest).encode())
+            self.assertTrue(url.startswith(BASE_URL), url)
+            name = next(entry["name"] for entry in manifest["files"] if url == BASE_URL + entry["hash"])
+            return fixtures.Response(contents[name])
+
+        return open_url
+
+    def file_url(self, content):
+        return BASE_URL + hashlib.sha1(content).hexdigest()
+
+    def prepare(self, revision=5354, contents=None):
+        manifest, contents = self.release(revision, contents)
+        calls = []
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            result = self.agent.prepare_plutonium({"revision": revision})
+        return manifest, contents, calls, result
+
+    def cache_content(self):
+        root = self.agent.root / "agent" / "plutonium-cache"
+        pointer = json.loads((root / "current.json").read_text())
+        return root / pointer["key"] / "content"
+
+    def app(self, name="app"):
+        app = Path(self.temp.name) / name
+        app.mkdir(exist_ok=True)
+        return app
+
+    def assert_runtime(self, app, manifest, contents):
+        runtime = app / "runtime" / "plutonium"
+        self.assertEqual(json.loads((runtime / "cdn_info.json").read_text()), manifest)
+        for name, content in contents.items():
+            if name.startswith("launcher"):
+                self.assertFalse((runtime / name).exists())
+            else:
+                self.assertEqual((runtime / name).read_bytes(), content)
+
+    def test_cold_preparation_verifies_downloads_and_reports_inventory(self):
+        self.assertIsNone(self.agent.plutonium_cache_inventory())
+        manifest, contents, calls, result = self.prepare()
+        expected_bytes = sum(len(content) for name, content in contents.items() if not name.startswith("launcher"))
+        self.assertEqual(result, {"revision": 5354, "bytes": expected_bytes})
+        self.assertEqual(self.agent.plutonium_cache_inventory(), result)
+        self.assertEqual(calls[0], module.PLUTONIUM_MANIFEST_URL)
+        self.assertEqual(set(calls[1:]), {self.file_url(content) for name, content in contents.items() if not name.startswith("launcher")})
+        self.assertEqual(json.loads((self.cache_content() / "cdn_info.json").read_text()), manifest)
+        self.assertEqual((self.agent.root / "agent" / "plutonium-cache").stat().st_mode & 0o077, 0)
+
+    def test_warm_preparation_only_fetches_manifest_and_dispatches(self):
+        manifest, contents, _, first = self.prepare()
+        calls = []
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            result = self.agent.dispatch({"kind": "plutonium.prepare", "payload": {"revision": 5354}})
+        self.assertEqual(result, first)
+        self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
+
+    def test_refresh_downloads_changed_files_and_reuses_verified_unchanged_files(self):
+        _, contents, _, _ = self.prepare()
+        changed = {**contents, "bin/plutonium.dll": b"new-runtime", "storage/t6/new.ff": b"new-zone"}
+        manifest, changed, calls, result = self.prepare(5355, changed)
+        self.assertEqual(result["revision"], 5355)
+        self.assertEqual(set(calls[1:]), {self.file_url(changed["bin/plutonium.dll"]), self.file_url(changed["storage/t6/new.ff"])})
+        app = self.app()
+        self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
+        self.assert_runtime(app, manifest, changed)
+
+    def test_changed_manifest_is_refreshed_even_when_revision_number_is_unchanged(self):
+        _, contents, _, _ = self.prepare()
+        previous = self.cache_content()
+        manifest, changed, calls, result = self.prepare(5354, {**contents, "bin/plutonium.dll": b"corrected-runtime"})
+        self.assertEqual(result["revision"], 5354)
+        self.assertNotEqual(self.cache_content(), previous)
+        self.assertEqual(calls[1:], [self.file_url(changed["bin/plutonium.dll"])])
+        app = self.app()
+        self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
+        self.assert_runtime(app, manifest, changed)
+
+    def test_failed_refresh_preserves_the_older_complete_cache(self):
+        manifest, contents, _, result = self.prepare()
+        previous = self.cache_content()
+        next_manifest, next_contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
+        next_contents["bin/plutonium.dll"] = b"bad-runtime"
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(next_manifest, next_contents)):
+            with self.assertRaises(AgentError):
+                self.agent.prepare_plutonium({"revision": 5355})
+        self.assertEqual(self.cache_content(), previous)
+        self.assertEqual(self.agent.plutonium_cache_inventory(), result)
+        app = self.app()
+        self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
+        self.assert_runtime(app, manifest, contents)
+
+    def test_interrupted_preparation_resumes_without_advertising_a_partial_cache(self):
+        manifest, contents = self.release()
+        open_url = self.opener(manifest, contents)
+
+        def interrupted_open(url):
+            if url == self.file_url(contents["bin/plutonium.dll"]):
+                raise urllib.error.URLError("simulated network interruption")
+            return open_url(url)
+
+        with patch.object(self.agent, "plutonium_open", side_effect=interrupted_open):
+            with self.assertRaises(AgentError):
+                self.agent.prepare_plutonium({})
+        self.assertIsNone(self.agent.plutonium_cache_inventory())
+        self.assertFalse((self.agent.root / "agent/plutonium-cache/current.json").exists())
+        _, _, calls, result = self.prepare()
+        self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
+        self.assertEqual(result["revision"], manifest["revision"])
+
+    def test_agent_restart_reconciles_preparation_instead_of_accepting_an_older_cache(self):
+        _, contents, _, _ = self.prepare()
+        operation = {"id": "prepare-new-release", "kind": "plutonium.prepare", "payload": {"revision": 5355}}
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "running")
+        self.agent.journal.close()
+        self.agent = fixtures.Agent(self.config, self.docker)
+        self.assertEqual(self.agent.journal.get(operation["id"])["state"], "unknown")
+        manifest, contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
+        calls = []
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(json.loads(result["result"])["revision"], 5355)
+        self.assertTrue(json.loads(result["result"])["reconciled"])
+        self.assertEqual(self.agent.plutonium_cache_inventory()["revision"], 5355)
+        self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL, self.file_url(contents["bin/plutonium.dll"])])
+        with patch.object(self.agent, "plutonium_open", side_effect=AssertionError("Completed operation repeated")):
+            self.assertEqual(self.agent.execute(operation)["result"], result["result"])
+
+    def test_heartbeat_defers_preparation_only_for_explicit_true(self):
+        self.assertFalse(self.agent.defer_plutonium_preparation)
+        for response, expected in (
+            ({"deferPlutoniumPreparation": True}, True),
+            ({}, False),
+            ({"deferPlutoniumPreparation": False}, False),
+            ({"deferPlutoniumPreparation": 1}, False),
+            ({"deferPlutoniumPreparation": "true"}, False),
+            ({"deferPlutoniumPreparation": None}, False),
+        ):
+            self.agent.defer_plutonium_preparation = not expected
+            with self.subTest(response=response), patch.object(self.agent, "request", return_value=response), patch.object(self.agent, "inventory", return_value={}), patch.object(self.agent, "record_health"):
+                self.agent.heartbeat()
+            self.assertIs(self.agent.defer_plutonium_preparation, expected)
+
+    def assert_preparation_yields_and_resumes(self, *, restarted):
+        manifest, contents = self.release()
+        operation = {"id": "cooperative-prepare", "kind": "plutonium.prepare", "payload": {"revision": 5354}}
+        if restarted:
+            self.agent.journal.accept(operation)
+            self.agent.journal.update(operation["id"], "running")
+            self.agent.journal.close()
+            self.agent = fixtures.Agent(self.config, self.docker)
+            self.assertEqual(self.agent.journal.get(operation["id"])["state"], "unknown")
+        agent = self.agent
+        open_url = self.opener(manifest, contents)
+
+        class BusyResponse(fixtures.Response):
+            def read(self, size=-1):
+                chunk = super().read(size)
+                if chunk:
+                    agent.defer_plutonium_preparation = True
+                return chunk
+
+        def become_busy_during_download(url):
+            if url == self.file_url(contents["bin/plutonium.dll"]):
+                return BusyResponse(contents["bin/plutonium.dll"])
+            return open_url(url)
+
+        with patch.object(self.agent, "plutonium_open", side_effect=become_busy_during_download):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "accepted", result)
+        self.assertIn("waiting for other host operations", json.loads(result["result"])["message"])
+        self.assertIsNone(self.agent.executing_id)
+        self.assertIsNone(self.agent.plutonium_cache_inventory())
+        self.assertTrue(self.agent.defer_plutonium_preparation)
+        # A yielded preparation must let other mutations finish before retrying.
+        self.create()
+        self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 1)["running"])
+        self.agent.defer_plutonium_preparation = False
+        calls = []
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(result["id"], operation["id"])
+        self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
+        self.assertIn(self.file_url(contents["bin/plutonium.dll"]), calls)
+        self.assertEqual(self.agent.plutonium_cache_inventory()["revision"], 5354)
+
+    def test_busy_host_yields_mid_download_and_resumes_completed_files(self):
+        self.assert_preparation_yields_and_resumes(restarted=False)
+
+    def test_reconciled_preparation_yields_mid_download_and_resumes_same_operation(self):
+        self.assert_preparation_yields_and_resumes(restarted=True)
+
+    def test_resumption_removes_orphans_before_storage_check_and_publication(self):
+        manifest, contents = self.release()
+        raw = json.dumps(manifest).encode()
+        staging = self.agent.root / "agent/plutonium-cache" / (module.digest(raw) + ".partial/content")
+        bootstrapper = staging / BOOTSTRAPPER
+        bootstrapper.parent.mkdir(parents=True)
+        bootstrapper.write_bytes(contents[BOOTSTRAPPER])
+        orphans = [staging / ".download-abandoned", staging / "bin/.ec-abandoned", staging / "unlisted/extra.dat", staging / "cdn_info.json"]
+        for orphan in orphans:
+            orphan.parent.mkdir(parents=True, exist_ok=True)
+            orphan.write_bytes(b"orphaned bytes")
+        outside = Path(self.temp.name) / "outside-cache"
+        outside.mkdir()
+        (outside / "untouched").write_bytes(b"private outside bytes")
+        link = staging / "outside-link"
+        link.symlink_to(outside, target_is_directory=True)
+        original_disk_usage = module.shutil.disk_usage
+
+        def checked_storage(path):
+            for orphan in orphans:
+                self.assertFalse(orphan.exists(), "orphan must be removed before storage accounting")
+            self.assertFalse(link.is_symlink())
+            return original_disk_usage(path)
+
+        calls = []
+        with patch.object(module.shutil, "disk_usage", side_effect=checked_storage), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            result = self.agent.prepare_plutonium({})
+        content = self.cache_content()
+        expected = {name for name in contents if not name.startswith("launcher")}
+        actual = {path.relative_to(content).as_posix() for path in content.rglob("*") if path.is_file()}
+        self.assertEqual(actual, expected | {"cdn_info.json"})
+        self.assertEqual(result["bytes"], sum(len(contents[name]) for name in expected))
+        self.assertEqual(sum(path.stat().st_size for path in content.rglob("*") if path.is_file()), result["bytes"] + len(raw))
+        self.assertNotIn(self.file_url(contents[BOOTSTRAPPER]), calls)
+        self.assertEqual((outside / "untouched").read_bytes(), b"private outside bytes")
+        app = self.app()
+        self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
+        self.assert_runtime(app, manifest, contents)
+        self.assertFalse((app / "runtime/plutonium/unlisted/extra.dat").exists())
+
+    def test_public_downloads_reject_other_origins_and_never_send_host_credentials(self):
+        # Call the class method to bypass the fixture's default network blocker;
+        # the underlying opener is mocked, so no request reaches the network.
+        with patch.object(module.urllib.request, "build_opener") as build:
+            for url in (
+                "http://cdn.plutoniummod.com/updater/prod/info.json",
+                "https://attacker.example/updater/prod/info.json",
+                "https://cdn.plutoniummod.com.attacker.example/updater/prod/info.json",
+                "https://secret@cdn.plutoniummod.com/updater/prod/info.json",
+                "https://cdn.plutoniummod.com/updater/dev/info.json",
+            ):
+                with self.subTest(url=url), self.assertRaises(AgentError):
+                    fixtures.Agent.plutonium_open(self.agent, url)
+            build.assert_not_called()
+            fixtures.Agent.plutonium_open(self.agent, module.PLUTONIUM_MANIFEST_URL)
+            build.return_value.open.assert_called_once_with(module.PLUTONIUM_MANIFEST_URL, timeout=30)
+            handler = build.call_args.args[0]
+            for redirect in (module.PLUTONIUM_MANIFEST_URL, "https://attacker.example/collect"):
+                with self.subTest(redirect=redirect), self.assertRaises(AgentError):
+                    handler.redirect_request(None, None, 302, "Found", {}, redirect)
+
+    def test_rejects_revision_downgrade_and_invalid_preparation_arguments(self):
+        self.prepare()
+        manifest, contents = self.release(5353)
+        with patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents)):
+            with self.assertRaises(AgentError):
+                self.agent.prepare_plutonium({"revision": 5354})
+        for payload in ({"revision": -1}, {"revision": True}, {"revision": "5354"},
+                        {"minimumFreeBytes": -1}, {"minimumFreeBytes": True}, {"minimumFreeBytes": "10"}):
+            with self.subTest(payload=payload), self.assertRaises(AgentError):
+                self.agent.prepare_plutonium(payload)
+
+    def test_disk_reserve_failure_leaves_existing_cache_usable(self):
+        _, contents, _, result = self.prepare()
+        manifest, contents = self.release(5355, {**contents, "bin/plutonium.dll": b"new-runtime"})
+        calls = []
+        usage = type(module.shutil.disk_usage(self.agent.root))(10000, 9900, 100)
+        with patch.object(module.shutil, "disk_usage", return_value=usage), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+            with self.assertRaises(AgentError):
+                self.agent.prepare_plutonium({"minimumFreeBytes": 1000})
+        self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
+        self.assertEqual(self.agent.plutonium_cache_inventory(), result)
+
+    def test_malformed_manifest_never_publishes_a_cache(self):
+        good, contents = self.release()
+        malformed = [
+            None, [], {},
+            {**good, "product": "other-product"},
+            {**good, "revision": 0},
+            {**good, "revision": True},
+            {**good, "revision": "5354"},
+            {**good, "baseUrl": "http://cdn.plutoniummod.com/updater/prod/"},
+            {**good, "baseUrl": "https://attacker.example/"},
+            {**good, "files": []},
+            {**good, "files": {}},
+            {**good, "files": good["files"][1:]},
+            {**good, "files": good["files"] + [good["files"][0]]},
+        ]
+        for manifest in malformed:
+            with self.subTest(manifest=manifest), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents)):
+                with self.assertRaises(AgentError):
+                    self.agent.prepare_plutonium({})
+                self.assertIsNone(self.agent.plutonium_cache_inventory())
+
+    def test_manifest_rejects_unsafe_paths_sizes_and_hashes_before_downloading(self):
+        good, contents = self.release()
+        bad_entries = [
+            {"name": name} for name in (
+                ".", "../escape", "/absolute", "bin/../../escape", "bin//file", "bin/./file",
+                "bin\\file", "C:/escape", "bin/file\x00", "cdn_info.json",
+            )
+        ] + [
+            {"size": size} for size in (-1, True, "5", None)
+        ] + [
+            {"hash": value} for value in ("", "0" * 39, "z" * 40, None)
+        ]
+        for override in bad_entries:
+            manifest = copy.deepcopy(good)
+            manifest["files"][1].update(override)
+            calls = []
+            with self.subTest(override=override), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, contents, calls)):
+                with self.assertRaises(AgentError):
+                    self.agent.prepare_plutonium({})
+                self.assertEqual(calls, [module.PLUTONIUM_MANIFEST_URL])
+                self.assertIsNone(self.agent.plutonium_cache_inventory())
+
+    def test_short_oversize_and_wrong_hash_downloads_are_not_published(self):
+        manifest, contents = self.release()
+        for body in (b"short", contents[BOOTSTRAPPER] + b"extra", b"X" * len(contents[BOOTSTRAPPER])):
+            with self.subTest(body=body), patch.object(self.agent, "plutonium_open", side_effect=self.opener(manifest, {**contents, BOOTSTRAPPER: body})):
+                with self.assertRaises(AgentError):
+                    self.agent.prepare_plutonium({})
+                self.assertIsNone(self.agent.plutonium_cache_inventory())
+
+    def test_seeds_are_private_and_preserve_existing_lobby_state(self):
+        manifest, contents, _, _ = self.prepare()
+        first, second = self.app("first"), self.app("second")
+        private = {
+            "configs/server.cfg": b"secret config",
+            "runtime/plutonium/plugins/private.dll": b"private plugin",
+            "runtime/plutonium/storage/t5/logs/games_mp.log": b"lobby log",
+        }
+        for name, body in private.items():
+            target = first / name
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(body)
+        self.assertTrue(self.agent.seed_plutonium(first, IMAGE))
+        self.assertTrue(self.agent.seed_plutonium(second, IMAGE))
+        self.assert_runtime(first, manifest, contents)
+        self.assert_runtime(second, manifest, contents)
+        for name, body in private.items():
+            self.assertEqual((first / name).read_bytes(), body)
+            self.assertFalse((second / name).exists())
+        first_file = first / "runtime/plutonium" / BOOTSTRAPPER
+        second_file = second / "runtime/plutonium" / BOOTSTRAPPER
+        cached_file = self.cache_content() / BOOTSTRAPPER
+        self.assertNotEqual(first_file.stat().st_ino, second_file.stat().st_ino)
+        self.assertNotEqual(first_file.stat().st_ino, cached_file.stat().st_ino)
+        first_file.write_bytes(b"container-updated")
+        self.assertEqual(second_file.read_bytes(), contents[BOOTSTRAPPER])
+        self.assertEqual(cached_file.read_bytes(), contents[BOOTSTRAPPER])
+
+    def test_absent_or_corrupt_cache_and_unsupported_image_fall_back(self):
+        app = self.app()
+        self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
+        self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
+        self.prepare()
+        for image in ("example.invalid/plutainer:latest", "ghcr.io/ayymoss/plutainer-other:latest", "ghcr.io/attacker/plutainer:latest"):
+            with self.subTest(image=image):
+                self.assertFalse(self.agent.seed_plutonium(app, image))
+        (self.cache_content() / BOOTSTRAPPER).write_bytes(b"broken-cache")
+        self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
+        self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
+        self.assertIsNone(self.agent.plutonium_cache_inventory())
+
+    def test_interrupted_seed_does_not_install_revision_marker_and_can_retry(self):
+        manifest, contents, _, _ = self.prepare()
+        app = self.app()
+        marker = app / "runtime/plutonium/cdn_info.json"
+        marker.parent.mkdir(parents=True)
+        marker.write_text(json.dumps({"revision": 5353}))
+        copy_file = module.managed_copy
+        copied = []
+
+        def interrupted_copy(root, name, source, *args, **kwargs):
+            self.assertFalse(marker.exists(), "revision marker must be installed only after all files")
+            if copied:
+                raise OSError("simulated full disk")
+            copied.append(name)
+            return copy_file(root, name, source, *args, **kwargs)
+
+        with patch.object(module, "managed_copy", side_effect=interrupted_copy):
+            with self.assertRaises((AgentError, OSError)):
+                self.agent.seed_plutonium(app, IMAGE)
+        self.assertFalse(marker.exists())
+        self.assertTrue(copied)
+        self.assertTrue(self.agent.seed_plutonium(app, IMAGE))
+        self.assert_runtime(app, manifest, contents)
+
+    def test_failed_seed_prevents_container_start(self):
+        self.prepare()
+        with patch.object(module, "managed_copy", side_effect=OSError("simulated full disk")):
+            payload = {"game": "t5", "libraryId": self.library_id, "image": IMAGE, "port": 28960, "env": {}}
+            result = self.agent.execute(self.operation("instance.create", payload, "failed-seed"))
+        self.assertEqual(result["state"], "failed", result)
+        self.assertFalse(any(call[:2] == ("container", "start") for call in self.docker.calls))
+        self.assertFalse((self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium/cdn_info.json").exists())
+
+    def test_symlink_in_cache_cannot_seed_outside_data(self):
+        _, contents, _, _ = self.prepare()
+        cached = self.cache_content() / BOOTSTRAPPER
+        outside = Path(self.temp.name) / "outside"
+        outside.write_bytes(contents[BOOTSTRAPPER])
+        cached.unlink()
+        cached.symlink_to(outside)
+        app = self.app()
+        self.assertFalse(self.agent.seed_plutonium(app, IMAGE))
+        self.assertFalse((app / "runtime/plutonium/cdn_info.json").exists())
+
+    def test_create_applies_initial_overlay_after_seed_and_starts_without_cache(self):
+        self.create()
+        self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 1)["running"])
+        self.assertFalse((self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium/cdn_info.json").exists())
+        manifest, contents, _, _ = self.prepare()
+        payload = self.agent.spec(fixtures.INSTANCE, 1)
+        payload["port"] += 1
+        payload["files"].append({"path": "runtime/plutonium/" + BOOTSTRAPPER, "contentBase64": base64.b64encode(b"custom-overlay").decode()})
+        other_id = "00000000-0000-4000-8000-000000000002"
+        result = self.agent.execute({**self.operation("instance.create", payload, "second-create"), "instanceId": other_id})
+        self.assertEqual(result["state"], "completed", result)
+        app = self.agent.instance_dir(other_id, 1) / "app"
+        self.assertEqual((app / "runtime/plutonium" / BOOTSTRAPPER).read_bytes(), b"custom-overlay")
+        self.assertEqual(json.loads((app / "runtime/plutonium/cdn_info.json").read_text()), manifest)
+        self.assertEqual((app / "configs/server.cfg").read_bytes(), b"original")
+
+    def test_rebuild_seeds_the_new_generation_from_current_cache(self):
+        payload = self.create()
+        manifest, contents, _, _ = self.prepare()
+        self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+        result = self.agent.execute(self.operation("instance.rebuild", {**payload, "nextGeneration": 2}, "rebuild"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertTrue(self.agent.inspect(fixtures.INSTANCE, 2)["running"])
+        self.assert_runtime(self.agent.instance_dir(fixtures.INSTANCE, 2) / "app", manifest, contents)
+
+    def test_restart_preserves_the_instances_downloaded_runtime(self):
+        self.prepare()
+        self.create()
+        runtime_file = self.agent.instance_dir(fixtures.INSTANCE, 1) / "app/runtime/plutonium" / BOOTSTRAPPER
+        runtime_file.write_bytes(b"container-updated-runtime")
+        self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+        with patch.object(self.agent, "seed_plutonium", side_effect=AssertionError("Restart must retain its runtime")):
+            result = self.agent.execute(self.operation("instance.restart", op_id="restart"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(runtime_file.read_bytes(), b"container-updated-runtime")
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_release_signing.py

@@ -0,0 +1,243 @@
+"""Release packaging is unsigned; only the explicit offline command may sign."""
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+AGENT = Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location("package_release", AGENT / "package_release.py")
+PACKAGER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(PACKAGER)
+SPEC = importlib.util.spec_from_file_location("sign_release", AGENT / "sign_release.py")
+SIGNER = importlib.util.module_from_spec(SPEC)
+with patch.dict(sys.modules, {"package_release": PACKAGER}):
+    SPEC.loader.exec_module(SIGNER)
+SPEC = importlib.util.spec_from_file_location("signed_pipeline_updater", AGENT / "ec_agent_update.py")
+UPDATER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(UPDATER)
+
+
+class ReleaseFixture:
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name).resolve()
+        self.source = self.base / "source"
+        self.source.mkdir()
+        self.target = self.base / "release"
+        self.gnupghome = self.base / "offline-keyring"
+        self.gnupghome.mkdir(mode=0o700)
+        self.fingerprint = "A" * 40
+        (self.source / "ec_agent.py").write_text('VERSION = "4.2.0"\nraise RuntimeError("Do not run")\n')
+        (self.source / "ec_agent_update.py").write_text('raise RuntimeError("Do not run")\n')
+        (self.source / "install.sh").write_text("#!/bin/bash\nexit 1\n")
+        PACKAGER.package(self.source, self.target)
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+
+class PackagedReleaseTests(ReleaseFixture, unittest.TestCase):
+    def test_manifest_is_deterministic_and_covers_every_executable(self):
+        first = (self.target / "manifest.json").read_bytes()
+        PACKAGER.package(self.source, self.target)
+        self.assertEqual(first, (self.target / "manifest.json").read_bytes())
+        manifest = json.loads(first)
+        self.assertEqual(manifest["updaterProtocolVersion"], 2)
+        for field, filename in (("sha256", "ec-agent.py"),
+                                ("updaterSha256", "ec-agent-update.py"),
+                                ("installerSha256", "install.sh")):
+            self.assertEqual(manifest[field], hashlib.sha256((self.target / filename).read_bytes()).hexdigest())
+        self.assertFalse((self.target / "manifest.json.sig").exists())
+
+    def test_packaging_preserves_signature_bytes_only_for_identical_manifest(self):
+        signature = self.target / "manifest.json.sig"
+        for filename in ("ec_agent.py", "ec_agent_update.py", "install.sh"):
+            with self.subTest(filename=filename):
+                signature.write_bytes(b"offline signature")
+                with patch.object(subprocess, "run", side_effect=AssertionError("Builds must never sign")):
+                    PACKAGER.package(self.source, self.target)
+                    self.assertEqual(signature.read_bytes(), b"offline signature")
+                    source = self.source / filename
+                    source.write_bytes(source.read_bytes() + b"# reviewed change\n")
+                    PACKAGER.package(self.source, self.target)
+                self.assertFalse(signature.exists())
+
+    def test_orphaned_signature_is_removed_when_manifest_was_missing(self):
+        (self.target / "manifest.json").unlink()
+        (self.target / "manifest.json.sig").write_bytes(b"orphaned signature")
+        PACKAGER.package(self.source, self.target)
+        self.assertFalse((self.target / "manifest.json.sig").exists())
+
+    def test_signer_requires_full_fingerprint_before_invoking_gpg(self):
+        for fingerprint in ("", "release@example.com", "A" * 16, "Z" * 40, "A" * 41):
+            with self.subTest(fingerprint=fingerprint), patch.object(subprocess, "run") as run:
+                with self.assertRaisesRegex(ValueError, "full OpenPGP key fingerprint"):
+                    SIGNER.sign_release(self.target, self.gnupghome, fingerprint)
+                run.assert_not_called()
+
+    def test_signer_rejects_private_key_homes_in_repository_or_release(self):
+        for home in (AGENT, self.target, self.target / "private"):
+            with self.subTest(home=home), patch.object(subprocess, "run") as run:
+                with self.assertRaisesRegex(ValueError, "outside the repository and release"):
+                    SIGNER.sign_release(self.target, home, self.fingerprint)
+                run.assert_not_called()
+
+    def test_signer_refuses_unreviewed_or_inconsistent_release_bytes(self):
+        for filename in ("ec-agent.py", "ec-agent-update.py", "install.sh", "manifest.json"):
+            with self.subTest(filename=filename), patch.object(subprocess, "run") as run:
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"\n")
+                with self.assertRaisesRegex(ValueError, "does not match packaged artifacts"):
+                    SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint)
+                run.assert_not_called()
+                path.write_bytes(original)
+
+    def test_failed_signing_does_not_replace_existing_signature(self):
+        signature = self.target / "manifest.json.sig"
+        signature.write_bytes(b"previous valid signature")
+        with patch.object(subprocess, "run", side_effect=subprocess.CalledProcessError(2, "gpg")):
+            with self.assertRaises(subprocess.CalledProcessError):
+                SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint)
+        self.assertEqual(signature.read_bytes(), b"previous valid signature")
+        self.assertEqual(list(self.target.glob(".sign-release-*")), [])
+
+    def test_release_mutation_during_signing_never_publishes_a_signature(self):
+        def mutate(command, **kwargs):
+            Path(command[command.index("--output") + 1]).write_bytes(b"-----BEGIN PGP SIGNATURE-----\nstaged\n")
+            (self.target / "ec-agent.py").write_bytes(b'VERSION = "9.0.0"\n')
+        with patch.object(subprocess, "run", side_effect=mutate):
+            with self.assertRaisesRegex(ValueError, "does not match packaged artifacts"):
+                SIGNER.sign_release(self.target, self.gnupghome, self.fingerprint)
+        self.assertFalse((self.target / "manifest.json.sig").exists())
+
+
+@unittest.skipUnless(shutil.which("gpg") and shutil.which("gpgv"), "GnuPG required for real offline signing tests")
+class OfflineSigningTests(ReleaseFixture, unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.keys = tempfile.TemporaryDirectory(prefix="ec-sign-")
+        cls.keyhome = Path(cls.keys.name)
+        cls.keyhome.chmod(0o700)
+        command = ["gpg", "--no-options", "--homedir", str(cls.keyhome), "--batch"]
+        subprocess.run(command + ["--pinentry-mode", "loopback", "--passphrase", "",
+                                  "--quick-generate-key", "Ephemeral release test <release@example.invalid>",
+                                  "ed25519", "sign", "0"], check=True, capture_output=True)
+        listing = subprocess.run(command + ["--with-colons", "--list-secret-keys"],
+                                 check=True, capture_output=True, text=True).stdout
+        cls.key_fingerprint = next(line.split(":")[9] for line in listing.splitlines() if line.startswith("fpr:"))
+        cls.public_key = subprocess.run(command + ["--export", cls.key_fingerprint],
+                                        check=True, capture_output=True).stdout
+
+    @classmethod
+    def tearDownClass(cls):
+        if shutil.which("gpgconf"):
+            subprocess.run(["gpgconf", "--homedir", str(cls.keyhome), "--kill", "gpg-agent"],
+                           check=False, capture_output=True)
+        cls.keys.cleanup()
+
+    def setUp(self):
+        super().setUp()
+        self.keyring = self.base / "release-signing.gpg"
+        self.keyring.write_bytes(self.public_key)
+        self.update_dir = self.base / "update"
+        self.update_dir.mkdir(mode=0o700)
+        original_lstat = Path.lstat
+        protected_paths = {self.keyring, *self.keyring.parents, self.update_dir}
+        def root_owned(path):
+            info = original_lstat(path)
+            if path not in protected_paths:
+                return info
+            fields = list(info)
+            fields[4] = 0
+            if stat.S_ISDIR(fields[0]):
+                fields[0] &= ~0o022
+            return os.stat_result(fields)
+        self.root_owned = root_owned
+
+    def verify_installer(self):
+        blocks = re.findall(r"<<'PY'\n(.*?)\nPY", (AGENT / "install.sh").read_text(), re.DOTALL)
+        release = next(block for block in blocks if "# INSTALLATION_RELEASE:" in block)
+        argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"),
+                str(self.target / "ec-agent-update.py"), str(self.keyring)]
+        # Only filesystem ownership is simulated; real GnuPG verifies the
+        # separately exported public key and the offline signer's exact bytes.
+        with patch.object(sys, "argv", argv), patch.object(Path, "lstat", self.root_owned):
+            exec(compile(release, "installer-release", "exec"), {})
+
+    def verify_updater(self):
+        with patch.object(UPDATER, "UPDATE_DIR", self.update_dir), \
+                patch.object(Path, "lstat", self.root_owned):
+            UPDATER.verify_release_signature((self.target / "manifest.json").read_bytes(),
+                                             (self.target / "manifest.json.sig").read_bytes(), self.public_key)
+
+    def verify(self):
+        return subprocess.run([
+            "gpg", "--no-options", "--homedir", str(self.keyhome), "--batch", "--verify",
+            str(self.target / "manifest.json.sig"), str(self.target / "manifest.json"),
+        ], capture_output=True)
+
+    def test_offline_signature_verifies_exact_bytes_and_survives_identical_build(self):
+        SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint)
+        signature = (self.target / "manifest.json.sig").read_bytes()
+        self.assertTrue(signature.startswith(b"-----BEGIN PGP SIGNATURE-----\n"))
+        self.assertEqual(self.verify().returncode, 0)
+        PACKAGER.package(self.source, self.target)
+        self.assertEqual((self.target / "manifest.json.sig").read_bytes(), signature)
+        self.assertEqual(self.verify().returncode, 0)
+        with (self.target / "manifest.json").open("ab") as manifest:
+            manifest.write(b" ")
+        self.assertNotEqual(self.verify().returncode, 0)
+
+    def test_unknown_signing_key_fails_without_publishing_a_signature(self):
+        with self.assertRaises(subprocess.CalledProcessError):
+            SIGNER.sign_release(self.target, self.keyhome, "F" * 40)
+        self.assertFalse((self.target / "manifest.json.sig").exists())
+
+    def test_offline_signed_release_is_accepted_by_installer_and_updater(self):
+        SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint)
+        self.verify_installer()
+        self.verify_updater()
+
+    def test_controller_cannot_authorize_changed_code_by_rehashing_manifest(self):
+        SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint)
+        for filename in ("ec-agent.py", "ec-agent-update.py"):
+            with self.subTest(filename=filename):
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"# controller's unauthorized change\n")
+                (self.target / "manifest.json").write_bytes(PACKAGER.manifest_bytes(
+                    (self.target / "ec-agent.py").read_bytes(),
+                    (self.target / "ec-agent-update.py").read_bytes(),
+                    (self.target / "install.sh").read_bytes()))
+                with self.assertRaisesRegex(SystemExit, "signature verification failed"):
+                    self.verify_installer()
+                with self.assertRaisesRegex(UPDATER.IntegrityError, "signature is invalid"):
+                    self.verify_updater()
+                path.write_bytes(original)
+
+    def test_signed_manifest_rejects_changed_agent_and_updater_downloads(self):
+        SIGNER.sign_release(self.target, self.keyhome, self.key_fingerprint)
+        for filename, message in (("ec-agent.py", "Agent release checksum"),
+                                  ("ec-agent-update.py", "Updater release checksum")):
+            with self.subTest(filename=filename):
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"# controller's unauthorized change\n")
+                with self.assertRaisesRegex(SystemExit, message):
+                    self.verify_installer()
+                path.write_bytes(original)
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_temperatures.py

@@ -0,0 +1,96 @@
+import errno
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_agent_temperatures", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class TemperatureTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.hwmon = self.root / "class"
+        self.hwmon.mkdir()
+        self.patch = patch.object(module, "HWMON_ROOT", self.hwmon)
+        self.patch.start()
+        self.addCleanup(self.patch.stop)
+        self.agent = object.__new__(module.Agent)
+
+    def chip(self, number, device, readings):
+        physical = self.root / (device + str(number)) / "hwmon" / ("hwmon" + str(number))
+        physical.mkdir(parents=True)
+        (self.hwmon / physical.name).symlink_to(physical, target_is_directory=True)
+        (physical / "name").write_text(device)
+        for channel, label, value in readings:
+            (physical / (channel + "_input")).write_text(str(value))
+            if label is not None:
+                (physical / (channel + "_label")).write_text(label)
+        return physical
+
+    def test_ec2_sensors_and_units_without_guessing_unknown_devices(self):
+        self.chip(0, "nvme", [("temp1", "Composite", 50900), ("temp2", "Sensor 1", 50900)])
+        self.chip(1, "pch_cometlake", [("temp1", None, 62000)])
+        self.chip(2, "coretemp", [("temp1", "Package id 0", 65000), ("temp2", "Core 0", 63000)])
+        self.chip(3, "iwlwifi_1", [("temp1", None, "unavailable")])
+        samples = self.agent.temperatures()
+        self.assertEqual([s["celsius"] for s in samples], [50.9, 50.9, 62.0, 65.0, 63.0])
+        self.assertEqual([s["kind"] for s in samples], ["nvme", "nvme", "chipset", "cpu-package", "cpu-core"])
+        self.assertEqual(len({s["id"] for s in samples}), len(samples))
+
+    def test_fault_disabled_bad_value_and_enodata_do_not_lose_other_sensors(self):
+        chip = self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), value)
+                         for i, value in enumerate([65000, 60000, "bad", 500000, 61000, 63000, 0], 1)])
+        (chip / "temp1_fault").write_text("1")
+        (chip / "temp2_enable").write_text("0")
+        original = Path.read_text
+        def read(path, *args, **kwargs):
+            if path.name == "temp5_input":
+                raise OSError(errno.ENODATA, "No data available")
+            return original(path, *args, **kwargs)
+        with patch.object(Path, "read_text", read):
+            self.assertEqual([s["celsius"] for s in self.agent.temperatures()], [63.0, 0.0])
+
+    def test_amd_keeps_physical_die_readings_and_excludes_control_offsets(self):
+        self.chip(0, "k10temp", [("temp1", "Tctl", 85000), ("temp2", "Tdie", 65000), ("temp3", "Tccd1", 64000)])
+        self.assertEqual([s["label"] for s in self.agent.temperatures()], ["Tdie", "Tccd1"])
+
+    def test_sensor_identity_survives_hwmon_renumbering_and_separates_drives(self):
+        physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
+        self.chip(2, "nvme", [("temp1", "Composite", 51000)])
+        before = {s["celsius"]: s["id"] for s in self.agent.temperatures()}
+        renamed = physical.with_name("hwmon9")
+        physical.rename(renamed)
+        (self.hwmon / "hwmon1").unlink()
+        (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
+        self.assertEqual(before, {s["celsius"]: s["id"] for s in self.agent.temperatures()})
+        self.assertEqual(len(set(before.values())), 2)
+
+    def test_missing_hwmon_and_disappearing_devices_are_optional(self):
+        self.assertEqual(self.agent.temperatures(), [])
+        (self.hwmon / "hwmon0").symlink_to(self.root / "missing")
+        self.assertEqual(self.agent.temperatures(), [])
+        with patch.object(Path, "glob", side_effect=PermissionError()):
+            self.assertEqual(self.agent.temperatures(), [])
+
+    def test_direct_hwmon_child_keeps_identity_when_renumbered(self):
+        physical = self.chip(1, "nvme", [("temp1", "Composite", 50000)])
+        direct = physical.parent.parent / "hwmon8"
+        physical.rename(direct)
+        (self.hwmon / "hwmon1").unlink()
+        (self.hwmon / "hwmon8").symlink_to(direct, target_is_directory=True)
+        before = self.agent.temperatures()[0]["id"]
+        renamed = direct.with_name("hwmon9")
+        direct.rename(renamed)
+        (self.hwmon / "hwmon8").unlink()
+        (self.hwmon / "hwmon9").symlink_to(renamed, target_is_directory=True)
+        self.assertEqual(self.agent.temperatures()[0]["id"], before)
+
+    def test_inventory_size_is_bounded(self):
+        self.chip(0, "coretemp", [("temp" + str(i), "Core " + str(i), 65000) for i in range(1, 200)])
+        self.assertEqual(len(self.agent.temperatures()), 128)

agent/tests/test_uninstall.py

@@ -0,0 +1,350 @@
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_uninstall_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542"
+
+
+class CleanupDocker:
+    def __init__(self):
+        self.calls = []
+        self.items = {}
+        self.images = {}
+
+    def inspect(self, identity):
+        return self.items.get(identity)
+
+    def call(self, *args, **kwargs):
+        self.calls.append(args)
+        if args[:2] == ("container", "ls"):
+            return "\n".join(self.items)
+        if args[:2] == ("container", "rm"):
+            self.items.pop(args[-1], None)
+            return ""
+        if args[:2] == ("image", "inspect"):
+            item = self.images.get(args[-1])
+            return json.dumps([item]) if item else None
+        if args[:2] == ("image", "rm"):
+            self.images.pop(args[-1], None)
+            return ""
+        raise AssertionError("Unexpected Docker request: " + str(args))
+
+
+class UninstallTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name).resolve()
+        self.root = self.base / "host-data"
+        self.config_dir = self.base / "etc" / "electric-cherry"
+        self.bin_dir = self.base / "libexec"
+        self.unit_dir = self.base / "systemd"
+        for directory in (self.config_dir, self.bin_dir, self.unit_dir):
+            directory.mkdir(parents=True)
+        replacements = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
+            "AGENT_UNIT": self.unit_dir / "electric-cherry-agent.service",
+            "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
+            "UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
+            "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_EXECUTABLE": self.bin_dir / "electric-cherry-update.py",
+            "UPDATE_UNIT": self.unit_dir / "electric-cherry-update.service",
+            "UPDATE_TIMER": self.unit_dir / "electric-cherry-update.timer",
+        }
+        self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
+        # Unit tests run without root privileges. Keep actual path, symlink,
+        # file-type and containment checks while substituting owner checks.
+        def test_protected_path(value, missing=False):
+            path = Path(value)
+            if any(current.is_symlink() for current in (path, *path.parents)):
+                raise module.AgentError("Symlink ownership is invalid")
+            if not missing and not path.exists():
+                raise module.AgentError("Host uninstall files are unavailable")
+            return path
+        self.patches.extend([patch.object(module, "protected_path", side_effect=test_protected_path), patch.object(module, "systemctl"), patch.object(module, "sync_parent")])
+        for entry in self.patches:
+            entry.start()
+        self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
+        self.agent = module.Agent(self.config)
+        self.docker = CleanupDocker()
+        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT, module.UPDATE_EXECUTABLE, module.UPDATE_UNIT, module.UPDATE_TIMER):
+            path.write_text("installed")
+        for path in (module.AGENT_EXECUTABLE, module.UNINSTALL_EXECUTABLE):
+            path.with_suffix(".py.update").write_text("interrupted update")
+        module.atomic_json(module.AGENT_CONFIG, self.config)
+        module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
+        self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
+        self.game.parent.mkdir(parents=True)
+        self.game.write_bytes(b"game DLC")
+        (self.base / "unrelated.txt").write_text("keep")
+        self.operation = {"id": "delete-one", "kind": "host.decommission", "payload": {"hostId": "host-one", "operationId": "delete-one", "receiptToken": "receipt-only-secret", "receiptUrl": "/api/agent/decommission/delete-one"}}
+
+    def tearDown(self):
+        self.agent.journal.close()
+        for entry in reversed(self.patches):
+            entry.stop()
+        self.temp.cleanup()
+
+    def prepare(self):
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "running", result)
+        return module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+
+    def container(self, identity, owned=True, host="host-one", root=None, image="image-one"):
+        labels = {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1", module.LABEL + ".host": host} if owned else {}
+        return {"Id": identity, "Image": image, "Config": {"Labels": labels}, "State": {"Running": False, "Status": "exited"}, "Mounts": [{"Type": "bind", "Source": str((root or self.root) / "instances" / INSTANCE / "1" / "app"), "Destination": "/home/plutainer/app"}]}
+
+    def test_final_receipt_follows_data_and_self_removal(self):
+        worker = self.prepare()
+        stages = []
+        def receipt(stage, error=None):
+            stages.append(stage)
+            self.assertFalse(self.root.exists())
+            self.assertFalse(module.AGENT_CONFIG.exists())
+            self.assertFalse(module.AGENT_EXECUTABLE.exists())
+            self.assertFalse(module.AGENT_UNIT.exists())
+            self.assertFalse(module.UPDATE_EXECUTABLE.exists())
+            self.assertFalse(module.UPDATE_UNIT.exists())
+            self.assertFalse(module.UPDATE_TIMER.exists())
+            self.assertFalse(module.AGENT_EXECUTABLE.with_suffix(".py.update").exists())
+            self.assertFalse(module.UNINSTALL_EXECUTABLE.with_suffix(".py.update").exists())
+            if stage == "cleaned":
+                self.assertTrue(module.UNINSTALL_REQUEST.exists())
+                self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
+            else:
+                self.assertFalse(self.config_dir.exists())
+                self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
+                self.assertFalse(module.UNINSTALL_UNIT.exists())
+        worker.receipt = receipt
+        worker.run()
+        self.assertEqual(stages, ["cleaned", "completed"])
+        self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
+        module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
+        module.systemctl.assert_any_call("disable", "--now", "electric-cherry-update.timer")
+        module.systemctl.assert_any_call("stop", "electric-cherry-update.service")
+        module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
+
+    def test_handoff_is_durable_and_never_reports_operation_completed(self):
+        worker = self.prepare()
+        self.assertEqual(worker.job["operationId"], "delete-one")
+        self.assertNotIn("credential", worker.job)
+        self.assertTrue(self.agent.stopping.is_set())
+        module.systemctl.assert_any_call("enable", "--now", "--no-block", "electric-cherry-uninstall.service")
+        self.assertEqual(self.agent.journal.get("delete-one")["state"], "running")
+
+    def test_failed_checkpoint_cannot_leave_a_runnable_uninstall_request(self):
+        with patch.object(self.agent.journal, "checkpoint", side_effect=OSError("journal unavailable")):
+            result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "failed")
+        self.assertFalse(module.UNINSTALL_REQUEST.exists())
+        module.systemctl.assert_not_called()
+
+    def test_pending_cleanup_resumes_after_controller_unavailable(self):
+        worker = self.prepare()
+        worker.receipt = lambda *_: (_ for _ in ()).throw(module.AgentError("offline"))
+        with self.assertRaisesRegex(module.AgentError, "offline"):
+            worker.run()
+        self.assertFalse(self.root.exists())
+        self.assertTrue(module.UNINSTALL_REQUEST.exists())
+        self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
+        self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["dataRemoved"])
+        before = len(self.docker.calls)
+        resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        stages = []
+        resumed.receipt = lambda stage, error=None: stages.append(stage)
+        resumed.run()
+        self.assertEqual(stages, ["cleaned", "completed"])
+        self.assertEqual(len(self.docker.calls), before)
+
+    def test_lost_final_receipt_retries_after_all_files_are_gone(self):
+        worker = self.prepare()
+        stages = []
+        def receipt(stage, error=None):
+            stages.append(stage)
+            if stage == "completed":
+                self.assertFalse(self.config_dir.exists())
+                if stages.count("completed") == 1:
+                    raise module.AgentError("lost reply")
+        worker.receipt = receipt
+        with patch.object(module.time, "sleep") as wait:
+            worker.run()
+        self.assertEqual(stages, ["cleaned", "completed", "completed"])
+        wait.assert_called_once_with(15)
+
+    def test_partial_self_removal_retries_without_its_unit_or_executable(self):
+        worker = self.prepare()
+        stages = []
+        worker.receipt = lambda stage, error=None: stages.append(stage)
+        original = shutil.rmtree
+        attempts = 0
+        def interrupted(path, *args, **kwargs):
+            nonlocal attempts
+            if Path(path) == self.config_dir:
+                attempts += 1
+                if attempts == 1:
+                    self.assertFalse(module.UNINSTALL_UNIT.exists())
+                    self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
+                    raise OSError("temporarily busy")
+            return original(path, *args, **kwargs)
+        with patch.object(module.shutil, "rmtree", side_effect=interrupted), patch.object(module.time, "sleep") as wait:
+            worker.run()
+        self.assertEqual(stages, ["cleaned", "failed", "completed"])
+        self.assertFalse(self.config_dir.exists())
+        wait.assert_called_once_with(15)
+
+    def test_partial_tree_removal_recovers_without_deleted_marker(self):
+        worker = self.prepare()
+        original = shutil.rmtree
+        def interrupted(path, *args, **kwargs):
+            if Path(path) == self.root:
+                (self.root / module.ROOT_MARKER).unlink()
+                self.game.unlink()
+                raise OSError("power loss")
+            return original(path, *args, **kwargs)
+        worker.receipt = lambda *_: None
+        with patch.object(module.shutil, "rmtree", side_effect=interrupted):
+            with self.assertRaisesRegex(OSError, "power loss"):
+                worker.run()
+        self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["removalStarted"])
+        self.assertFalse((self.root / module.ROOT_MARKER).exists())
+        resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        resumed.receipt = lambda *_: None
+        resumed.run()
+        self.assertFalse(self.root.exists())
+
+    def test_replaced_data_root_cannot_be_deleted_on_resume(self):
+        self.prepare()
+        self.root.rename(self.base / "original-data")
+        self.root.mkdir()
+        (self.root / "precious.txt").write_text("keep")
+        with self.assertRaisesRegex(module.AgentError, "changed"):
+            module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        self.assertTrue((self.root / "precious.txt").exists())
+
+    def test_only_owned_containers_and_unused_recorded_images_are_removed(self):
+        aliases = {"example/runtime:one": "image-one", "example/runtime:shared": "image-shared", "example/runtime:changed": "image-old"}
+        module.atomic_json(self.root / "agent" / "runtime-images.json", aliases)
+        self.docker.images = {"example/runtime:one": {"Id": "image-one"}, "example/runtime:shared": {"Id": "image-shared"}, "example/runtime:changed": {"Id": "image-new"}, "example/unrelated:one": {"Id": "image-unrelated"}}
+        self.docker.items = {"ours": self.container("ours"), "other": self.container("other", host="host-two", root=self.base / "different-install", image="image-shared")}
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        worker.run()
+        self.assertNotIn("ours", self.docker.items)
+        self.assertIn("other", self.docker.items)
+        self.assertNotIn("example/runtime:one", self.docker.images)
+        self.assertIn("example/runtime:shared", self.docker.images)
+        self.assertIn("example/runtime:changed", self.docker.images)
+        self.assertIn("example/unrelated:one", self.docker.images)
+        self.assertFalse(any("prune" in call or "volume" in call for call in self.docker.calls))
+
+    def test_unrelated_container_using_root_blocks_any_removal(self):
+        self.docker.items = {"foreign": self.container("foreign", owned=False)}
+        worker = self.prepare()
+        stages = []
+        worker.receipt = lambda stage, error=None: stages.append((stage, error))
+        with self.assertRaisesRegex(module.AgentError, "unrelated container"):
+            worker.run()
+        self.assertEqual(stages[0][0], "failed")
+        self.assertTrue(self.game.exists())
+        self.assertTrue(module.AGENT_EXECUTABLE.exists())
+        self.assertIn("foreign", self.docker.items)
+
+    def test_running_or_unknown_managed_container_blocks_removal(self):
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        for state in ({"Running": True, "Status": "running"}, {"Running": False, "Status": "restarting"}, {}):
+            with self.subTest(state=state):
+                self.docker.items = {"ours": {**self.container("ours"), "State": state}}
+                with self.assertRaisesRegex(module.AgentError, "still running or its state is unknown"):
+                    worker.run()
+                self.assertTrue(self.game.exists())
+                self.assertIn("ours", self.docker.items)
+        self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
+
+    def test_legacy_container_requires_exact_managed_bind_mount(self):
+        worker = self.prepare()
+        legacy = self.container("legacy")
+        legacy["Config"]["Labels"].pop(module.LABEL + ".host")
+        self.assertTrue(worker.owned_container(legacy))
+        legacy["Mounts"][0]["Source"] = str(self.base / "elsewhere")
+        self.assertFalse(worker.owned_container(legacy))
+
+    def test_missing_marker_and_wrong_host_never_schedule_helper(self):
+        (self.root / module.ROOT_MARKER).unlink()
+        self.assertFalse(self.agent.can_uninstall())
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "failed")
+        module.systemctl.assert_not_called()
+        self.assertTrue(self.game.exists())
+
+    def test_receipt_path_and_identity_cannot_be_redirected(self):
+        for key, value in (("receiptUrl", "https://attacker.example"), ("hostId", "other"), ("operationId", "other")):
+            with self.subTest(key=key):
+                operation = {**self.operation, "id": "check-" + key, "payload": {**self.operation["payload"], "operationId": "check-" + key, "receiptUrl": "/api/agent/decommission/check-" + key, key: value}}
+                self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        module.systemctl.assert_not_called()
+
+    def test_symlink_inside_owned_tree_is_unlinked_without_following(self):
+        outside = self.base / "unrelated.txt"
+        (self.root / "libraries" / "symlink").symlink_to(outside)
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        worker.run()
+        self.assertEqual(outside.read_text(), "keep")
+
+    def test_unsafe_roots_and_installation_overlap_are_rejected(self):
+        for value in ("/", "/var/lib", "/srv", "/var/lib/../lib", "/var//lib/anything", str(self.config_dir), str(self.bin_dir)):
+            with self.subTest(value=value):
+                with self.assertRaises(module.AgentError):
+                    module.uninstall_root(value)
+
+    def test_root_mount_or_nested_mount_cannot_be_recursively_deleted(self):
+        original = Path.read_text
+        for mount in (self.root, self.root / "libraries"):
+            def read_text(path, *args, **kwargs):
+                if str(path) == "/proc/1/mountinfo":
+                    return "21 1 8:1 / " + str(mount) + " rw - ext4 /dev/sda1 rw\n"
+                return original(path, *args, **kwargs)
+            with patch.object(Path, "read_text", read_text):
+                with self.assertRaisesRegex(module.AgentError, "Unmount"):
+                    module.uninstall_root(str(self.root))
+
+    def test_marker_directory_is_rejected_before_reading(self):
+        marker = self.root / module.ROOT_MARKER
+        marker.unlink()
+        marker.mkdir()
+        with self.assertRaisesRegex(module.AgentError, "regular file"):
+            module.uninstall_ownership(self.config)
+
+    def test_boot_reconciles_handoff_instead_of_marking_complete(self):
+        self.prepare()
+        self.agent.journal.close()
+        self.agent = module.Agent(self.config)
+        self.assertEqual(self.agent.journal.get("delete-one")["state"], "unknown")
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "running")
+        self.assertEqual(json.loads(module.UNINSTALL_REQUEST.read_text())["operationId"], "delete-one")
+
+
+class OwnershipPathTests(unittest.TestCase):
+    def test_root_ownership_group_writes_and_symlinks_are_rejected(self):
+        for mode, uid in ((stat.S_IFDIR | 0o777, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0)):
+            with self.subTest(mode=mode, uid=uid):
+                value = os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))
+                with patch.object(Path, "lstat", return_value=value):
+                    with self.assertRaisesRegex(module.AgentError, "only by root"):
+                        module.protected_path("/var/lib/electric-cherry")
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_update_integration.py

@@ -0,0 +1,234 @@
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import tempfile
+import threading
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_update_integration", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class UpdateIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name)
+        self.config_dir = self.base / "config"
+        self.config_dir.mkdir()
+        replacements = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
+        self.patches.append(patch.object(module, "protected_file", side_effect=lambda path, **_: path))
+        for entry in self.patches:
+            entry.start()
+        self.agent = module.Agent({"root": str(self.base / "data"), "configPath": str(module.AGENT_CONFIG), "controller": "https://controller.example", "hostId": "one", "credential": "secret"})
+        self.agent.can_update = lambda: True
+        self.agent.inventory = lambda: {"agentVersion": module.VERSION}
+
+    def tearDown(self):
+        self.agent.journal.close()
+        for entry in reversed(self.patches):
+            entry.stop()
+        self.temporary.cleanup()
+
+    def assert_activity_held(self):
+        with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+            with self.assertRaises(BlockingIOError):
+                fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+    def test_execution_and_reports_exclude_updates_but_long_poll_does_not(self):
+        operation = {"id": "one", "kind": "runtime.pull", "payload": {}}
+        calls = []
+        def request(method, path, body=None):
+            if method != "GET":
+                self.assert_activity_held()
+            else:
+                with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+                    fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            calls.append((method, path))
+            return {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            return {"ok": True}
+        self.agent.request = request
+        self.agent.dispatch = dispatch
+        self.agent.poll_operations()
+        self.assertEqual(self.agent.journal.get("one")["state"], "completed")
+        self.assertEqual(len(calls), 4)
+        self.assertIsNone(self.agent.current_operation)
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_updater_lock_defers_delivery_without_blocking_health(self):
+        self.agent.request = lambda method, path, body=None: {"host": {}}
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            with patch.object(self.agent, "request", wraps=self.agent.request) as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+                self.agent.heartbeat()
+                self.assertTrue(module.AGENT_HEALTH.exists())
+
+    def test_journal_blocks_operations_during_verification_and_recovery(self):
+        module.UPDATE_JOURNAL.parent.mkdir()
+        for content in ('{"id":"candidate","phase":"verifying"}', '{"id":"candidate","phase":"rollback"}', '{broken'):
+            module.UPDATE_JOURNAL.write_text(content)
+            with patch.object(self.agent, "request") as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+        module.UPDATE_JOURNAL.unlink()
+        with patch.object(self.agent, "request", return_value={"operations": []}) as request:
+            self.agent.poll_operations()
+            request.assert_called_once_with("GET", "/api/agent/operations?lane=write&wait=20")
+
+    def test_read_lane_completes_while_mutation_is_busy_and_excludes_updater(self):
+        read_done = threading.Event()
+        operation = {"id": "read-one", "kind": "host.inspect", "payload": {}}
+        self.agent.request = lambda method, path, body=None: {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            read_done.set()
+            self.agent.stopping.set()
+            return {"ok": True}
+        self.agent.dispatch = dispatch
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            reader = threading.Thread(target=self.agent.read_loop)
+            reader.start()
+            self.assertTrue(read_done.wait(2), "Read waited behind unrelated mutation activity")
+            reader.join(2)
+        self.assertFalse(reader.is_alive())
+        self.assertEqual(self.agent.journal.get("read-one")["state"], "completed")
+        self.assertIsNone(self.agent.executing_id)
+
+    def test_read_lane_never_dispatches_mutations(self):
+        def request(method, path, body=None):
+            self.agent.stopping.set()
+            return {"operations": [{"id": "bad-read", "kind": "instance.remove", "payload": {}}]}
+        self.agent.request = request
+        with patch.object(self.agent, "dispatch") as dispatch:
+            self.agent.read_loop()
+            dispatch.assert_not_called()
+
+    def test_health_lane_runs_while_file_receipt_for_same_instance_is_slow(self):
+        receipt_waiting, release_receipt, health_done = threading.Event(), threading.Event(), threading.Event()
+        file_op = {"id": "file-one", "kind": "files.read", "instanceId": "same", "payload": {}}
+        health_op = {"id": "health-one", "kind": "instance.inspect", "instanceId": "same", "payload": {}}
+        errors = []
+        def request(method, path, body=None):
+            if method == "GET":
+                return {"operations": [health_op if "lane=health" in path else file_op]}
+            if path.endswith("file-one") and body["state"] == "completed":
+                receipt_waiting.set()
+                release_receipt.wait(3)
+            return {}
+        def dispatch(operation):
+            if operation["id"] == "health-one":
+                health_done.set()
+                self.agent.stopping.set()
+            return {"ok": True}
+        def run(lane):
+            try: self.agent.read_loop(lane)
+            except Exception as error: errors.append(error)
+        self.agent.request, self.agent.dispatch = request, dispatch
+        files = threading.Thread(target=run, args=("read",))
+        health = threading.Thread(target=run, args=("health",))
+        files.start()
+        try:
+            self.assertTrue(receipt_waiting.wait(2))
+            health.start()
+            self.assertTrue(health_done.wait(2), "Health waited for a file receipt")
+        finally:
+            self.agent.stopping.set()
+            release_receipt.set()
+            files.join(3)
+            if health.ident is not None: health.join(3)
+        self.assertFalse(files.is_alive())
+        self.assertFalse(health.is_alive())
+        self.assertEqual(errors, [])
+        self.assertEqual(self.agent.journal.get("health-one")["state"], "completed")
+
+    def test_instance_mutations_and_global_mutations_exclude_observations(self):
+        operation = {"instanceId": "one"}
+        entered = threading.Event()
+        release = threading.Event()
+        def read():
+            with self.agent.read_gate.read(), self.agent.instance_lock(operation):
+                entered.set()
+                release.wait(2)
+        with self.agent.instance_lock(operation):
+            reader = threading.Thread(target=read)
+            reader.start()
+            self.assertFalse(entered.wait(0.05))
+        self.assertTrue(entered.wait(2))
+        writer_entered = threading.Event()
+        def write():
+            with self.agent.read_gate.write():
+                writer_entered.set()
+        writer = threading.Thread(target=write)
+        writer.start()
+        self.assertFalse(writer_entered.wait(0.05))
+        release.set()
+        reader.join(2)
+        writer.join(2)
+        self.assertTrue(writer_entered.is_set())
+
+    def test_failed_controller_contact_never_writes_healthy_receipt(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.heartbeat()
+        self.assertFalse(module.AGENT_HEALTH.exists())
+
+    def test_health_proves_current_process_version_and_transaction(self):
+        module.atomic_json(module.UPDATE_JOURNAL, {"id": "candidate-id", "phase": "verifying"})
+        with patch.object(self.agent, "request", return_value={"host": {}}), patch.object(module.time, "time", return_value=12345.5):
+            self.agent.heartbeat()
+        self.assertEqual(json.loads(module.AGENT_HEALTH.read_text()), {"version": module.VERSION, "pid": os.getpid(), "checkedAt": 12345.5, "updateId": "candidate-id"})
+        module.UPDATE_JOURNAL.unlink()
+        self.agent.record_health()
+        self.assertNotIn("updateId", json.loads(module.AGENT_HEALTH.read_text()))
+
+    def test_real_agent_health_receipt_is_accepted_by_the_independent_updater(self):
+        updater_spec = importlib.util.spec_from_file_location("ec_update_contract", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+        updater_module = importlib.util.module_from_spec(updater_spec)
+        updater_spec.loader.exec_module(updater_module)
+        journal = {"id": "candidate-id", "phase": "verifying", "targetVersion": module.VERSION, "switchedAt": module.time.time() - 1}
+        module.atomic_json(module.UPDATE_JOURNAL, journal)
+        with patch.object(self.agent, "request", return_value={"host": {}}):
+            self.agent.heartbeat()
+        updater = updater_module.Updater()
+        with patch.object(updater, "check_uninstall"), patch.object(updater_module, "AGENT_HEALTH", module.AGENT_HEALTH), patch.object(updater_module, "trusted", side_effect=lambda path, **_: path), patch.object(updater_module, "service_state", return_value=(True, os.getpid())):
+            self.assertTrue(updater.wait_healthy(journal))
+
+    def test_operation_exception_releases_update_lock(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.poll_operations()
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_update_inventory_excludes_local_recovery_and_secret_fields(self):
+        module.atomic_json(module.UPDATE_STATUS, {"state": "failed", "targetVersion": "1.3.0", "currentVersion": "untrusted", "error": "Restored previous agent", "checkedAt": 1700000000, "credential": "secret", "journal": {"private": True}, "failedSha256": "digest"})
+        self.assertEqual(self.agent.update_inventory(), {"enabled": True, "state": "failed", "currentVersion": module.VERSION, "targetVersion": "1.3.0", "error": "Restored previous agent", "checkedAt": "2023-11-14T22:13:20Z"})
+
+    def test_malformed_update_status_is_visible_without_breaking_inventory(self):
+        module.UPDATE_STATUS.write_text("[]")
+        self.assertEqual(self.agent.update_inventory()["state"], "failed")
+
+    def test_shutdown_never_polls_work(self):
+        self.agent.stopping.set()
+        with patch.object(self.agent, "request") as request:
+            self.agent.poll_operations()
+            request.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_updates.py

@@ -0,0 +1,653 @@
+import contextlib
+import fcntl
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_agent_update", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+PREVIOUS = b'VERSION = "1.0.0"\n# previous agent\n'
+CANDIDATE = b'VERSION = "1.1.0"\n# next agent\n'
+
+
+class PowerLoss(BaseException):
+    pass
+
+
+class Response(io.BytesIO):
+    def __init__(self, value, headers=None):
+        super().__init__(value)
+        self.headers = headers or {}
+        self.status = 200
+
+
+class UpdateTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name).resolve()
+        self.config_dir = self.base / "etc" / "electric-cherry"
+        self.bin_dir = self.base / "libexec"
+        self.root = self.base / "data"
+        for directory in (self.config_dir, self.bin_dir, self.root / "agent"):
+            directory.mkdir(parents=True)
+        paths = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
+            "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
+            "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_LOCK": self.config_dir / "update.lock",
+            "UPDATE_DIR": self.config_dir / "update",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, name, value) for name, value in paths.items()]
+        def trusted_for_test(value, missing=False, directory=False):
+            path = Path(value)
+            if any(item.is_symlink() for item in (path, *path.parents)):
+                raise module.UpdateError("Unsafe symbolic link")
+            if not path.exists():
+                if missing:
+                    return path
+                raise module.UpdateError("File unavailable")
+            if directory and not path.is_dir() or not directory and not path.is_file():
+                raise module.UpdateError("Wrong file type")
+            return path
+        self.public_key = b"repository public key"
+        key_patch = patch.object(module, "fetch_release_key", return_value=self.public_key)
+        self.fetch_key = key_patch.start()
+        self.addCleanup(key_patch.stop)
+        self.signature_patch = patch.object(module, "verify_release_signature")
+        self.patches.extend([patch.object(module, "trusted", side_effect=trusted_for_test), patch.object(module, "sync_parent"), patch.object(module, "systemctl", side_effect=self.systemctl), self.signature_patch])
+        for item in self.patches:
+            mocked = item.start()
+            if item is self.signature_patch:
+                self.signature_verifier = mocked
+        self.config = {"hostId": "host-one", "root": str(self.root), "controller": "https://controller.example", "credential": "secret-enrollment-credential"}
+        self.config_bytes = json.dumps(self.config).encode()
+        module.AGENT_CONFIG.write_bytes(self.config_bytes)
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"}))
+        module.AGENT_EXECUTABLE.write_bytes(PREVIOUS)
+        module.UNINSTALL_EXECUTABLE.write_bytes(PREVIOUS)
+        (self.root / "games.ff").write_bytes(b"large game files unchanged")
+        self.commands = []
+        self.active = True
+        self.pid = 123
+        self.write_health = True
+        self.downloads = []
+        self.source = CANDIDATE
+        self.signature = b"detached signature"
+        self.release = {"version": "1.1.0", "sha256": module.sha256(CANDIDATE), "protocolVersion": 1, "updaterProtocolVersion": 2, "platform": "debian-amd64"}
+
+    def tearDown(self):
+        for item in reversed(self.patches):
+            item.stop()
+        self.temp.cleanup()
+
+    def systemctl(self, *args):
+        self.commands.append(args)
+        if args == ("stop", module.AGENT_SERVICE):
+            self.active = False
+            return ""
+        if args == ("start", module.AGENT_SERVICE):
+            # The candidate must be able to take the operation lock at startup.
+            with module.exclusive_lock(self.root / "agent" / "activity.lock") as activity:
+                self.assertIsNotNone(activity, "Updater held the operation lock while starting the agent")
+            self.active = True
+            self.pid += 1
+            if self.write_health and module.UPDATE_JOURNAL.exists():
+                journal = json.loads(module.UPDATE_JOURNAL.read_text())
+                module.AGENT_HEALTH.write_text(json.dumps({"version": module.source_version(module.AGENT_EXECUTABLE.read_bytes()), "pid": self.pid, "checkedAt": time.time(), "updateId": journal["id"]}))
+            return ""
+        if args == ("show", module.AGENT_SERVICE, "--property=ActiveState", "--property=MainPID"):
+            return "ActiveState=" + ("active" if self.active else "inactive") + "\nMainPID=" + str(self.pid if self.active else 0) + "\n"
+        self.fail("Unexpected service command " + str(args))
+
+    def fetch(self, path, limit):
+        self.downloads.append(path)
+        if path.endswith("manifest.json"):
+            return json.dumps(self.release).encode()
+        if path.endswith("manifest.json.sig"):
+            if isinstance(self.signature, Exception):
+                raise self.signature
+            return self.signature
+        if path.endswith("ec-agent.py"):
+            return self.source
+        self.fail("Unexpected download " + path)
+
+    def updater(self):
+        updater = module.Updater()
+        updater.fetch = self.fetch
+        return updater
+
+    def status(self):
+        return json.loads(module.UPDATE_STATUS.read_text())
+
+    def assert_unchanged(self):
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+
+    def test_verified_upgrade_waits_for_authenticated_candidate_health(self):
+        self.assertEqual(self.updater().run(), "updated")
+        self.signature_verifier.assert_called_once_with(json.dumps(self.release).encode(), self.signature, self.public_key)
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+        self.assertEqual(self.status()["currentVersion"], "1.1.0")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertEqual(next(command for command in self.commands if command[0] != "show"), ("stop", module.AGENT_SERVICE))
+
+    def test_invalid_signature_rejects_attacker_source_and_matching_manifest_checksum(self):
+        self.source = b'VERSION = "1.1.0"\nraise RuntimeError("attacker code")\n'
+        self.release["sha256"] = module.sha256(self.source)
+        self.signature_verifier.side_effect = module.IntegrityError("Invalid release signature")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"])
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse((module.UPDATE_DIR / "failed-release.json").exists())
+        self.assertNotIn("targetVersion", self.status())
+
+    def test_signature_is_verified_before_manifest_json_is_parsed(self):
+        updater = self.updater()
+        updater.fetch = lambda path, limit: b"untrusted invalid JSON" if path.endswith("manifest.json") else self.signature
+        self.signature_verifier.side_effect = module.IntegrityError("Invalid release signature")
+        with patch.object(module.json, "loads", side_effect=AssertionError("Parsed unsigned metadata")):
+            with self.assertRaisesRegex(module.IntegrityError, "Invalid release signature"):
+                updater.manifest()
+        self.signature_verifier.assert_called_once_with(b"untrusted invalid JSON", self.signature, self.public_key)
+
+    def test_missing_signature_never_downloads_code_or_stops_agent(self):
+        self.signature = module.ContactError("No signature is published")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertIn("signature", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.signature_verifier.assert_not_called()
+
+    def test_equal_or_older_versions_do_not_download_or_restart(self):
+        for target in ("1.0.0", "0.9.9"):
+            with self.subTest(target=target):
+                self.release["version"] = target
+                self.assertEqual(self.updater().run(), "current")
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_every_check_fetches_key_before_controller_even_without_a_new_version(self):
+        self.release["version"] = "1.0.0"
+        keys = [b"first public key", b"replacement public key"]
+        self.fetch_key.side_effect = keys
+        updater = self.updater()
+        def controller_fetch(path, limit):
+            self.assertGreater(self.fetch_key.call_count, self.signature_verifier.call_count)
+            return self.fetch(path, limit)
+        updater.fetch = controller_fetch
+        for _ in keys:
+            self.assertEqual(updater.run(), "current")
+        self.assertEqual(self.fetch_key.call_count, 2)
+        self.assertEqual([call.args[2] for call in self.signature_verifier.call_args_list], keys)
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+
+    def test_repository_failure_never_uses_old_host_key_or_contacts_controller(self):
+        (self.config_dir / "release-signing.gpg").write_bytes(b"old trusted key")
+        self.fetch_key.side_effect = module.ContactError("Repository unavailable")
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assert_unchanged()
+        self.assertEqual(self.downloads, [])
+        self.assertEqual(self.commands, [])
+        self.assertTrue(self.active)
+        self.signature_verifier.assert_not_called()
+
+    def test_busy_operation_defers_update_without_stopping_agent(self):
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("executing an operation", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_deliberately_stopped_agent_is_not_updated_or_started(self):
+        self.active = False
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("Agent service is not running", self.status()["error"])
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse(self.active)
+        self.assertFalse(any(command[0] in ("start", "stop") for command in self.commands))
+
+    def test_concurrent_worker_does_not_overwrite_active_worker_status(self):
+        module.UPDATE_STATUS.write_text('{"state":"updating"}')
+        with module.exclusive_lock(module.UPDATE_LOCK):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(self.status(), {"state": "updating"})
+        self.assertEqual(self.downloads, [])
+
+    def test_uninstall_request_prevents_any_download_or_mutation(self):
+        module.UNINSTALL_REQUEST.write_text("{}")
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("deletion", self.status()["error"])
+        self.assertEqual(self.downloads, [])
+        self.assertEqual(self.commands, [])
+        self.assert_unchanged()
+
+    def test_uninstall_starting_during_download_wins_before_code_changes(self):
+        updater = self.updater()
+        def fetch(path, limit):
+            data = self.fetch(path, limit)
+            if path.endswith("ec-agent.py"):
+                module.UNINSTALL_REQUEST.write_text("{}")
+            return data
+        updater.fetch = fetch
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+
+    def test_controller_unavailable_keeps_running_agent(self):
+        updater = self.updater()
+        updater.fetch = lambda *_: (_ for _ in ()).throw(module.ContactError("Controller unavailable"))
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertEqual(self.commands, [])
+
+    def test_checksum_failure_holds_artifact_without_executing_it(self):
+        self.source = CANDIDATE + b"# changed in transit"
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertIn("checksum", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.downloads.clear()
+        self.source = CANDIDATE
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"])
+
+    def test_invalid_or_mismatched_python_version_is_never_installed(self):
+        for source in (b'VERSION = "1.1.0"\ninvalid syntax ?\n', b'VERSION = "5.0.0"\n'):
+            with self.subTest(source=source):
+                self.source = source
+                self.release["sha256"] = module.sha256(source)
+                self.assertEqual(self.updater().run(), "failed")
+                self.assert_unchanged()
+                self.assertEqual(self.commands, [])
+
+    def test_incompatible_protocol_platform_or_version_never_downloads_code(self):
+        original = dict(self.release)
+        for key, value in (("protocolVersion", 2), ("protocolVersion", True), ("platform", "linux-arm64"), ("version", "1.1.0-rc1"), ("version", "01.1.0"), ("updaterProtocolVersion", 1), ("updaterProtocolVersion", 3)):
+            with self.subTest(key=key, value=value):
+                self.release = {**original, key: value}
+                self.assertIn(self.updater().run(), ("failed", "deferred"))
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_failed_health_rolls_back_both_files_then_holds_release(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(self.status()["currentVersion"], "1.0.0")
+        self.assertEqual(sum(command == ("start", module.AGENT_SERVICE) for command in self.commands), 2)
+        self.downloads.clear()
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json", "/api/agent/release/manifest.json.sig"])
+
+    def test_changed_artifact_of_failed_target_can_be_retried(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.source = CANDIDATE + b"# corrected release\n"
+        self.release["sha256"] = module.sha256(self.source)
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), self.source)
+
+    def crash_during_second_replacement(self):
+        original = module.atomic_bytes
+        def interrupt(path, value, mode=0o600):
+            if Path(path) == module.UNINSTALL_EXECUTABLE and value == CANDIDATE:
+                raise PowerLoss()
+            return original(path, value, mode)
+        with patch.object(module, "atomic_bytes", side_effect=interrupt):
+            with self.assertRaises(PowerLoss):
+                self.updater().run()
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_between_executable_replacements_recovers_from_backups(self):
+        self.crash_during_second_replacement()
+        self.downloads.clear()
+        self.signature_verifier.reset_mock()
+        self.fetch_key.reset_mock()
+        self.fetch_key.side_effect = module.ContactError("Repository offline during recovery")
+        self.signature_verifier.side_effect = module.IntegrityError("Signing keyring unavailable during recovery")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(self.downloads, [])
+        self.signature_verifier.assert_not_called()
+        self.fetch_key.assert_not_called()
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_after_verification_still_recovers_conservatively(self):
+        updater = self.updater()
+        updater.finish_journal = lambda: (_ for _ in ()).throw(PowerLoss())
+        with self.assertRaises(PowerLoss):
+            updater.run()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "verifying")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_corrupt_backup_keeps_journal_and_never_overwrites_code(self):
+        self.crash_during_second_replacement()
+        (module.UPDATE_DIR / "agent.backup.py").write_bytes(b"corrupt")
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_changed_enrollment_prevents_update_and_rollback(self):
+        self.crash_during_second_replacement()
+        changed = {**self.config, "hostId": "different-host"}
+        module.AGENT_CONFIG.write_text(json.dumps(changed))
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "different-host", "controller": "https://controller.example"}))
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_rollback_busy_lock_keeps_recovery_journal_for_next_tick(self):
+        self.crash_during_second_replacement()
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+
+    def test_unconfirmed_rollback_health_retains_backups_and_journal(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda _: False
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertTrue((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertIn("recovery is pending", self.status()["error"])
+        # A subsequent worker can safely confirm the restored release.
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_stale_wrong_pid_wrong_release_or_wrong_update_health_is_rejected(self):
+        updater = self.updater()
+        journal = {"id": "this-update", "targetVersion": "1.1.0", "switchedAt": time.time()}
+        valid = {"version": "1.1.0", "pid": self.pid, "checkedAt": time.time(), "updateId": "this-update"}
+        for key, value in (("pid", self.pid + 1), ("version", "1.0.0"), ("updateId", "previous-update"), ("checkedAt", journal["switchedAt"] - 1), ("checkedAt", float("nan")), ("checkedAt", time.time() + 100)):
+            with self.subTest(key=key, value=value):
+                module.AGENT_HEALTH.write_text(json.dumps({**valid, key: value}))
+                with patch.object(module, "HEALTH_TIMEOUT", 1), patch.object(module.time, "monotonic", side_effect=[0, 0, 2]), patch.object(module.time, "sleep"):
+                    self.assertFalse(updater.wait_healthy(journal))
+        module.AGENT_HEALTH.write_text(json.dumps(valid))
+        self.assertTrue(updater.wait_healthy(journal))
+
+    def test_symlinked_executable_is_rejected_without_touching_target(self):
+        external = self.base / "external.py"
+        external.write_bytes(PREVIOUS)
+        module.AGENT_EXECUTABLE.unlink()
+        module.AGENT_EXECUTABLE.symlink_to(external)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(external.read_bytes(), PREVIOUS)
+        self.assertEqual(self.commands, [])
+
+    def test_network_reader_enforces_header_and_stream_bounds(self):
+        updater = module.Updater()
+        updater.load_installation()
+        for response in (Response(b"abc", {"Content-Length": "100"}), Response(b"abcdef")):
+            with self.subTest(response=response), patch.object(module.urllib.request, "build_opener") as opener:
+                opener.return_value.open.return_value = response
+                with self.assertRaises(module.IntegrityError):
+                    updater.fetch("/api/agent/release/ec-agent.py", 5)
+
+    def test_trickling_or_truncated_download_defers_without_holding_release(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source")
+            with patch.object(module.time, "monotonic", side_effect=[0, 0, 121]):
+                with self.assertRaisesRegex(module.ContactError, "timed out"):
+                    updater.fetch("/api/agent/release/ec-agent.py", 100)
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            response = Response(b"source")
+            response.read1 = lambda *_: (_ for _ in ()).throw(module.http.client.IncompleteRead(b"partial"))
+            opener.return_value.open.return_value = response
+            with self.assertRaises(module.ContactError):
+                updater.fetch("/api/agent/release/ec-agent.py", 100)
+
+    def test_stale_code_staging_is_recovered_without_leaving_extra_executables(self):
+        temporary = module.AGENT_EXECUTABLE.with_name(module.AGENT_EXECUTABLE.name + ".update")
+        temporary.write_bytes(b"partial previous download")
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertFalse(temporary.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+
+    def test_download_uses_fixed_https_controller_and_no_enrollment_credentials(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source", {"Content-Length": "6"})
+            self.assertEqual(updater.fetch("/api/agent/release/ec-agent.py", 100), b"source")
+            request = opener.return_value.open.call_args.args[0]
+            self.assertEqual(request.full_url, "https://controller.example/api/agent/release/ec-agent.py")
+            self.assertIsNone(request.get_header("Authorization"))
+            redirect = opener.call_args.args[0]
+            with self.assertRaises(module.ContactError):
+                redirect.redirect_request(None, None, 302, "redirect", {}, "https://other.example")
+
+    def test_insecure_or_credential_embedded_controller_is_rejected(self):
+        for controller in ("http://controller.example", "https://token:secret@controller.example"):
+            with self.subTest(controller=controller):
+                module.AGENT_CONFIG.write_text(json.dumps({**self.config, "controller": controller}))
+                self.assertEqual(self.updater().run(), "failed")
+                self.assertEqual(self.downloads, [])
+                self.assertEqual(self.commands, [])
+
+
+class UpdateSecurityTests(unittest.TestCase):
+    def test_untrusted_ownership_writable_paths_and_special_files_are_rejected(self):
+        for mode, uid in ((stat.S_IFREG | 0o666, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0), (stat.S_IFIFO | 0o600, 0)):
+            with self.subTest(mode=mode, uid=uid), patch.object(Path, "lstat", return_value=os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))):
+                with self.assertRaises(module.UpdateError):
+                    module.trusted("/etc/electric-cherry/update/journal.json")
+
+    def test_source_is_parsed_and_compiled_without_execution(self):
+        source = b'VERSION="1.2.3"\nraise RuntimeError("Never execute downloaded code during verification")\n'
+        self.assertEqual(module.source_version(source), "1.2.3")
+
+    def test_numeric_version_order_and_strict_format(self):
+        self.assertGreater(module.version("1.10.0"), module.version("1.9.9"))
+        for value in ("v1.0.0", "1.0", "1.2.03", "1.0.0+local", "1.0.0-rc1", True, None, "1.0.000000000000"):
+            with self.subTest(value=value), self.assertRaises(module.IntegrityError):
+                module.version(value)
+
+
+class RepositoryKeyDownloadTests(unittest.TestCase):
+    def test_fixed_https_url_no_credentials_no_cache_and_no_redirects(self):
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"public keys")
+            self.assertEqual(module.fetch_release_key(), b"public keys")
+            request = opener.return_value.open.call_args.args[0]
+            self.assertEqual(request.full_url, "https://gitea.thewrightserver.net/josh/electric-cherry/raw/branch/main/agent/release-signing.gpg")
+            self.assertIsNone(request.get_header("Authorization"))
+            self.assertIsNone(request.get_header("Cookie"))
+            self.assertEqual(request.get_header("Cache-control"), "no-cache")
+            self.assertEqual(opener.return_value.open.call_args.kwargs["timeout"], 25)
+            redirect = opener.call_args.args[0]
+            for destination in ("https://attacker.example/key.gpg", "http://gitea.thewrightserver.net/key.gpg"):
+                with self.subTest(destination=destination), self.assertRaises(module.ContactError):
+                    redirect.redirect_request(None, None, 302, "redirect", {}, destination)
+
+    def test_missing_empty_oversized_and_truncated_keys_fail_closed(self):
+        cases = [Response(b""), Response(b"x", {"Content-Length": str(module.MAX_KEYRING + 1)}),
+                 Response(b"x" * (module.MAX_KEYRING + 1)),
+                 Response(b"partial", {"Content-Length": "100"}),
+                 Response(b"key", {"Content-Length": "invalid"})]
+        for status in (301, 304, 404, 500):
+            response = Response(b"public key")
+            response.status = status
+            cases.append(response)
+        for response in cases:
+            with self.subTest(status=response.status, headers=response.headers), patch.object(module.urllib.request, "build_opener") as opener:
+                opener.return_value.open.return_value = response
+                with self.assertRaises(module.UpdateError):
+                    module.fetch_release_key()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.side_effect = OSError("TLS or network failure")
+            with self.assertRaisesRegex(module.ContactError, "release-signing repository is unavailable"):
+                module.fetch_release_key()
+
+    def test_trickling_repository_download_has_a_deadline(self):
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"public key")
+            with patch.object(module.time, "monotonic", side_effect=[0, 0, 121]):
+                with self.assertRaisesRegex(module.ContactError, "timed out"):
+                    module.fetch_release_key()
+
+
+class SignatureVerificationFixture:
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.base = Path(self.temporary.name)
+        self.keyring = self.base / "release-signing.gpg"
+        self.keyring.write_bytes(b"trusted host public keys")
+        self.update_dir = self.base / "update"
+        self.update_dir.mkdir(mode=0o700)
+        entry = patch.object(module, "UPDATE_DIR", self.update_dir)
+        entry.start()
+        self.addCleanup(entry.stop)
+        # Test fixtures are owned by the test runner rather than system root.
+        # Root ownership and unsafe modes are checked independently above.
+        entry = patch.object(module, "trusted", side_effect=lambda path, **_: Path(path))
+        self.trusted = entry.start()
+        self.addCleanup(entry.stop)
+
+
+class SignatureVerificationTests(SignatureVerificationFixture, unittest.TestCase):
+    def test_gpgv_uses_only_downloaded_keyring_and_exact_bytes_with_an_isolated_home(self):
+        manifest = b'{"version": "1.2.3"}\n'
+        signature = b"binary detached signature\x00\xff"
+        def verify(command, **kwargs):
+            home = Path(command[2])
+            self.assertEqual(command[:2], ["gpgv", "--homedir"])
+            self.assertEqual(command[3:5], ["--keyring", str(home / "release-signing.gpg")])
+            self.assertEqual(Path(command[4]).read_bytes(), self.keyring.read_bytes())
+            self.assertEqual(home.parent, self.update_dir)
+            self.assertEqual(stat.S_IMODE(home.stat().st_mode), 0o700)
+            self.assertEqual(Path(command[5]).read_bytes(), signature)
+            self.assertEqual(Path(command[6]).read_bytes(), manifest)
+            self.assertEqual(kwargs["timeout"], 30)
+            self.assertEqual(kwargs["stdin"], subprocess.DEVNULL)
+            return subprocess.CompletedProcess(command, 0)
+        with patch.object(module.subprocess, "run", side_effect=verify) as run:
+            module.verify_release_signature(manifest, signature, self.keyring.read_bytes())
+            run.assert_called_once()
+        self.assertEqual(self.trusted.call_args_list[0].args, (self.update_dir,))
+        self.assertEqual(list(self.update_dir.iterdir()), [])
+
+    def test_empty_or_oversized_keyring_or_missing_signature_never_runs_a_verifier(self):
+        with patch.object(module.subprocess, "run") as run:
+            for key in (b"", b"x" * (module.MAX_KEYRING + 1)):
+                with self.subTest(size=len(key)), self.assertRaisesRegex(module.IntegrityError, "keyring"):
+                    module.verify_release_signature(b"manifest", b"signature", key)
+            with self.assertRaisesRegex(module.IntegrityError, "detached manifest signature"):
+                module.verify_release_signature(b"manifest", b"", self.keyring.read_bytes())
+            run.assert_not_called()
+
+    def test_bad_unknown_or_unavailable_verifier_fails_closed_and_cleans_temporary_files(self):
+        for result in (subprocess.CompletedProcess([], 1), subprocess.CompletedProcess([], 2), FileNotFoundError("gpgv missing"), subprocess.TimeoutExpired("gpgv", 30)):
+            with self.subTest(result=result):
+                kwargs = {"side_effect": result} if isinstance(result, Exception) else {"return_value": result}
+                with patch.object(module.subprocess, "run", **kwargs):
+                    with self.assertRaises(module.IntegrityError):
+                        module.verify_release_signature(b"manifest", b"signature", self.keyring.read_bytes())
+                self.assertEqual(list(self.update_dir.iterdir()), [])
+
+
+@unittest.skipUnless(shutil.which("gpg") and shutil.which("gpgv"), "GnuPG signing and verification tools are required")
+class RealSignatureVerificationTests(SignatureVerificationFixture, unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.keys = tempfile.TemporaryDirectory()
+        cls.addClassCleanup(cls.keys.cleanup)
+        cls.homes = []
+        for index in range(2):
+            home = Path(cls.keys.name) / str(index)
+            home.mkdir(mode=0o700)
+            cls.homes.append(home)
+            subprocess.run(["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", "", "--quick-generate-key", f"Release Test {index} <release-{index}@example.test>", "ed25519", "sign", "0"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            if shutil.which("gpgconf"):
+                cls.addClassCleanup(subprocess.run, ["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
+        cls.public_key = subprocess.run(["gpg", "--homedir", str(cls.homes[0]), "--batch", "--export"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
+
+    def sign(self, manifest, index=0):
+        return subprocess.run(["gpg", "--homedir", str(self.homes[index]), "--batch", "--pinentry-mode", "loopback", "--passphrase", "", "--digest-algo", "SHA512", "--detach-sign"], input=manifest, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout
+
+    def test_repository_key_rotation_removes_old_signer_on_the_next_check(self):
+        replacement = subprocess.run(["gpg", "--homedir", str(self.homes[1]), "--batch", "--export"],
+                                     check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
+        manifest = json.dumps({"version": "1.1.0", "sha256": module.sha256(CANDIDATE),
+                               "protocolVersion": 1, "updaterProtocolVersion": 2,
+                               "platform": "debian-amd64",
+                               "keyUrl": "https://controller.example/attacker-key"}).encode()
+        old_signature, new_signature = self.sign(manifest), self.sign(manifest, 1)
+        updater = module.Updater()
+        updater.installation = {"controller": "https://controller.example"}
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.side_effect = [Response(value) for value in (
+                self.public_key, manifest, old_signature,
+                replacement, manifest, old_signature,
+                replacement, manifest, new_signature)]
+            self.assertEqual(updater.manifest()["version"], "1.1.0")
+            with self.assertRaisesRegex(module.IntegrityError, "not signed by a trusted release key"):
+                updater.manifest()
+            self.assertEqual(updater.manifest()["version"], "1.1.0")
+            urls = [call.args[0].full_url for call in opener.return_value.open.call_args_list]
+            self.assertEqual(urls[::3], [module.RELEASE_KEY_URL] * 3)
+            self.assertNotIn("https://controller.example/attacker-key", urls)
+        self.assertEqual(list(self.update_dir.iterdir()), [])
+
+    def test_real_signatures_authenticate_exact_manifest_and_reject_unknown_keys(self):
+        self.keyring.write_bytes(self.public_key)
+        manifest = b'{"version":"1.1.0","sha256":"trusted-code-checksum"}\n'
+        signature = self.sign(manifest)
+        module.verify_release_signature(manifest, signature, self.keyring.read_bytes())
+        for changed_manifest, changed_signature in ((manifest + b" ", signature), (manifest.replace(b"trusted-code-checksum", b"attacker-code-checksum"), signature), (manifest, signature[:-1]), (manifest, self.sign(manifest, 1))):
+            with self.subTest(manifest=changed_manifest, signature=changed_signature):
+                with self.assertRaises(module.IntegrityError):
+                    module.verify_release_signature(changed_manifest, changed_signature, self.keyring.read_bytes())
+        self.assertEqual(list(self.update_dir.iterdir()), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

docker-compose.yml

@@ -6,39 +6,31 @@ services:
         container_name: electric-cherry
         restart: unless-stopped
         ports:
-      - "3000:3000"
+            - '3000:3000'
-    # Lets RCON_HOST=host.docker.internal reach game servers running on the
-    # same Komodo host. Docker Engine 20.10+ resolves this alias when the
-    # extra_hosts entry is present.
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
         volumes:
-      # in prod (Komodo host), the game server tree is at /root/zombies
+            # Preserve the existing controller database by default. Set EC_DATA_DIR
-      - /root/zombies:/root/zombies:rw
+            # to a dedicated persistent directory for a new controller installation.
-      - /root/zombies/.electric-cherry:/data:rw
+            - ${EC_DATA_DIR:-/root/zombies/.electric-cherry}:/data:rw
         environment:
-      # SvelteKit's Node adapter caps request bodies at 512KB by default,
+            ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000}
-      # which rejects real-world plugin .dll uploads with HTTP 413. Same
+            EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-}
-      # trust boundary as the CSRF disable (see svelte.config.js) — this is
+            EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-}
-      # LAN-only, so "Infinity" (no cap) is fine. NOTE: "0" does NOT mean
+            EC_DISCORD_APPLICATION_ID: ${EC_DISCORD_APPLICATION_ID:-}
-      # unlimited in this adapter — it blocks anything > 0 bytes. SvelteKit's
+            EC_DISCORD_PUBLIC_KEY: ${EC_DISCORD_PUBLIC_KEY:-}
-      # own error message calls this out: "To disable body size limits,
+            EC_DISCORD_BOT_TOKEN: ${EC_DISCORD_BOT_TOKEN:-}
-      # specify Infinity rather than 0." If exposed to the public internet,
+            EC_BANK_MAX_BALANCE: ${EC_BANK_MAX_BALANCE:-250000}
-      # replace with a concrete byte cap (e.g. "128M") instead.
+            EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false}
-      BODY_SIZE_LIMIT: "Infinity"
+            # Enable only behind a trusted proxy that sets the client-address header.
-      SERVERS_ROOT: /root/zombies/t5
+            ADDRESS_HEADER: ${ADDRESS_HEADER:-}
-      T6_SERVERS_ROOT: /root/zombies/t6
+            XFF_DEPTH: ${XFF_DEPTH:-1}
-      REL_DEST: runtime/plutonium/storage/t5/raw/scripts/sp
+            # Legacy host imports stream whole game files; buffering routes enforce their own caps.
-      BETA_ARCHIVE_DIR: /root/zombies/beta-archive
+            BODY_SIZE_LIMIT: 'Infinity'
             DB_PATH: /data/cherry.db
-      DEPLOY_KEEP_DAYS: "7"
+            GAME_FILES_ROOT: /data/game-files
-      DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1}
+            BETA_ARCHIVE_DIR: /data/beta-archive
-      DEPLOYMENT_CREATE_CONCURRENCY: ${DEPLOYMENT_CREATE_CONCURRENCY:-1}
+            # These are logical legacy paths; native instance files live on agents.
-      KOMODO_ADDRESS: ${KOMODO_ADDRESS}
+            SERVERS_ROOT: /data/legacy/t5
-      KOMODO_API_KEY: ${KOMODO_API_KEY}
+            T6_SERVERS_ROOT: /data/legacy/t6
-      KOMODO_API_SECRET: ${KOMODO_API_SECRET}
+            PLUTAINER_IMAGE: ${PLUTAINER_IMAGE:-ghcr.io/ayymoss/plutainer:latest}
-      KOMODO_TARGET_SERVER: ${KOMODO_TARGET_SERVER}
+            # Optional. Native instances otherwise receive individual random secrets.
-      PLUTAINER_T6_GAMEFILES_VOLUME: ${PLUTAINER_T6_GAMEFILES_VOLUME:-t6_game_files}
+            RCON_PASSWORD: ${RCON_PASSWORD:-}
-      RCON_PASSWORD: ${RCON_PASSWORD}
-      RCON_HOST: ${RCON_HOST:-host.docker.internal}
-      RCON_TIMEOUT_MS: ${RCON_TIMEOUT_MS:-3000}

docs/README.md

@@ -0,0 +1,60 @@
+# Documentation
+
+Start with [Getting started](guides/getting-started.md) to set up a controller
+and launch your first server.
+
+## Guides
+
+### Setup and administration
+
+- [Getting started](guides/getting-started.md) — requirements, installation, and first launch.
+- [Native hosts](guides/native-hosts.md) — enrollment, game files, capacity, agent updates, and recovery.
+- [Users and access](guides/authentication.md) — roles, sessions, telemetry credentials, and password recovery.
+- [Fleet settings](guides/settings.md) — concurrency, retention, and cleanup.
+- [Discord companion](guides/discord.md) — live server boards, private browsing, leaderboards, and availability alerts.
+
+### Fleet operations
+
+- [Provisioning](guides/provisioning.md) — lifecycle actions, safety checks, queues, and diagnostics.
+- [Collections](guides/collections.md) — reusable map lineups and promoted launch files.
+- [Lobby autoscaling](guides/autoscaling.md) — policies, ownership, pause controls, and the pilot procedure.
+- [Rebalancing](guides/native-hosts.md#rebalance-instances) — preview and safely spread empty lobbies across active hosts.
+- [Maintenance](guides/maintenance.md) — health recovery, drift repair, updates, and scheduled restarts.
+
+### Releases and analytics
+
+- [Beta test plans](guides/beta-test-plans.md) — game reporters, test requirements, coverage, and outcomes.
+- [Plugin releases](guides/plugin-releases.md) — GitHub release discovery and imports.
+- [Metrics](guides/metrics.md) — utilization, capacity pressure, and hosting experiments.
+- [Player bank](guides/bank.md) — shared balances, per-server credentials, transfers and recovery.
+- [AFK reserve](guides/afk.md) — persistent break time, gameplay rewards, purchases and session recovery.
+- [Player moderation](guides/moderation.md) — restrictions, live actions, cases, evidence and game integration.
+- [Leaderboard and matches](guides/leaderboard.md) — player statistics, match history, and the read API.
+
+## Development
+
+- [Development guide](development/README.md) — local setup, architecture, checks, and database diagnostics.
+- [UI patterns](development/ui-patterns.md) — shared components, forms, navigation, and refresh behavior.
+
+## Plans
+
+[Player achievements and visual refresh](plans/player-achievements-and-visual-refresh-plan.md)
+proposes calendar leaderboards, personal-best highlights, shareable result cards,
+and consistent map and status presentation.
+
+[Lobby autoscaling](plans/lobby-autoscaling-plan.md) records the implemented design
+and the production pilot measurements and rollout still to complete.
+Current operating instructions live in the guides above.
+
+## Research
+
+- [Reducing idle CPU use](research/t5-t6-idle-cpu-research.md) — runtime tuning and lobby availability experiments.
+- [Leaderboard query benchmark](research/leaderboard-query-benchmark.md) — historical SQL measurements and methodology.
+
+Research records its baseline and measurement limits; consult the guides for
+current behavior.
+
+Keep operating instructions in `guides/`, contributor guidance in `development/`,
+unfinished delivery or rollout work in `plans/`, and investigations in `research/`.
+When a plan is complete, fold its lasting guidance into the relevant guide and
+remove the superseded document. Git history preserves earlier reviews and plans.

docs/beta-test-plans.md

@@ -1,209 +0,0 @@
-# Beta test plans
-
-Create a plan in Betas, choose its servers, and deploy. The server's game script
-reports gameplay to Electric Cherry. The same reports update Fleet and any active
-beta on that server.
-
-## Game reporter protocol
-
-Send JSON to `POST /api/telemetry` with `Content-Type: application/json`.
-There are four events. Send them whether or not a beta is active.
-
-| Event | Fields |
-| --- | --- |
-| `game_start` | `server`, `map`, `players` with each player's `name` and `guid` |
-| `round_change` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
-| `heartbeat` | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats |
-| `game_end` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
-
-`server` is the exact registered Electric Cherry server name. `map` is the
-current map. `players` is the full player array, including an empty array when
-there are no players. Names and GUIDs are strings; keep GUIDs as strings to
-preserve their precision. Round and stat values are nonnegative JSON integers.
-Each GUID must appear only once within a report's player array.
-
-### Game start
-
-Send when the game starts. Round and stats are not required.
-
-```json
-{
-  "event": "game_start",
-  "server": "t6_zombie_town_1",
-  "map": "zm_transit",
-  "players": [
-    { "name": "Josh", "guid": "12345678901234567" },
-    { "name": "Alex", "guid": "23456789012345678" }
-  ]
-}
-```
-
-### Round change
-
-Send the new round and each player's current stats.
-
-```json
-{
-  "event": "round_change",
-  "server": "t6_zombie_town_1",
-  "map": "zm_transit",
-  "round": 25,
-  "players": [
-    { "name": "Josh", "guid": "12345678901234567", "kills": 420, "headshots": 160, "downs": 1, "revives": 3, "score": 32000 },
-    { "name": "Alex", "guid": "23456789012345678", "kills": 380, "headshots": 140, "downs": 3, "revives": 1, "score": 28000 }
-  ]
-}
-```
-
-### Game end
-
-Send the final round and each player's final stats when the game ends.
-
-```json
-{
-  "event": "game_end",
-  "server": "t6_zombie_town_1",
-  "map": "zm_transit",
-  "round": 26,
-  "players": [
-    { "name": "Josh", "guid": "12345678901234567", "kills": 445, "headshots": 170, "downs": 2, "revives": 3, "score": 33500 },
-    { "name": "Alex", "guid": "23456789012345678", "kills": 401, "headshots": 146, "downs": 4, "revives": 1, "score": 29750 }
-  ]
-}
-```
-
-Success returns HTTP 200 with `{ "ok": true, "telemetry": { ... } }`.
-The returned telemetry contains the event and an Electric Cherry `receivedAt`
-timestamp. Invalid JSON or fields return 400, an unknown/destroyed server 404,
-and a non-JSON content type 415.
-
-The script sends each event once, in gameplay order, directly to Electric Cherry.
-There is no discovery request, beta token, boot/session ID, sequence number,
-custom feature event, external helper, or separate beta acknowledgement.
-Basic payload checks happen in the receiver.
-
-Electric Cherry saves the latest complete report, including player stats, and
-streams it to connected clients. A new start clears the previous round display;
-an end closes the match. Receipt order determines the latest report. This small
-protocol does not attempt to identify delayed events from earlier games or
-provide exactly-once delivery across games.
-
-Reports also feed the persistent [Leaderboard](leaderboard.md), independently
-of beta activity. That page documents cumulative stat accounting, round credit,
-and the limits of identifying matches from this protocol.
-
-The game scripts are uploaded artifacts, not source maintained in this
-repository. Their reporter should emit the four payloads above.
-
-### Telemetry diagnostics
-
-Open a server's **Diagnostics → Telemetry API requests** to inspect incoming
-`POST /api/telemetry` calls whose top-level `server` exactly matches its registered
-name. Each entry includes the Electric Cherry receipt time, HTTP result, error
-reason, event, map, round, player count, content type, and submitted JSON. Rejected
-reports are included, even when the content type is wrong but the body is readable
-JSON. Processing failures return 500 and record the internal error when diagnostic
-storage is available; failed database transactions do not update the scoreboard.
-
-The panel refreshes every five seconds while visible and has a manual refresh
-button. History is stored in SQLite starting with this version, with up to 200
-requests per server from the last seven days and a fleet-wide cap of 10,000
-requests. Old entries are pruned when new requests arrive, and expired entries
-are excluded from reads. Payload previews are limited to 16,384 characters and
-marked when truncated. Request headers other than content type are not stored.
-`GET /api/servers/{name}/telemetry-requests` returns the same recent history,
-newest first, with caching disabled.
-
-- **No entries:** no attributable requests are retained for this exact name.
-  Check the reporter's destination and server field. Calls that never reach
-  Electric Cherry, malformed JSON, and bodies without a usable server name
-  cannot appear in this server's log. Earlier requests are not backfilled.
-- **400 / 415:** inspect the error and payload. One invalid player's stats or a
-  duplicate GUID rejects the entire report. These requests do not update the
-  scoreboard.
-- **404:** the reported server was not registered or had been destroyed at the
-  time of the request.
-- **200:** Electric Cherry processed the report. An empty `players` array
-  produces an empty scoreboard, including when sent with `game_end`. If a
-  populated accepted report is missing from the scoreboard, investigate the
-  live update/browser path.
-- **500:** Electric Cherry failed to process the request. Inspect the recorded
-  error and the application log.
-
-RCON's online status and player count are separate from gameplay telemetry, so
-an occupied server can still have no scoreboard report.
-
-## Requirements
-
-Requirements on a qualifying server are ANDed:
-
-| Requirement | Evidence |
-| --- | --- |
-| Reach round X | A round change, heartbeat, or game end reports round >= X with the required player count |
-| Complete qualifying matches | Game end at round 2 or later, optionally qualified by a higher round, player count, and duration |
-| Populated minutes | Time between consecutive game reports with enough players at both ends |
-| Player-hours | Populated minutes multiplied by the lower reported player count, divided by 60 |
-| Reach player count | Player array length in a game report |
-| Sustained population | Consecutive report intervals above the player threshold within one game |
-| Continuous healthy minutes | Consecutive fresh container/RCON observations |
-| Maximum health outages | Transitions from observed healthy to unhealthy |
-| Successful recovery cycles | Repeat the other requirements after each additional empty-server restart |
-| Tester checklist | Tester name and observation entered on the beta detail page |
-
-All gameplay requirements use the script reports. Match counts do not read game
-logs. The reported round and required player count must occur together in the
-same game; observations from different games cannot combine to qualify a match.
-The final game-end report can supply that qualifying round. Consecutive duplicate
-game ends do not add another completion.
-
-Match duration requires a game start and uses receipt times. Population time
-advances only when another report arrives in the same game; it uses the lower
-player count at the interval's endpoints. There is no heartbeat requirement or
-short interval limit between rounds. These are estimates between reported
-snapshots, not continuous measurements of player presence.
-
-Health checks and safe restart gates use the existing container/RCON monitoring.
-Health intervals spanning more than 60 seconds earn no healthy time.
-
-## Coverage, waves and outcomes
-
-Coverage can require all selected servers, a quorum, one per selected map,
-one per selected game, or one per selected game/map combination. Coverage
-metadata is pinned when the beta opens.
-
-Set wave size to 1 for a canary, a larger number for batches, or 0 for all
-selected servers. Later waves receive files after the previous waves pass.
-Health-outage budget violations on any tested server block promotion.
-
-The test window starts when the beta opens, including time waiting for empty
-servers. Missing evidence at the deadline produces **inconclusive**.
-Health monitoring continues after a pass until promotion.
-
-For failures, select hold as failed, pause for investigation, or automatic
-rollback. Rollback restores only files touched by the beta. Restarts can wait
-for empty servers or remain manual. Promotion is always manual.
-
-Pause holds evaluation and wave advancement. Resume retains completed progress and the
-original deadline, but starts new timing intervals. A fresh attempt resets
-progress and restarts deployed servers when empty. Recovery cycles repeat
-gameplay milestones and checklist sign-offs while carrying health outages forward.
-
-Plans, progress, and evidence persist in SQLite. The detail page shows the most
-recent 200 evidence entries. On upgrading from the old reporter protocol,
-active progress resets. Unsupported checks in existing plans and saved presets
-become manual review items; they cannot silently pass.
-
-## API and storage
-
-`POST /api/betas` accepts `testPlan`.
-`GET /api/betas/:id` includes the plan, outcome, per-server progress, and
-recent evidence under `test`.
-
-`POST /api/betas/:id/test` accepts `pause`, `resume`, `retry`, or
-`sign-off` (with `server`, `requirementId`, and `note`).
-`GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are
-copied into a beta; later edits do not change an active plan.
-
-Internal progress records support attempts and recovery cycles. The reporter
-does not know about or supply these records. Legacy betas without a test plan
-also use `game_end` at round 2 or later for their match-completion gate.

docs/deployment-blueprints.md

@@ -1,240 +0,0 @@
-# Deployment collections
-
-Open **Operations → Collections** to manage reusable server lineups. A
-collection describes a whole lineup: for example, two Kino lobbies, one Moon lobby,
-and two Town lobbies. Counts are desired fleet totals: existing and queued
-servers count toward each target, and only missing servers are created.
-Collections save game/map selections and lobby counts. Every new server uses the
-latest promoted scripts, plugins, baseline, and matching configuration overlays
-when provisioning begins.
-
-For example, a collection targeting four Kino and two Der Riese lobbies, with
-two Kino already registered, creates two Kino and two Der Riese lobbies.
-The preview shows **Target / Existing or queued / To create** for each map.
-Matching is by game and exact map variant; T6 Town and Farm are distinct even
-though they share an engine map. Provisioning identities are counted once when
-they appear in both the server registry and active creation queue.
-
-Registered servers count even if stopped or unhealthy; conforming does not
-restart or reconfigure them. Failed creation jobs without a registered server,
-destroyed records, and servers awaiting removal do not count. The default
-**Fill missing servers** mode keeps excess servers and maps outside the collection.
-It fills deficits without changing existing servers' files.
-
-## Build and launch a collection
-
-1. Choose **Create collection** on **Operations → Collections** and name the lineup.
-2. Add maps from Black Ops I or II and set the desired total lobbies for each.
-3. Choose **Save & preview** (or **Preview fleet changes** on a saved collection).
-   Choose **Fill missing servers** or **Match collection exactly**, then review
-   the server IDs and game ports.
-4. Lobbies with a saved key show **Saved key ready** and reuse it automatically.
-   Enter a Plutonium key only for lobbies missing one. Update saved keys in **Keys**.
-5. Choose **Queue N servers** (or confirm removals and choose **Apply** in exact
-   mode) and follow the collection progress page.
-   If every target is already met, choose **Done**; no operation is created.
-
-### Match collection exactly
-
-Exact mode applies to the entire registered and queued fleet across both games.
-The preview lists servers to keep, create, and remove. It keeps deployed lobbies
-before unstarted creations, then the lowest lobby numbers within each game/map
-variant. It removes excess lobbies and maps outside the collection. Retained
-servers keep their containers, files, ports, and configuration without a restart.
-The option is chosen per launch; opening another preview defaults to fill mode.
-
-Review the removal list and check its confirmation before applying. Removed
-servers lose their containers, files, and registrations; saved keys remain.
-Unstarted creations are cancelled without booting a container. Running servers
-use the normal removal queue and require a fresh zero-player RCON response.
-Occupied or unreachable servers wait. Already pending removals are reused.
-
-Creation reservations and removals are queued in one database transaction. A
-missing key or changed fleet rejects the entire request. Any fleet identity,
-map, port, or pending-removal change since an exact preview requires a new review.
-Servers added after submission are not included. Applying the same preview again
-returns the same collection without repeating operations.
-
-Ports belonging to reviewed removals can be reserved for new lobbies even when
-the current port range is full. A new lobby waits until its port's removal has
-completed; a failed, interrupted, or cancelled removal keeps that launch blocked,
-with the reason shown in its progress. Other launches with available capacity can
-proceed. Reservations and dependencies survive app restarts. The progress page
-tracks creations and removals, including removal-only plans, and lists the servers
-that were retained. The existing operation detail page offers removal cancellation
-and cleanup recovery where applicable.
-
-Collections support up to 40 distinct game/map selections, 1–20 lobbies per map,
-and 100 servers total. The host's configured port range and existing fleet may
-allow fewer. A collection can be saved even when there is insufficient capacity
-to launch it. Saving a collection does not capture or pin managed file versions.
-
-Use **Edit** on a collection card to change its name, description, maps, or lobby
-counts. Already reviewed launches keep their original lineup, including after
-collection edits or removal; their managed files still follow the latest promotions.
-Previously saved collections automatically use this policy without being resaved.
-Collections saved by the earlier single-server release load as one-server lineups
-and can be expanded directly. Any file snapshots in older collections are ignored
-when preparing new launches.
-
-A collection preview does not reserve capacity. At submission, all lobby IDs,
-ports, and keys are validated again. The entire collection is then queued in one
-database transaction. If any server lacks a valid key or capacity has changed,
-no jobs, reservations, or key changes from that submission are committed. Refreshing
-the preview retains typed keys only for unchanged server IDs still missing a saved
-key, and resets removal confirmation.
-
-Submitting the same preview again returns the same collection, including after
-a lost response or an application restart. Reusing the collection with a new
-preview recomputes the shortfall, including servers queued by earlier launches.
-It does not add another full copy of the lineup. If the shortfall or assigned
-capacity changes before submission, the request is rejected without changes and
-requires a fresh preview. Previously saved additive previews are checked against
-the current shortfall too.
-
-All collections and individual launches share one startup queue, ordered by request
-time. By default, one server starts at a time. Its startup slot remains occupied
-until Komodo is healthy, RCON answers, and the reported map matches the selection.
-Only then can the next queued server start. Set `DEPLOYMENT_CREATE_CONCURRENCY` to
-a positive integer to allow more simultaneous startups. Each worker pass admits
-at most the available slots, even if an attempt fails immediately.
-
-Names, ports, and queue order survive application restarts. A server still booting
-or waiting for health keeps its slot until it passes verification or is removed.
-A lost startup response also keeps the slot because the container may already be
-running. A confirmed creation failure is recorded for review, allowing later
-queued servers to proceed on subsequent passes without undoing successful launches.
-
-The collection page separates ready, launching, queued, and failed servers. It
-highlights the current startup, shows each waiting server's queue position, and
-links to earlier launches when another collection occupies the available capacity.
-Every server links to its operation timeline and recovery options, and each child
-operation links back to its collection.
-
-## Create and review a launch
-
-1. Choose **Single server** on Provisioning (or the deploy action on the fleet).
-2. Choose the current promoted setup and select a game/map, or select a collection.
-3. Choose **Preview launch**. Review the image reference, configuration sources,
-   managed files and versions, selected map, lobby, port, and prerequisite checks.
-4. Enter the Plutonium key for the displayed server ID, or reuse its saved key.
-5. Choose **Launch server** and follow the operation to verified health.
-
-A preview lasts 30 minutes. It neither reserves capacity nor saves a key. If
-another launch takes the lobby or port, submission stops before saving the key;
-refresh the preview and review the newly assigned ID. Refreshing a preview clears
-the typed key so it cannot accidentally be saved against a different lobby.
-
-Komodo connection settings, target host, RCON configuration, config filename,
-lobby availability, and port availability are checked locally. These checks do
-not probe the host or verify game/DLC files, images, Docker volumes, firewall
-rules, or credentials. Container and RCON verification happen after provisioning.
-T6 variants retain their exact catalog slug and launch ruleset; RCON health checks
-verify the engine map, not the variant's start location.
-
-## Save and reuse a collection
-
-Expand **Make this a reusable collection** in a launch preview and give it a name
-and optional description. You can save the map even when host configuration or
-available capacity blocks launching. Names are unique without regard to case.
-
-Each map in a collection retains the game, exact map variant, and desired lobby
-count. Image references, managed files, and composed configuration are resolved
-from the current setup for each new launch. Credentials, lobby IDs, and ports are
-not copied from the source server.
-
-Use **Conform fleet** on a collection card to review missing servers. Remove a
-collection from the library when it is no longer needed; its existing servers,
-reviewed previews, and deployment history remain intact.
-
-On a creation detail page, **Launch again** selects the original game and map
-with the next available lobby and port, using the latest promoted setup. The
-preview also lets you save that map as a collection with a target of one lobby.
-The recorded files remain visible as deployment history. Older deployments
-without a recorded recipe retain their history but cannot offer this action.
-
-## Promotions and deployment history
-
-A preview shows the currently promoted managed files, configuration, and image
-reference for the selected map. It does not pin those versions. New launches
-resolve the promoted setup again when submitted and when provisioning begins, so
-promotions made after a preview or while a job is queued apply to that server.
-This applies to collection collections, single-server launches, and **Launch again**.
-
-The files selected at provisioning, their hashes and version metadata, composed
-configuration, selected map, and image reference are recorded in SQLite for
-deployment history. These records do not override promotions on future launches.
-Existing API clients that omit a preview and jobs queued before this update
-also use the promoted setup at provisioning time.
-
-A submitted preview is bound to one deployment: retrying the same submission
-returns the original operation, including after an application restart or a lost
-HTTP response. The reviewed game/map, lobby ID, port, and key checks are preserved.
-Already running servers continue to receive updates through campaigns, betas,
-and drift reconciliation using the server's game/map scopes.
-Container tags can resolve to newer images; use a digest for a fixed image.
-Without a promoted baseline, Plutainer seeds its own defaults on first boot,
-which are not captured. Host, network, game-file volumes, destination directories,
-and RCON/Plutonium credentials use the environment and key store at launch time.
-The current main config filename and launch settings apply to new deployments.
-
-Recipes do not include the separately stored Plutonium or integration credentials.
-Managed files can themselves contain sensitive configuration, so protect database
-backups as you would the existing artifact library and saved-key database. Public
-recipe responses contain file metadata and hashes, not file contents.
-
-## Progress and interrupted operations
-
-Creation verifies container health, then RCON, then the selected engine map.
-The detail page explains the current condition, displays elapsed time, and keeps
-its activity history. A completed container start is not reported as a completed
-server launch until the health checks pass.
-
-For failed or interrupted creations, **Resume health verification** appears only
-when the original server is still registered on its assigned port and no later
-lifecycle operation has targeted that ID. It resumes observation only; it never
-recreates or restarts a container. Missing servers and later destruction attempts
-require inspection rather than replaying provisioning automatically.
-After an application restart, the worker automatically resumes observation of
-eligible interrupted startups so their occupied slots are retained. It never
-repeats container creation or deployment as part of this recovery.
-
-## API
-
-Collections retain the existing `/api/blueprints` endpoints and `blueprintId` /
-`blueprintName` fields for compatibility. Saved data needs no migration. Old
-`/provisioning?view=blueprints` links redirect to `/collections`.
-
-- `GET /api/blueprints`: list lineup summaries with game/map selections and counts.
-- `POST /api/blueprints`: save `{ name, description?, previewId }` or
-  `{ name, description?, deploymentId }` for a single-map target of one lobby, or
-  `{ name, description?, entries: [{ game, mapSlug, count }] }` for a collection.
-- `PUT /api/blueprints/:id`: edit `{ name, description?, entries }`. All entries
-  use the latest promoted setup when launched.
-- `DELETE /api/blueprints/:id`: remove a library entry.
-- `POST /api/deployment-collections/preview`: preview `{ blueprintId, mode? }`,
-  where mode is `fill` (default) or `exact`. Returns `mode`, `retainedServers`,
-  `removals` (name/reason), creation `servers`, and configured `launchConcurrency`.
-- `POST /api/deployment-collections`: atomically queue `{ previewId, keys? }`,
-  where `keys` maps exact preview server IDs to optional replacement keys.
-  A satisfied preview returns `{ ok: true, collectionId: null }` with HTTP 200,
-  without creating a collection, saving keys, or kicking the worker.
-- `GET /api/deployment-collections`: list collection progress summaries with
-  separate `queued`, `provisioning`, and `verifying` counts.
-- `GET /api/deployment-collections/:id`: collection mode, retained server IDs,
-  per-operation progress with `action: create | destroy`,
-  each waiting server's global `queuePosition`, and the shared queue's concurrency,
-  active startups, and queued count.
-- `POST /api/deployments/preview`: preview `{ game, mapSlug }` or `{ deploymentId }`
-  for a new single server using the latest promoted setup. All collections, including one-server
-  targets, must use the collection endpoint so existing servers count correctly.
-  Returns an ID, expiry, recipe summary, capacity, key
-  availability (never its value), and prerequisite checks.
-- `POST /api/servers`: existing request plus optional `previewId`; the game and
-  map must match that preview. `expectedName` remains supported.
-- `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
-  `canResumeVerification` for lifecycle operations.
-- `POST /api/deployments/:id/resume`: resume eligible creation health checks.
-
-Scheduled game nights and beta-gated expansion remain future layers. Collection
-launches begin processing immediately after all requests are queued.

docs/development/README.md

@@ -0,0 +1,247 @@
+# Development
+
+The controller uses SvelteKit, TypeScript and SQLite (`better-sqlite3`), with the
+Node adapter and server-sent events. The host agent uses Python 3's standard
+library and runs as a systemd service; release verification calls `gpgv`.
+Hosts do not need Node.js.
+
+Agents connect outbound to the controller over HTTPS and manage Docker/Plutainer
+and local RCON on each host. The controller owns fleet metadata and queues; it
+does not require a Docker socket or direct access to host filesystems.
+
+## Run locally
+
+Use Node.js 22.13+ (the container uses Node.js 22), npm and Python 3. Install
+GnuPG's `gpg`, `gpg-agent` and `gpgv` to run real release-signing integration tests; local test
+runs skip those tests when either executable is absent.
+
+```sh
+npm ci
+export EC_SETUP_TOKEN="$(openssl rand -hex 32)"
+export BETA_ARCHIVE_DIR=./data/beta-archive
+EC_AUTH_ALLOW_HTTP=true npm run dev
+```
+
+Use the exported token at `/setup` to create the first administrator, then remove
+it from the environment. Local metadata defaults to `./data/cherry.db`; use a
+separate database when testing enrollment. HTTP login is for local development;
+production agent transport requires HTTPS. See [authentication](../guides/authentication.md)
+for access and recovery details.
+
+## Synthetic UI preview
+
+Create a disposable fixture without connecting game hosts:
+
+```sh
+npx svelte-kit sync
+fixture_parent=$(mktemp -d "${TMPDIR:-/tmp}/ec-ui.XXXXXX")
+npm run seed:fixture -- --db "$fixture_parent/preview/cherry.db" --watch
+```
+
+The sync step generates the SvelteKit TypeScript configuration that the seeder
+needs in a fresh checkout.
+
+The seeder prints the generated paths. Leave that terminal open to refresh
+synthetic observations every 15 seconds. In a second terminal, load the printed
+environment file and start the controller on localhost:
+
+```sh
+source /absolute/path/printed/above/preview.env
+npm run dev -- --host 127.0.0.1 --port 3000
+```
+
+Open `http://127.0.0.1:3000/login`. The private `fixture.json` next to the database
+contains the `demo` account's generated password. It is not printed in command
+output. To run a fixed snapshot instead of a live feeder, use a fresh directory
+and replace `--watch` with `--now 2026-10-07T12:00:00Z`. Domain data is deterministic
+for that timestamp; account credentials are always generated separately. Old
+snapshots naturally show stale observations when viewed against today's clock.
+
+The fixture contains 3 hosts, 12 canonical Black Ops I/II servers, and 500 match
+records: 494 historical and 6 live. It includes solo through four-player squads,
+unknown starting counts, enough players and matches to exercise pagination,
+rosters, seven days of occupancy samples with unknown player limits, and retained Activity
+from campaigns, betas, host operations, maintenance, recovery, and autoscaling.
+The fleet has 6 live matches, 2 empty lobbies, a preparing server, and 3 problems:
+a stopped server, a server not responding, and a server on the offline host.
+Script and config downloads contain harmless synthetic text.
+
+`--db` must be an absolute path in a **new directory** whose parent exists. The
+seeder refuses existing directories, databases, sidecars and symlinks, and never
+uses an inherited `DB_PATH`. It runs the application's database migrations with
+isolated paths, without starting the web server or workers. A failed seed removes
+only its newly reserved directory. Completed fixtures remain after Ctrl+C.
+
+The preview controller runs its usual local workers. Automatic maintenance and
+autoscaling are paused; all seeded operations are terminal. Hostnames, runtime
+images, library IDs and credentials are synthetic, so there are no runnable game
+files or authenticated agents. Use this environment only for local UI review.
+The feeder checks the fixture marker and restores only its known observations,
+including the deliberately offline host. Stop both terminals when finished and
+remove the generated fixture directory.
+
+## Checks and builds
+
+```sh
+npm run lint
+npm run check
+npm test
+npm run test:agent
+npm run build
+```
+
+The [Gitea Actions workflow](../../.gitea/workflows/ci.yml) runs these checks on
+every push and pull request after `npm ci`. It uses a Node.js 22 Debian container
+with Python 3, `gpg`, `gpg-agent`, `gpgv` and native build tools for `better-sqlite3`. Signing
+tests create disposable keys in temporary directories; CI never receives release
+private keys. The runner must support Docker jobs with the `ubuntu-latest` label.
+`npm run check` runs before the tests
+to generate the SvelteKit configuration needed in a fresh checkout. Application
+tests use two workers in CI to bound resource use for database-heavy fixtures.
+The build step runs the same `npm run build` command used by the Dockerfile,
+including agent packaging and the report and launch bundles.
+
+Run `npm run format` to apply Prettier formatting before committing. It uses
+four-space indentation and a 100-column target for supported source, config and
+documentation files, including Svelte components. Python, shell and game scripts
+are outside Prettier's supported languages. Generated agent downloads and the npm
+lockfile keep their generator's formatting.
+
+`npm run lint` checks formatting without changing files, then runs the recommended
+JavaScript, TypeScript and Svelte ESLint rules. Generated build output and local
+runtime data are excluded. Type checking remains part of `npm run check`.
+The ESLint config documents exceptions for existing Svelte navigation, list and
+reactivity patterns. Individual suppressions explain intentional validation and
+test fixtures. Node.js 22.13 is the minimum supported 22.x version for the lint
+toolchain.
+
+The `package.json` overrides keep SvelteKit 2 on patched `devalue` 5.x (5.9.4 or
+newer) and `cookie` 0.7.x releases until its dependency ranges include those fixes.
+These dependencies ship in the server bundle even though SvelteKit is a
+development dependency.
+Dependency security checks must include development dependencies: use `npm audit`
+without `--omit=dev`.
+
+`npm run dev` and `npm run build` regenerate the downloadable agent, installer,
+standalone updater and checksum manifest. Packaging preserves an existing
+`static/agent/manifest.json.sig` only when its manifest bytes remain identical;
+changes to the agent, updater or installer remove the stale signature. Builds
+never sign, and hosts reject unsigned releases while retaining installed code.
+Finalize and review release code, run `npm run agent:package`, and explicitly
+sign it on the offline publisher machine:
+
+```sh
+python3 agent/sign_release.py --gnupghome /secure/electric-cherry-signing --fingerprint FULL_SIGNING_KEY_FINGERPRINT
+```
+
+Commit or distribute the signed public artifacts before building the controller
+image from the same source revision. Keep the signing home outside the repository, Docker context, controller
+and CI. Follow [Publishing signed releases](../guides/native-hosts.md#publishing-signed-releases)
+for key creation, host provisioning and rotation.
+
+Builds also bundle report workers
+under `build/reports` and launch preparation under `build/launch`. Development
+and test commands compile those workers automatically. Restart `npm run dev`
+after changing agent or worker code, or run `npm run agent:package` to refresh
+only the downloadable agent assets while the development server is running.
+
+To run the production build locally, run these commands from the repository root
+with the controller environment configured as above:
+
+```sh
+npm run build
+NODE_ENV=production node build
+```
+
+`NODE_ENV=production` makes the report and launch workers load from `build/reports`
+and `build/launch`. Without it, they look under `.svelte-kit/`, where the production
+build does not create worker bundles. The Docker image sets this variable already.
+
+Automated tests simulate Docker and game hosts. Real Debian installation and
+T5/T6 server validation use the [host acceptance checklist](../guides/native-hosts.md#development-and-validation).
+
+## Code layout
+
+Paths below are relative to the repository root.
+
+| Path                                    | Responsibility                                              |
+| --------------------------------------- | ----------------------------------------------------------- |
+| `src/routes`                            | Pages and API endpoints                                     |
+| `src/lib/ui`                            | Shared components, stores and [UI patterns](ui-patterns.md) |
+| `src/lib/server/hosts`                  | Enrollment, readiness and command transport                 |
+| `src/lib/server/backends`               | Instance identity and remote file access                    |
+| `src/lib/server/provisioning/native.ts` | Lifecycle and campaign queues connected to agents           |
+| `src/lib/server/game-files`             | Immutable uploads and streaming distribution                |
+| `src/lib/server/db`                     | SQLite schema, migrations and shared artifact storage       |
+| `src/lib/server/reports`                | Metrics, leaderboard and capacity workers                   |
+| `agent`                                 | Python service, installer, packaging and tests              |
+
+## Performance diagnostics
+
+Set `EC_PERF_LOG=true` to log slow requests, report computation/cache timings and
+event-loop delay. Responses expose `Server-Timing`; browser navigation records an
+`ec:navigation` performance measure. Report workers use separate read-only SQLite
+connections and bounded caches. Metrics snapshots can be reused for up to 30
+seconds when their inputs are unchanged.
+
+Benchmark the leaderboard against an isolated synthetic database:
+
+```sh
+npm run benchmark:leaderboard
+npm run benchmark:leaderboard -- --scenario all --baseline-ref <commit>
+```
+
+The comparison checks response parity and warm query timings.
+
+Set `EC_SLOW_SQL_MS=20` to log runtime statements taking at least 20 ms. Logging
+omits bound values and string literals and is disabled by default. Capture timings
+under the same fleet load before and after database changes. Inspect database
+size, large tables/indexes, WAL size and free pages inside the controller with:
+
+```sh
+npm run diagnostics:database -- /data/cherry.db
+```
+
+This opens an existing database read-only without running migrations. Substitute
+`./data/cherry.db` for a default local installation.
+
+## Storage and upgrades
+
+Game-file uploads live in dedicated filesystem storage. Script, plugin and
+configuration artifact bodies are stored once by SHA-256 in SQLite; versions,
+staging plans, rollback snapshots and launch commands reference those shared
+bytes. Startup transactionally backfills older rows. Current agents fetch
+assigned artifacts through an authenticated endpoint and verify their hashes;
+older agents receive inline content until they update.
+
+Back up the persistent data directory before upgrading. Startup logs show
+migration phases, elapsed time and resident memory. Completed migrations are
+retained across restarts; interrupted transactions roll back and retry on startup.
+The one-time vacuum needs disk space for a temporary database copy. **Do not
+remove `cherry.db-wal` to recover from an interrupted upgrade.**
+
+Database upgrades are ordered in `src/lib/server/db/migrations.ts` and tracked by
+SQLite's `PRAGMA user_version`. Startup applies only pending versions, including
+host, authentication and game-file schemas, before serving requests or starting
+workers. Databases from older releases begin at version 0; the compatibility
+steps honor their existing migration markers so completed backfills are preserved.
+An application cannot open a database with a newer version than it supports.
+
+For a schema or data upgrade, append a consecutive version to that list. Do not
+edit an applied migration or rely on changing `schema.sql` to upgrade databases.
+Each step and its version commit in one transaction. Table rebuilds can select
+`foreign-keys-off`, which disables enforcement before the transaction and checks
+references before committing. The resumable incremental-vacuum step uses
+`autocommit` because SQLite forbids `VACUUM` inside a transaction; its version is
+recorded only after the rebuild and host preparation boundary recovery succeed.
+Connection tuning and query optimization still run on every open.
+
+The controller uses a 128 MiB page cache; each reader worker uses 32 MiB.
+Main-database migrations use 16 MiB with memory mapping disabled. Temporary
+storage stays disk-backed. Containers use `/data/.sqlite-tmp` for scratch files;
+override `SQLITE_TMPDIR` only with another writable, disk-backed directory.
+
+For a local data disk, `EC_SQLITE_MMAP_MB=256` enables memory-mapped reads; leave it
+at zero for network mounts. Query statistics are optimized at startup and hourly.
+WAL reuse is capped at 64 MiB after checkpoints, though active transactions can
+temporarily grow it larger.

docs/development/ui-patterns.md

@@ -0,0 +1,151 @@
+# UI patterns
+
+Use the shared components in `src/lib/ui/components` to keep pages consistent.
+Primitives live in its `primitives` directory.
+
+## Page structure and controls
+
+- Use `SectionHeader` for page titles and primary actions. The page owns spacing
+  between sections.
+- Use `Field` with `Input`, `Select` or `Textarea` for labels, help and errors.
+  Give controls distinct `ariaLabel` values when several share one field. Use
+  `hiddenLabel` on `Field` for compact filters with a visually hidden label.
+  `Input` and `Select` forward native validation, accessibility and event props;
+  use `bind:element` when a workflow needs to focus the native control. `Select`
+  accepts either `options` or child `option` / `optgroup` elements and preserves
+  the bound value's type. Keep layout on the field or its parent; controls own
+  their appearance.
+- `npm run check:styles` rejects raw selects and text-like inputs outside their
+  primitives, including search, password, numeric and date/time fields. Native
+  checkboxes, radios, file pickers, hidden inputs, range/color pickers and input
+  buttons are allowed. The check parses Svelte markup, so comments and examples
+  in scripts are ignored.
+- Pair `Tabs` with a labeled panel. Use `ListFooter` for long lists.
+- Use `StatusBadge` for workflow states and `ServerIdentity` for server names.
+  Keep raw IDs available for copying and troubleshooting.
+
+## Dialogs and feedback
+
+Use `Dialog` for modal workflows and `confirmAction` from `src/lib/ui/confirm.ts`
+for ordinary confirmations. Browser unload protection remains native.
+
+Keep loaded data visible when a refresh fails, show the error and provide a
+retry action. Distinguish loading, empty and failed states so operators know
+whether to wait or act.
+
+## Live data
+
+Use `pollVisible` from `src/lib/ui/poll.ts` for scheduled page refreshes. It avoids
+overlapping requests, pauses while the page is hidden and returns a cleanup
+function for use when the component unmounts.
+
+Read live server data through `src/lib/ui/stores/fleet.ts` instead of adding
+another `/api/servers` polling loop. Server detail views use a scoped Fleet store
+so their snapshots do not replace the shared fleet.
+
+## Accessibility and small screens
+
+Preserve visible keyboard focus, associated field labels and separate navigation
+and action targets. Avoid nested buttons or placing a button inside a link.
+Allow long server identities to wrap and tab/filter rows to scroll on narrow
+screens.
+
+### Responsive widths
+
+Use the shared breakpoint scale in `src/lib/ui/styles/tokens.css`:
+
+| Token             | Width   | Use                                        |
+| ----------------- | ------- | ------------------------------------------ |
+| `--breakpoint-xs` | 360 px  | Extra-small screen adjustments             |
+| `--breakpoint-sm` | 640 px  | Compact controls, cards and page summaries |
+| `--breakpoint-md` | 960 px  | Sidebar switches to the navigation drawer  |
+| `--breakpoint-lg` | 1100 px | Dense, multi-column layouts reflow         |
+
+CSS custom properties cannot be used inside media conditions. Write the literal
+value, for example `@media (max-width: 640px)`, and choose the shared width that
+fits the layout. Related page and component changes should use the same width.
+For JavaScript behavior, use the identical query and invert `matches` when you
+need its complement; avoid adding a pixel to the boundary.
+
+`npm run check:styles` reads this scale and rejects other widths in Svelte and
+CSS `@media` queries. The file-and-width allowlist in `scripts/check-ui-styles.mjs`
+records existing layouts awaiting migration. Remove entries as those layouts
+move to the shared scale; do not add exceptions for new components. The app
+shell, Servers page and server cards use only shared widths.
+
+## Navigation
+
+Use the shared navigation metadata for Fleet, Releases, Activity, Players, and
+Admin. Campaigns stays under Releases beside Scripts, Configs, Plugins, Game
+files, and Betas. The Servers page is the Fleet landing page. Desktop navigation
+uses a collapsible sidebar; mobile navigation uses a modal drawer.
+
+Use Cmd/Ctrl K to search pages, servers, hosts, players, and recorded matches.
+Common actions open their existing forms and retain role checks and confirmations.
+Keyboard shortcut help is available from the account menu and with `?`.
+
+Detail pages use `Breadcrumbs`. Its parent link restores the validated `returnTo`
+URL, including filters, sorting, pagination, and anchors. Supply `fallback` when
+the default parent also needs filters. Do not add a duplicate Back link.
+
+### Activity history
+
+Use `ActivityLog` for the fleet timeline and scoped history. The server History
+tab fixes the server filter; Maintenance fixes the maintenance/recovery source.
+The API counts and pages retained records together, with stable ordering. Parent
+operations represent their host receipts and maintenance verification so a single
+user action does not appear multiple times. Routine successful host checks stay
+in the technical Host operation history. Keep detail links contextual so their
+parent breadcrumb restores filters and pagination.
+
+### Type and numbers
+
+Use the shared `--text-*` scale, with 12 px as the minimum. Mobile inputs use
+`--text-input` (16 px) to avoid browser zoom. `npm run check:styles`, also included
+in `npm run check`, rejects raw pixel font sizes. Use Inter with tabular figures
+for counts, scores, durations and charts; reserve mono for code, IDs, paths and
+logs. Static values use text or state colors; the accent marks links and controls.
+
+### Product language
+
+| Concept                    | Visible label                                           |
+| -------------------------- | ------------------------------------------------------- |
+| A managed game instance    | Server; use the friendly map and lobby name             |
+| Creating a server          | Create server                                           |
+| Removing a server          | Remove server                                           |
+| Favourite server           | Pin                                                     |
+| Game does not answer       | Not responding                                          |
+| Games                      | Black Ops I / Black Ops II; BO1 / BO2 in compact labels |
+| Releasing file changes     | Campaigns                                               |
+| Player completion evidence | Present in final report / Not present in final report   |
+
+Keep internal values such as `t5`, `t6`, `destroy`, instance IDs, and filenames in
+API fields and technical details. User-facing actions use the labels above.
+Player presence in the final report does not establish a win or continuous
+survival. A missing timestamp is **Not recorded**, not **Never**.
+
+Use `RelativeTime` for observed history: seconds, minutes or hours for the last
+24 hours, Yesterday for the preceding calendar day, then the shared absolute
+format. It includes the full timestamp in a tooltip and follows the account's
+timezone. Scheduled future actions use `timeFormat.dateTime` directly. Database
+timestamps without an offset are UTC; never parse them as browser-local time.
+Receipt freshness uses the browser clock and shows Paused or a retry reason
+when updates are unavailable. These clocks are not live regions.
+
+Provisioning detail pages show the requester, absolute start time and duration
+in the summary. Keep the finish time, raw IDs and copy controls in collapsed
+Technical details. Show freshness only while the operation is active. Poll active
+operations and finished operations whose recovery capacity is still held.
+Use `OperationTime` for steps, checks and activity: offsets from
+the operation start (`+m:ss` or `+h:mm:ss`), with negative offsets for earlier
+evidence. Missing timestamps stay **Not recorded**.
+
+Use `plural` or `countLabel` from `src/lib/ui/plural.ts` for count-dependent
+language, including verbs: `plural(count, 'needs', 'need')`. The helper uses
+`Intl.PluralRules`; supply irregular plural forms explicitly. Keep separators
+inside string expressions when a Svelte conditional would trim their leading
+space, for example `{offline ? ' · Offline' : ''}`.
+
+Recommend an agent upgrade only when its reported version is known to lack a
+needed capability. Missing calibration samples can simply mean the host is
+still collecting measurements.

docs/guides/afk.md

@@ -0,0 +1,144 @@
+# Persistent AFK time
+
+AFK time is a GUID account shared by T5 and T6, separate from the points bank. The scripts own `.afk`, `.back`, explicit purchase confirmation, actual gameplay contribution detection, protection and controls, the center-screen countdown, and all-player-AFK / match-ending behavior. Electric Cherry owns policy, durable receipts, atomic points purchases, reservations, and reporting. The immutable native instance's existing bank credential authenticates both APIs; the shared telemetry token and browser sessions cannot authorize game requests.
+
+## Policy and accounting
+
+All balances are integer seconds. Defaults:
+
+| Setting                                        | Environment variable                                             | Default     |
+| ---------------------------------------------- | ---------------------------------------------------------------- | ----------- |
+| Free rolling allowance                         | `EC_AFK_FREE_SECONDS`                                            | 1800        |
+| Combined earned/purchased capacity             | `EC_AFK_CAPACITY_SECONDS`                                        | 21600       |
+| Active play interval / reward                  | `EC_AFK_ACTIVE_INTERVAL_SECONDS`, `EC_AFK_ACTIVE_REWARD_SECONDS` | 300 / 60    |
+| Participating round reward                     | `EC_AFK_ROUND_REWARD_SECONDS`                                    | 15          |
+| Kill interval / reward                         | `EC_AFK_KILL_INTERVAL`, `EC_AFK_KILL_REWARD_SECONDS`             | 100 / 30    |
+| Headshot interval / reward                     | `EC_AFK_HEADSHOT_INTERVAL`, `EC_AFK_HEADSHOT_REWARD_SECONDS`     | 25 / 15     |
+| Successful teammate revive                     | `EC_AFK_REVIVE_REWARD_SECONDS`                                   | 30          |
+| Performance reward cap per rolling active hour | `EC_AFK_BONUS_HOURLY_SECONDS`                                    | 480         |
+| Points / seconds per purchase pack             | `EC_AFK_PURCHASE_POINTS`, `EC_AFK_PURCHASE_SECONDS`              | 10000 / 600 |
+
+Free time is spent first. Its available amount is the allowance minus outstanding free reservations and confirmed free consumption charged within the preceding 24 hours. Consumption is charged at the controller's receipt of a durable stop, so free time replenishes 24 hours after that receipt. This deliberately conservative rolling window cannot grant more than the allowance in a rolling day, even with delayed delivery or inaccurate server clocks. Each charge expires independently; unused days never accumulate. Reads, reconnects, map changes, and changing game/server do not reset anything. An unclosed reservation never expires, even after 24 hours. Profile `freeRenewsAt` is the next known charge expiration, not a promise to release unresolved authorization.
+
+Active seconds and partial kill/headshot thresholds persist by GUID across matches and servers. Cumulative snapshots contribute only their increase over the last accepted snapshot for that originating immutable instance, match key, and GUID. Active play rewards use every completed interval. Participating rounds, kill thresholds, and headshot thresholds share one rolling 3600-_active-second_ budget: each performance grant occupies budget until another 3600 verified active seconds elapses. Kill and headshot rewards can stack. Spectating, AFK, idle connections, wall-clock waiting, and raw score increases do not advance this clock. Each legitimate successful teammate revive earns the full configured revive reward, including repeated revives of the same teammate; there is no revive count limit or performance cap. Only overall saved capacity applies.
+
+Capacity includes saved seconds currently reserved by sessions. Overflow rewards and performance bonuses above the rolling budget are discarded, while partial threshold progress persists; capacity does not bank deferred whole rewards. The scripts report only actual gameplay events, eligible active seconds, and rounds with participation. The backend requires `activity:"playing"`, rejects current reward submissions while a session is reserved or started (an unresolved, stopped crash may resume gameplay), enforces monotonic counters, and rejects cumulative active time above the match's wall duration plus a 60-second telemetry tolerance. This scoped trusted-server API cannot independently observe a game engine's successful revive or verify its eligibility. The native/GSC collection paths are part of the trust boundary.
+
+## Machine protocol
+
+Use `POST /api/afk/{balance,event,purchase,reserve,start,settle}` with `Content-Type: application/json` and `Authorization: Bearer <instance bank credential>`. The body limit is 4096 bytes; responses have `Cache-Control: no-store`. Purchases, reservations, start acknowledgments, and balance reads require the exact current active telemetry match and a present GUID on the authenticated server. Gameplay backlog can additionally reference a retained historical match containing that GUID when the immutable instance lifetime proves origin. Historical active time is bounded to the recorded match end or last received telemetry, and its grants still respect held saved capacity. Gameplay whose historical membership evidence has been removed is rejected without granting time. Settlement remains valid for the original instance after disconnect, match completion, or match/server history deletion.
+
+All requests contain `server`, `guid`, and `match` (the existing telemetry `reporter_match_key`, not a separate AFK match identifier). Mutations additionally require `request_id`. GUID, match, request and session IDs are 1–64 characters from `[A-Za-z0-9_.:-]`. Optional `event_time` is a parseable timestamp no more than five minutes in the future; it records the reported event time, while `created_at` records controller receipt time. Keep the complete body immutable across retries, including `event_time`.
+
+| Action     | Additional request fields                                                                                                                                              |
+| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `balance`  | None                                                                                                                                                                   |
+| `event`    | `seq` positive integer; `active_seconds`, `kills`, `headshots`, `rounds`, `revives` nonnegative cumulative integer counters (maximum 2147483647); `activity:"playing"` |
+| `purchase` | `packs` integer 1–1000; `confirmed:true`; `expected_purchase_points` and `expected_purchase_seconds`, copied from the displayed quote, per pack                        |
+| `reserve`  | `session_id`; `seconds` integer 1–604800, the requested maximum authorization                                                                                          |
+| `start`    | `session_id`                                                                                                                                                           |
+| `settle`   | `session_id`; `consumed_seconds` integer 0–604800 or `null`; `end_reason`; optional `confirmed_seconds` checkpoint                                                     |
+
+`end_reason` is one of `manual`, `disconnect`, `death`, `match_end`, `expired`, `never_started`, or `crash`. `expired` requires exact consumption of the authorization. `never_started` requires zero. An unknown crash duration uses `consumed_seconds:null,end_reason:"crash",confirmed_seconds:<last durable elapsed lower bound>`. Confirmed consumption must fit the original authorization and cannot fall below an already-recorded checkpoint.
+
+Every application result has:
+
+```json
+{
+    "ok": true,
+    "state": "reserved",
+    "code": "ok",
+    "request_id": "request-123",
+    "free_seconds": 0,
+    "saved_seconds": 600,
+    "available_seconds": 600,
+    "reserved_seconds": 1800,
+    "capacity_seconds": 21600,
+    "authorized_seconds": 1800,
+    "bank_balance": 20000,
+    "purchase_points": 10000,
+    "purchase_seconds": 600,
+    "max_purchase_packs": 2,
+    "session_id": "session-123",
+    "consumed_seconds": null
+}
+```
+
+`available_seconds` excludes held authorizations; `reserved_seconds` is held time for this GUID across all servers. `authorized_seconds` is meaningful for session results and is the original authorization, never an extension. `session_id` and `consumed_seconds` appear on session results; `awarded_seconds` appears on reward and purchase results. `max_purchase_packs` is the smaller of affordable packs and capacity, or zero when a bank transfer is prepared. Recheck the quote through `balance` before confirmation; changed pricing rejects with `price_changed` without spending points. There is no automatic purchase.
+
+States are `balance`, `rewarded`, `purchased`, `reserved`, `started`, `settled`, `unresolved`, `corrected`, and `rejected`. HTTP 200 can contain an application rejection (`ok:false`), such as `empty`, `session_pending`, `bank_pending`, `insufficient_balance`, `reserve_capacity`, `price_changed`, `invalid_active_time`, `invalid_consumption`, `progress_regression`, or `progress_conflict`. Authentication failures are 401/403; invalid syntax/amounts are 400. `not_in_match` is a nonreceipted HTTP 409 for unknown/current membership publication lag on new work, allowing the same immutable request to retry after telemetry catches up. Known completed/obsolete matches and recorded players who have departed (`present=0`) are terminal HTTP 200 rejections for new work. Only never-observed membership remains eligible for the publication-lag retry. Gameplay missing historical membership/origin proof is also a terminal HTTP 200 rejection so an unrecoverable old snapshot cannot block the durable queue; subsequent cumulative snapshots can still include its counters if membership is established. Changed request identity or ownership is HTTP 409 `request_conflict`/`session_conflict`. Transaction failures are 503 and can retry unchanged. Early authentication/body failures may contain only `ok` and `code`.
+
+Purchase debits the existing bank account and grants saved time in the same immediate SQLite transaction, with linked point/time ledger entries. A prepared bank transfer blocks purchase until its own existing commit/cancel workflow completes. No second points bank is created.
+
+## Ordering, authorization, and outages
+
+1. Native code durably writes the immutable request before sending it. Use one stable request ID per operation, with separate IDs for reserve, start, and settle. Stable session IDs bind all session operations.
+2. `reserve` atomically takes free time first and saved time second, authorizing `min(requested,available)` seconds. Only one outstanding session can hold a GUID's funds. An identical retry returns the saved result. A new operation referring to an already-known session cannot reset or extend it.
+3. Durable native state records the reservation and the start intent. `start` must return the `started` acknowledgment before protection is enabled. Scripts additionally recheck current player/map eligibility and activate only once. A delayed acknowledgment cannot reactivate an ended local session. The countdown uses the locally recorded authorization and elapsed clock; it does not require heartbeat renewal.
+4. Manual return, disconnect, death, match end, or zero countdown removes protection and restores controls before producing a durable stop. Native code records the confirmed elapsed seconds and immutable `settle` request, then retries delivery until acknowledged. Round elapsed time up to whole seconds and cap it to authorization. At zero the scripts immediately restore normal vulnerability and controls; there is no grace, automatic buy, or spectator transition. Manual `.back` retains its separate gameplay behavior.
+5. `settle` records confirmed consumption and releases unused reserved seconds exactly once. It accepts legitimate delayed reports after membership disappears. `never_started` settlement can arrive before its reserve and installs a tombstone; delayed reserve/start cannot reopen that session. Completed/corrected sessions remain terminal. A retry of an earlier operation returns that operation's original receipt, which may predate settlement; native terminal journal state must win over such an old response.
+6. If the controller is unavailable, an already-started session keeps its locally authorized countdown to zero. If reserve/start was never acknowledged, do not enable protection. A crashed process must first be proven stopped before recovery. A checkpoint is only a lower bound on elapsed time; an absent heartbeat or old checkpoint is never evidence for a refund. Recover unknown elapsed time as `unresolved`, retaining authorization until reliable stop evidence or an explicit administrator correction resolves it.
+
+Receipts bind the complete body and action to server, immutable instance, and request ID. Reusing an ID with any changed parameter is a conflict. Accepted same/lower-sequence gameplay counters award no additional time; same sequence with changed counters conflicts, a later sequence with a decreased total is rejected, and a lower sequence cannot introduce a higher counter. Native/GSC code must preserve cumulative per-GUID totals across reconnects within the same match. A distinct revive increments `revives` even for the same teammate; retrying a snapshot never increments it.
+
+For host teardown, `POST /api/agent/afk/settle` uses the assigned host credential and `{operationId,action,transfer}`. Only an unfinished `instance.remove`, `instance.rebuild`, or `instance.cancel-create` for that host/immutable instance is accepted. `action:"settle"` reconciles session evidence; `event` and `purchase` are receipt-only recovery and return `not_applied` if the original request never committed. Teardown cannot create a new purchase or reward. Instance retirement revokes normal game API credentials, including reads; the assigned host recovery path permits settlement after retirement without granting a replacement instance ownership of old sessions.
+
+## Crash correction and reporting
+
+An administrator can reconcile an explicitly stopped `unresolved` session with `POST /api/afk/corrections` using their browser session and same-origin protections:
+
+```json
+{
+    "session_id": "session-123",
+    "request_id": "admin-case-456",
+    "consumed_seconds": 120,
+    "reason": "Confirmed process stopped; reviewed native checkpoint and host logs.",
+    "stopped": true
+}
+```
+
+The selected charge must be at least the confirmed checkpoint and no more than authorization. Correction cannot release a live reserved/started session, and does not convert an uncertain duration into an exact one: `state` becomes `corrected`, `consumedSeconds` remains null, `chargedSeconds` records the accounting decision, and `confirmedSeconds` retains the evidence lower bound. The audit ledger records the actor, reason, charge, and release. The correction ID is idempotent and rejects changed parameters. A true confirmed stop report can instead settle an unresolved session with its verified exact duration.
+
+Open **Players → AFK** (`/afk`) to browse all AFK accounts, sessions, and time-ledger entries. The accounts view supports player-name and exact-GUID searches and links each account to its player profile. Session history includes unresolved reservations; session and ledger history can be filtered by game, server, and date. Filters and pagination remain in the URL when opening a player or retained match, so the back link restores the same overview. Refresh reads current balances without issuing grants, releasing reservations, or otherwise changing accounts. This reporting page has no purchase or correction controls.
+
+The AFK overview and player profiles use the same signed-in viewer access as Bank. Player profiles also retain balances and recent time-ledger entries alongside bank information. Reports distinguish cumulative verified active seconds, confirmed AFK duration, unresolved sessions, expiration count, manual returns, purchases, and corrected charges. Session starts, end reasons, operation/session IDs, server, match when retained, reported event time, receipt time, resulting balances, free grants, rewards, point-linked purchases, reservations, consumption, releases, and corrections survive match retention and server deletion.
+
+## Coordinated rollout and validation
+
+Back up the controller database and persistent game journals. Deploy EC first;
+migration 30 creates the AFK accounts, operations, progress, sessions, bonus budget
+and ledger tables without match/server cascading deletion. Install host agent
+**1.12.0+**, including its `afk.settlement` capability, before activating the game
+scripts. Package and offline-sign the agent release using the normal host update
+workflow; a local build does not sign or deploy it.
+
+Install TWS AFK scripts **2.0.0** with matching native plugins **1.4.0+** for both
+engines, common **1.4.0+**, telemetry **1.0.0+**, and Plutonium **r5334+**. Fully
+restart each game-server process for the plugin update. Keep the existing private
+bank credential configuration and persistent `tws-afk/t5/` / `tws-afk/t6/` storage.
+See the [scripts AFK guide](https://gitea.thewrightserver.net/josh/tws-zombies-scripts/src/branch/main/docs/afk.md)
+for commands, active-play criteria, protection ownership, game dvars and the
+in-game validation matrix.
+
+Run `npx vitest run tests/unit/afk.test.ts tests/unit/bank.test.ts tests/unit/ui-afk-profile.test.ts`,
+the normal EC checks/build, and `python3 -m unittest discover -s agent/tests -v`.
+The scripts repository supplies `node tools/check_afk.mjs` and
+`python3 tools/test_plugin_afk.py` with the pinned native JSON dependency headers.
+Portable native tests use sanitizers and engine adapters; script tests use engine
+shims. Neither proves Windows DLL linking or live game behavior.
+
+Before production rollout, exercise both T5 and T6 with a short test allowance:
+manual return, immediate zero-time expiry, downed/dead/spectator states, `.hud`
+hidden, repeated revives, partial reward thresholds across reconnects, explicit
+purchase confirmation and capacity rejection. Compare the center countdown to
+the authorized seconds, ensure controls/weapons and normal vulnerability return
+at zero, and confirm active-play metrics exclude AFK time (manual `.back` restores reward eligibility immediately, including during its separate protection grace). Test last-active
+death/disconnect and stock afterlife/Grief recovery on deployed maps.
+
+Drop reserve/start/purchase/settle responses, reorder deliveries, disconnect EC
+after authorization and restart game/controller/host processes. Confirm the local
+deadline survives the outage, durable stops release unused time once, and ambiguous
+crashes remain unconfirmed until an audited correction. Rebuild/remove an instance
+with both games' journals and a retired game credential; host recovery must retain
+files until acknowledged and must never create a new purchase or gameplay reward.
+Live T5/T6 gameplay and deployment validation require the actual game servers.

docs/guides/authentication.md

@@ -0,0 +1,229 @@
+# Users and access
+
+Electric Cherry requires a user session for operator pages and management APIs.
+HTTPS protects the connection; server-side role checks protect each operation.
+There is no anonymous management mode or public registration.
+
+Leaderboard API reads (`GET` and `HEAD /api/leaderboard`) are public and need no
+session or credentials. This exception covers only that exact API route;
+leaderboard/profile pages, match-history routes and management operations retain
+their access checks. HTTPS requirements still apply. See the
+[leaderboard API reference](leaderboard.md#read-api) for requests and response fields.
+
+## First administrator and upgrades
+
+1. Generate a setup secret: `openssl rand -hex 32`.
+2. Set `EC_SETUP_TOKEN` to that value in the controller environment (or Compose
+   `.env`), set `EC_PUBLIC_URL` to the external HTTPS URL, and start EC. For a
+   direct Node deployment, set `ORIGIN` to that same URL.
+3. Visit EC. `/setup` asks for the setup token, a username, display name and
+   password. Only the holder of the token can create the first administrator.
+4. Remove `EC_SETUP_TOKEN` after setup and restart/recreate the controller.
+   Setup remains closed once any user exists, even if the token is still set.
+5. Use **Admin → Users** to create additional accounts. Share temporary
+   passwords privately; each user must change theirs on first login.
+
+Existing installations become protected on upgrade. Existing game servers and
+agent enrollment credentials continue working independently of browser login.
+Preserve the current database when upgrading: accounts and sessions are stored
+there alongside the fleet metadata. Never publish a blank setup page with a
+guessable token. An unset setup token holds setup closed.
+
+## Roles
+
+The Moderator role has Viewer-level fleet access plus private player moderation.
+It permits live player actions, case review and server/game restrictions lasting
+up to seven days, without granting fleet operations or access to raw configs and
+credentials. Administrators also control permanent/fleet restrictions, integration
+configuration and revoking another moderator's restriction. Super Viewers can
+read moderation; Operators and Viewers cannot. See [Player moderation](moderation.md).
+
+| Capability                                                                                    | Viewer         | Super Viewer | Operator    | Administrator |
+| --------------------------------------------------------------------------------------------- | -------------- | ------------ | ----------- | ------------- |
+| Fleet, metrics, matches and operation summaries                                               | Yes            | Yes          | Yes         | Yes           |
+| Raw configs, file diffs, logs, saved RCON and telemetry diagnostics                           | No             | Read         | Read        | Read          |
+| Deploy/start/stop/restart/rebuild/remove servers and run diagnostic probes                    | No             | No           | Yes         | Yes           |
+| Scripts, plugins, configs, campaigns, betas and collections                                   | Read summaries | Read         | Manage      | Manage        |
+| Host setup, deletion, enrollment, credentials, runtime preparation and game-file upload/cache | Read status    | Read status  | Read status | Manage        |
+| Server keys and global settings/maintenance policies                                          | No             | Read         | No          | Manage        |
+| Users and account audit activity                                                              | No             | Read         | No          | Manage        |
+| Create users, change roles, disable accounts and reset passwords                              | No             | No           | No          | Yes           |
+| Own password and sign out                                                                     | Yes            | Yes          | Yes         | Yes           |
+
+Super Viewer is a read-only administrator. It includes the Admin navigation and
+access to sensitive configuration and diagnostic content, but cannot change fleet
+state, settings, keys, or other accounts. Cleanup previews remain available because
+they do not save settings or run cleanup. Assign it only to people trusted to read
+administrative information. Existing accounts and sessions are preserved on upgrade.
+
+Operators can launch using available game-pool keys already saved by an administrator. They
+cannot save keys indirectly through a launch request. An operator's content
+permissions include executable scripts and plugins; assign that role only to
+people trusted to operate the game fleet.
+
+Drift previews (`/api/drift`) and GitHub release lookups
+(`/api/plugins/github/release`) require operator or administrator access, including
+`GET` and `HEAD` requests. These endpoints initiate fleet file scans or use GitHub
+API quota; Viewers and Super Viewers cannot trigger them.
+
+Saved RCON failure replies are available to Super Viewers, operators and administrators in the
+server's Diagnostics tab. Player addresses are redacted before diagnostics are
+saved; successful checks clear the saved reply. Fleet snapshots and live status
+streams contain no reply snippets. Upgrades discard older saved RCON snippets.
+
+Permissions are checked on the server for API calls, normal pages and Svelte
+page-data requests. Hiding UI controls is a convenience, not the access boundary.
+The controller prevents disabling or demoting the last enabled administrator.
+Accounts are disabled rather than deleted so history retains their identity.
+
+## Passwords, sessions and recovery
+
+Passwords are 8–128 characters and stored using salted scrypt hashes. Temporary
+passwords created or reset by administrators must be changed before any fleet
+access. Password changes require the current password.
+
+Sessions use random tokens in HTTP-only cookies, with only token hashes stored in
+SQLite. HTTPS uses a Secure `__Host-` cookie. By default, sessions expire after
+12 hours, or 30 minutes without authenticated requests; background dashboard
+polling counts as activity.
+
+Select **Keep me signed in on this device** at sign-in to stay signed in for
+30 days, including between visits from a phone's home-screen app. This option is
+unchecked by default and applies only to that browser or home-screen app. Remembered
+sessions have no inactivity timeout, but still expire 30 days after sign-in;
+activity does not extend that deadline. Use this option only on a device you trust.
+Existing sessions keep their original limits after upgrading.
+
+Password changes always require the current password. Remembered administrator
+sessions also require the administrator's current password when creating accounts
+or editing users, including access changes and password resets. A successful
+password change replaces the current session, preserving its remember-device
+choice with a fresh lifetime, and revokes all other sessions.
+
+For API clients, `POST /api/auth/login` accepts an optional boolean `rememberMe`
+(omitted or `false` keeps the default limits). When using a remembered session,
+`POST /api/users` and `PATCH /api/users/:id` require the administrator's password
+in `currentPassword`; the `password` field remains the target user's temporary
+password. The password-change endpoint preserves the current session's choice.
+
+Login replaces the existing browser session. Logout revokes it.
+Changing a password, changing a role, disabling an account, or revoking that
+user's sessions invalidates their existing sessions. Live event streams check
+revocation every 15 seconds and then close.
+
+Login and password changes for enabled Administrators and Super Viewers are exempt
+from per-account and per-address rate limits, including existing exhausted windows.
+Their attempts do not consume either limit. Other accounts use persistent
+15-minute login windows of 10 attempts per username-and-address pair and 30 per
+address across accounts. Attempts from one address do not exhaust another
+address's allowance for the same username. Initial setup and password changes
+retain their limits of 10 per account and 30 per address. Exemptions use the
+account's current database role; password verification is still required.
+
+A successful password login also sets a separate, HTTP-only login-device cookie
+for 30 days. This cookie lets that browser skip its username-and-address limit,
+while the password and shared address limit still apply. It does not sign you in,
+and is independent of **Keep me signed in**. It survives ordinary sign-out;
+password changes/resets, role changes, disabling an account, and revoking all of
+its sessions invalidate its login-device cookies. Only token hashes are stored in
+SQLite. Reusing a valid cookie does not extend its original expiry.
+
+The database stores audit entries for login
+results, account changes and successful management requests without recording
+passwords, tokens or request bodies. Administrators and Super Viewers can review
+recent entries in Users.
+
+For every role, the password service runs at most two scrypt operations at once
+and queues up to 16 more for up to two seconds. If that queue fills or the wait
+expires, the request returns HTTP 429 with `Retry-After: 2`. The persistent rate
+limits return `Retry-After: 900` when a non-exempt account or address reaches its
+attempt limit.
+
+An administrator can reset another user's password from Users. If the only
+administrator forgets their password, an operator with local controller/container
+access can run the recovery command in an interactive terminal:
+
+```sh
+docker compose exec electric-cherry node scripts/reset-admin.mjs admin
+# Direct Node deployment, using the correct database:
+DB_PATH=./data/cherry.db node scripts/reset-admin.mjs admin
+```
+
+Replace `admin` with the existing administrator's username. The command prompts
+without echoing the password, resets that enabled administrator only, revokes
+their sessions and trusted login devices, clears their account-specific login and
+password-change rate limits, and records the recovery. Other users' limits and
+shared address limits remain intact. It does not expose a remote recovery endpoint
+or accept passwords as command arguments.
+
+## Agent and telemetry connections
+
+Host agents retain enrollment/host-token authentication under `/api/agent/`.
+Installer and release files remain public so a new machine can enroll. A browser
+login never substitutes for a host credential, and host tokens cannot manage EC.
+The agent and standalone updater source contain no credentials. Automatic
+updates fetch the controller's public HTTPS release manifest and verify its
+checksums and compatibility. The installed host retains its own credential;
+the updater accepts an upgrade only after a fresh authenticated heartbeat from
+the new agent process. Updating agent code does not change browser sessions or
+user roles. See [automatic agent updates](native-hosts.md#automatic-agent-updates)
+for the one-time installer upgrade required by existing hosts.
+Host deletion uses a separate operation-scoped receipt token so the uninstall
+worker can confirm removal and retry its acknowledgment after host credentials
+are revoked. That token authorizes only the corresponding cleanup receipt.
+
+Game reporters must send a separate secret of at least 32 characters configured
+as `EC_TELEMETRY_TOKEN`:
+
+```http
+POST /api/telemetry
+Authorization: Bearer <telemetry-token>
+Content-Type: application/json
+```
+
+`X-EC-Telemetry-Token` is also supported. Cookies, query parameters and JSON fields
+cannot supply it. A missing server token disables ingestion; a missing or wrong
+request token is rejected before parsing or recording the payload. Existing JSON
+event fields are unchanged. Configure your uploaded reporter/plugin, or a trusted
+relay if that reporter cannot set headers, before upgrading. Use HTTPS for the
+reporter-facing endpoint. Rotating this shared token requires updating reporters.
+
+## Proxy and local development
+
+Keep Node's internal port behind your TLS proxy and configure `ORIGIN` correctly.
+Management writes require an exact matching `Origin` header, including login,
+JSON requests and multipart uploads. Do not configure a proxy to replace arbitrary
+incoming origins with your trusted origin. Machine endpoints use their own token
+checks and do not require a browser Origin.
+
+Login throttling uses the address returned by SvelteKit's adapter. Without
+`ADDRESS_HEADER`, this is the direct peer address, so users behind a reverse proxy
+share the proxy's address limit. Compose passes `ADDRESS_HEADER` and `XFF_DEPTH`
+from `.env` to the controller; direct Node deployments must set them in the
+server's environment. For one trusted proxy that sets `X-Forwarded-For`, use:
+
+```dotenv
+ADDRESS_HEADER=x-forwarded-for
+XFF_DEPTH=1
+```
+
+Recreate the Compose controller after changing these values with
+`docker compose up -d`. For a proxy chain, set `XFF_DEPTH` to the number of trusted
+proxies: the adapter selects that many positions from the right of the header.
+For example, `client, proxy1` with two trusted proxies requires `XFF_DEPTH=2`.
+The public-facing proxy must replace untrusted forwarding headers, and every
+trusted hop must supply the expected address chain. Restrict direct access to
+port 3000 so requests cannot bypass those proxies. Do not enable these settings
+for an untrusted proxy or copy a depth that does not match your deployment.
+See the [adapter's client-address settings](https://svelte.dev/docs/kit/adapter-node#Environment-variables-ADDRESS_HEADER-and-XFF_DEPTH).
+Forwarded host or user headers do not authenticate users.
+
+Local HTTP browser development requires `EC_AUTH_ALLOW_HTTP=true` and a loopback
+hostname (`localhost`, `127.0.0.1` or `::1`). This uses a separate development
+cookie. It never enables HTTP login for a LAN/public hostname. Agent HTTP
+development remains separately controlled by `EC_AGENT_ALLOW_HTTP` and the agent's
+loopback-only option.
+
+The implementation follows OWASP's guidance for [authorization on every request](https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html#validate-the-permissions-on-every-request),
+[password hashing](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#scrypt),
+and [session management](https://cheatsheetseries.owasp.org/cheatsheets/Session_Management_Cheat_Sheet.html).

docs/guides/autoscaling.md

@@ -0,0 +1,86 @@
+# Lobby autoscaling
+
+Open **Fleet → Autoscaling** to keep empty lobbies available for new groups. Policies apply to one map variant or every map in a game. A game-wide policy applies the same minimum, maximum, and spare targets independently to each map, including all variants. A partly occupied Zombies lobby is not an empty spare, even when it still has player slots.
+
+Autoscaling is opt-in. Installing the feature creates no policies and changes no lobby counts. Choose On and save a policy to enable autoscaling.
+
+T5 game-wide policies cover the ten standard Zombies maps; T6 policies cover nine co-op map selections. Dead Ops Arcade, Grief, and Turned are not supported.
+
+## Configure a policy
+
+Choose **Add policy**, select the game and scope, and enter an explicit maximum. The initial editor selects Off, with a minimum of one running lobby, one empty spare, and a 15-minute empty interval. Counts must be between 1 and 20; the minimum and spare target cannot exceed the maximum.
+
+Choose **Save policy** to apply changes directly. Saving checks the current fleet, policy revision, ownership conflicts, and host readiness. If saving is blocked, the form keeps your settings and shows the reason. Saved policies show lobby counts, key availability, and the current status. An active job appears once with its latest stage and a **View job** link. Use **Lobbies** for per-map spare coverage and occupancy evidence and **Activity** for decision changes grouped by job. [**Metrics**](metrics.md) shows observed lobby counts and empty spare hours, plus autoscaling additions, removals, failed or cancelled operations, and average/maximum request-to-ready time for the selected period, game, and map. The policy card retains its separate 31-day measurement totals, including spare deficits.
+
+| Autoscaling | Behavior                                                                                              |
+| ----------- | ----------------------------------------------------------------------------------------------------- |
+| Off         | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled.  |
+| On          | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. |
+
+Plutonium keys belong to a game pool shared by every map, manual launch, collection, beta, and autoscaling policy for that game. An eligible creation uses the lowest available lobby number from 1–20 and reserves any free game key atomically. Key labels do not select a map or lobby. The policy shows saved and available game keys; policy maxima can exceed pool size, and a policy can be saved before any keys are added. An empty or exhausted pool holds new growth with a clear reason; it does not prevent surplus removal. Queued launches reserve their key, stopped instances retain it, and confirmed cleanup returns it to the pool. Failed or uncertain launches keep their reservation until safely reconciled. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host.
+
+The desired count is `min(maximum, max(minimum, occupied lobbies + empty spare target))`. Each occupied lobby must reach **round 2 or greater**, confirmed by fresh telemetry for its current match, before it increases the creation target. Brief joins, round-one matches, and unavailable round evidence do not trigger an additional spare. The minimum and baseline spare target can still bootstrap a map without an active match. Existing spares remain protected by the ordinary desired count even when a match restarts or round evidence becomes unavailable.
+
+The controller evaluates policies every 15 seconds and requests at most one unresolved operation per policy. Game-wide policies apply the round requirement independently to each map and allow one outstanding operation across their maps, prioritizing additions before removals. Once an operation settles, the next evaluation can act immediately on another eligible lobby. Unsubmitted creates are withdrawn if they no longer meet the round or capacity requirements. Surplus lobbies must still satisfy the configured continuous empty duration before removal. Create uses the existing startup concurrency, CPU, memory, library, port, and host admission checks. Lack of host capacity leaves one request waiting; it does not create additional requests every tick. Creates use current promoted launch files when provisioning begins.
+
+When trimming a map, autoscaling keeps occupied lobbies and the lowest-numbered
+empty lobbies needed for its minimum and spare targets. It selects the
+highest-numbered eligible surplus lobby for removal. The required empty lobbies
+are protected before checking idle timers, so Lobby 1 stays available while a
+higher surplus lobby completes its empty interval. Existing numbering gaps remain
+until normal growth needs another lobby; autoscaling does not replace healthy
+lobbies just to renumber them.
+
+Existing policies saved in the removed Dry run or Create only modes switch to Off on upgrade. Review and turn autoscaling On to resume; pending work remains tracked until settled.
+
+## Ownership and blockers
+
+A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. While autoscaling is active, competing collection launches and manual Create, Start, Stop, or Destroy requests are rejected with a policy link. Pausing globally or pausing the policy allows these manual actions through their usual provisioning checks. Restart and Rebuild remain available through the usual provisioning checks. Their pending work and unresolved recovery hold scaling decisions for the affected map, so temporary downtime does not trigger replacement lobbies. Exact collection matching also checks managed maps absent from the collection.
+
+Unknown or stale occupancy, stopped or unhealthy members, active betas, campaigns, maintenance, host drains, and unresolved recovery hold the policy's decisions. Autoscaling does not repair these conditions itself. The page and linked Provisioning operation explain the current blocker, including capacity and game key waits.
+
+[Capacity rebalancing](native-hosts.md#rebalance-instances) preserves lobby identities
+and counts. Each move uses a Provisioning rebuild, which holds scaling for the
+affected map until the move settles. The temporary outage does not request a
+replacement lobby.
+
+Current positive occupancy from either telemetry or RCON vetoes an empty claim. Empty intervals reset when players arrive, evidence becomes unknown, the instance or lifecycle changes, or observation continuity breaks. Controller downtime does not count toward the empty duration. Failed or uncertain automatic operations pause further actions until an operator inspects their Provisioning recovery and reviews the policy again.
+
+## Pause and cancellation
+
+Use **Pause all** to stop new decisions globally, or **Pause** on one policy. Use **Resume** to enable it again; the usual activation checks still apply. A global pause retains each live policy's ownership to prevent overlapping policies, but allows manual fleet, server, and collection actions. Policies paused after a failure also allow manual actions. Work already in progress still uses the normal per-server provisioning checks. Setting a policy Off releases ownership once its outstanding operation settles. Resuming the global switch preserves each policy's On or Off setting and restores the manual-action guard for active policies.
+
+Unsubmitted creates and queued removals are withdrawn where possible. Cancelling an autoscaling operation from Provisioning also turns its policy Off so it cannot immediately request a replacement.
+
+An automatic removal requires fresh controller eligibility, a short-lived authorization, and a host-local occupancy check. The agent claims authorization once, repeats its occupancy check, and must begin mutation within five seconds of requesting the claim. Occupied, unknown, expired, interrupted-before-mutation, or revoked attempts end without waiting to destroy the lobby later. A new attempt needs a new decision and another complete empty interval. Automatic removals cannot use the administrator safety override.
+
+The authorization claim is the cancellation boundary. A pause after that point cannot guarantee withdrawal; the page retains the outstanding operation until its receipt is reconciled. Do not treat cancellation as proof that a delivered host command had no effect. Uncertain commands retain their identity and resource reservations.
+
+Turning a policy Off does not recreate lobbies it removed or remove lobbies it created. Once operations settle, use Collections to restore a fixed lineup through the normal capacity and empty-server gates.
+
+## Delete a policy
+
+Open the policy’s **More actions** menu, choose **Delete policy**, and confirm to remove its settings, decisions, and measurements. Existing lobbies, saved keys, and Provisioning history are kept. Deleting a live policy releases ownership without changing the lineup, and a new policy can reuse the same scope.
+
+Deletion requires the current policy revision and no unsettled autoscaling operation. If work is outstanding, turn the policy Off and wait for its operation to settle before deleting. Delivered or uncertain host commands retain their policy and admission evidence until reconciliation finishes.
+
+## Pilot
+
+The production observation period and live pilot have not been run as part of implementation. Begin them on the operator-selected maps after deploying the controller and compatible agents.
+
+1. Select one or two representative maps and record the baseline lineup, keys, versions, host CPU and memory, and player-hours.
+2. Select On after resolving blockers and checking agent capability. Set the minimum, maximum, and spare target, then choose Save policy.
+3. Use Metrics to monitor request-to-ready time, empty lobby-hours, and autoscaling removals for the selected period and scope. Check the policy card's 31-day spare deficits and individual Provisioning operations for delays; verify browser visibility and successful external joins separately. Increase the spare target if creation latency leaves new groups without a suitable lobby.
+4. Expand one map at a time when spare availability and gameplay meet your requirements. Pause if they regress.
+
+The page retains up to 31 days of decision evidence and observations, with the 30 most recent decision changes shown. Repeated unchanged evaluations update the decision's last-seen time instead of adding a row. Unresolved intent survives retention. Provisioning history retains the operation and policy backlink separately.
+
+Measurement totals exclude collection gaps. Request-to-verified-readiness summaries include queue time and only completed creates; they do not establish browser visibility or successful joins. [Metrics](metrics.md) shows average and peak lobby counts, empty spare hours, and map time with no reported empty lobby. Its autoscaling signals compare historical counts with current policy targets. Use host CPU and memory measurements alongside player-hours and spare availability when assessing savings.
+
+## API and implementation
+
+`GET /api/autoscaling` returns policies, current evidence, recent decisions, measurements, and global pause state. Viewers may read it; mutations require an operator or administrator, using the existing authentication and origin checks.
+
+`POST /api/autoscaling` accepts `save`, `delete`, or `global-pause`. Delete requires `id` and the current `revision`. Save requests contain `settings`, plus `id` and `revision` when editing. Use `mapSlug: '*'` for every map in the selected game; existing map-specific requests are unchanged. Scope cannot change when editing. Save validates the current fleet atomically with the policy update; no preview or token is required. Global pause uses a boolean `paused` field. Agent removal claims use a separate host-authenticated endpoint and are bound to the exact command, host, instance generation, policy revision, and expiry.
+
+Shared contracts live in `src/lib/autoscaling.ts`; the evaluator, snapshots, persistence, ownership, and authorization live in `src/lib/server/autoscaling/`. Provisioning remains the only lifecycle executor. See the [approved implementation plan](../plans/lobby-autoscaling-plan.md) for the design and rollout criteria.

docs/guides/bank.md

@@ -0,0 +1,236 @@
+# Player bank
+
+The bank stores one points balance per exact GUID across T5 and T6. Players use
+`.b`, `.d <amount|all>` and `.w <amount|all>` with the TWS bank scripts. Deposits
+move spendable match points into the bank; withdrawals return bank points to
+the current match. Neither changes earned `score_total`. Bank-assisted games
+still count toward the existing leaderboards; the ledger records their match
+IDs. This release does not introduce separate bank-assisted rankings.
+
+## Setup and rollout
+
+1. Back up the controller database. Update every host agent to **1.9.1 or later**
+   before installing the public-endpoint bank scripts. This version settles both
+   public and legacy controller-derived journals before removing runtime storage.
+   Package and offline-sign the agent
+   release using the [host update workflow](native-hosts.md).
+2. Deploy Electric Cherry and restart the controller. Migration 20 creates
+   accounts, requests and the ledger; migration 21 adds instance credentials and
+   binds existing requests to their currently active native instance, if any.
+3. Set `EC_BANK_MAX_BALANCE` (default 250000, range 1–1000000). Invalid configuration
+   refuses new deposits. Lowering the limit preserves existing balances and
+   prepared deposits; withdrawals remain available.
+4. Build/install TWS plugins `t5-plugin`/`t6-plugin` 1.1.0 or later and restart
+   the **whole game-server process**. Install `tws_t5_bank.gsc` or
+   `tws_t6_bank.gsc` 2.0.0, common 1.0.0+ and telemetry 1.0.0+.
+   For existing servers changing endpoints, use the rebuild procedure below.
+5. Create or rebuild the servers through EC. Launch preparation generates a
+   cryptographically random `ec_bank_` credential for each immutable instance,
+   stores it in the private controller database, and injects it through
+   `configs/electric_cherry_bank.cfg`. The launch command executes this private
+   file and sets `tws_server_id`. The host agent writes only `tws_bank_token` in
+   that private file; it no longer generates `tws_bank_url`.
+
+Both game scripts always send bank requests to
+`https://ec.thewrightserver.net/api/bank`, even if an old `tws_bank_url` remains
+in configuration. The host agent's enrolled HTTPS controller URL is separate
+and continues to carry agent traffic, including authenticated bank settlement.
+It may differ from the public endpoint reachable by game containers.
+
+There is **no `EC_BANK_TOKENS` environment map** and no per-server secret setup.
+The private configuration is sent only in that host's authenticated operation
+payload. It is excluded from shared recipes, promoted files and blob downloads.
+Do not upload `electric_cherry_bank.cfg`: its filename is reserved. Protect the
+controller database, host operation journal and per-instance runtime directory,
+which contain private credentials.
+
+Drift scans exclude the exact generated configuration path for both games.
+A maintenance rescan resolves old bank-only incidents and retains other file
+changes in mixed incidents. Private-file campaign plans saved before this fix
+are rejected at activation; cancel them and rescan rather than retrying the old
+plan. Upgrade host agents to **1.9.3 or later** before resuming drift recovery
+so stages already saved on a host also reject private-file reads and changes.
+Legacy private bank-file campaign diffs and their blob downloads are blocked;
+this does not erase previously captured bytes from databases or backups.
+
+Retries, rebuilds and host moves of the same instance reuse its credential.
+Retiring the instance immediately revokes authentication. Creating another
+server with the same display name produces a new instance ID and secret; it
+cannot finish the old instance's requests. Accounts remain attached to player
+GUIDs, so balances survive the servers. Telemetry tokens and browser sessions
+cannot authorize bank requests. Host credentials can only settle decisions
+through the separate, operation-scoped teardown endpoint.
+
+For an upgrade from the initial static-token implementation, drain outstanding
+transfers while the old controller/configuration still runs, then deploy and
+rebuild. Remove the obsolete environment map and any manually supplied token.
+Do not replace identities or discard pending journals to force an upgrade.
+
+### Upgrading from a controller-derived game endpoint
+
+Update agents to `1.9.1+` first. If an existing server's `tws_bank_url` differs
+from `https://ec.thewrightserver.net/api/bank`, rebuild that server through EC
+with bank scripts `2.0.0`. The agent stops the old game, settles its pending
+commit/cancel decisions with their original request IDs, then starts the
+replacement and removes old storage. The server identity and instance credential
+are preserved. Lost acknowledgments can replay the same decisions; EC's retained
+request IDs prevent duplicate ledger changes. An outage or invalid journal
+retains storage and blocks replacement until recovery succeeds.
+
+Do not deploy the new scripts with only a map restart when the endpoint changes:
+native journals are keyed by server and endpoint, and the new scripts cannot load
+the old scope. Do not rename, rewrite or delete pending journals. Keep the
+agent's enrolled controller URL unchanged until legacy journals drain; the
+agent accepts only the fixed public endpoint and the legacy `/api/bank` URL
+derived from that enrolled controller. Other legacy URLs require recovery with
+their original configuration before switching. If the old URL already matches
+the exact public endpoint, existing journals retain their identity and a map
+restart is sufficient for the script upgrade. No new native plugin build is
+required beyond `1.1.0`.
+
+## Machine API
+
+All routes below are `POST /api/bank/<action>`, with `Content-Type:
+application/json` and `Authorization: Bearer <server-bank-token>`. Responses
+are never cached. Bodies are capped at 4096 bytes. GUIDs and request IDs stay
+strings; never convert them to numbers.
+
+| Action     | Body                      | Meaning                                                              |
+| ---------- | ------------------------- | -------------------------------------------------------------------- |
+| `balance`  | `server`, `guid`          | Read the shared balance and currently available points               |
+| `deposit`  | Transfer fields below     | Reserve deposit capacity without crediting the account yet           |
+| `withdraw` | Transfer fields below     | Reserve bank funds without debiting the account yet                  |
+| `commit`   | Identical transfer fields | Apply a prepared transfer once and append its ledger entry           |
+| `cancel`   | Identical transfer fields | Release a reservation; also prevents a delayed prepare from applying |
+
+Transfer example:
+
+```json
+{
+    "server": "t5_zombie_theater_1",
+    "guid": "18446744073709551615",
+    "match": "reporter-match-key",
+    "request_id": "unique-process-token-17",
+    "operation": "deposit",
+    "amount": 2000,
+    "capacity": 995000
+}
+```
+
+`match` is the reporter's telemetry match key, **not** Electric Cherry's numeric
+match ID. `request_id` is unique for the server and at most 64 characters.
+`amount` is a positive integer up to 1000000; withdrawal also accepts `"all"`.
+The script resolves deposit `all` to its captured spendable points. `capacity`
+is the current room below the game's bank withdrawal cap of 1000000 points.
+An `all` withdrawal that exceeds this room is refused, rather than truncated.
+
+Preparation requires the GUID to be present in the server's latest active match,
+with the exact telemetry match key. Late joiners may need to wait for the next
+30-second heartbeat. Unknown/destroyed servers, absent players and old matches
+cannot prepare transfers. Settlement of an existing decision remains possible
+after match completion or retention. Host teardown can finish saved decisions
+after retirement; the retired game credential itself no longer authenticates.
+
+Successful application responses include `ok`, `state`, `code`, `request_id`,
+`amount` and `balance`. States are `balance`, `prepared`, `committed`, `cancelled`
+or `rejected`. For balance reads, `amount` is the available balance. A business
+rejection uses HTTP 200 with `ok: false`, `state: "rejected"` and a code such as
+`insufficient_balance`, `empty`, `bank_limit`, `points_limit`, `not_in_match` or
+`pending`. It never changes points. Authentication failures use 401/403,
+malformed input 400/413/415, identity conflicts 409, and unavailable service 503.
+
+Repeated requests return the saved result/state without applying a second
+change. Settled IDs never reopen; a committed result retains its original
+resulting balance even if later transfers change the account. Reusing an ID
+with different transfer fields returns 409. A GUID has at most one prepared
+transfer across all servers. SQLite transactions serialize checks, account
+updates and ledger insertion, so concurrent requests cannot overspend funds or
+deposit past the configured maximum.
+
+## Lost responses and recovery
+
+The game saves a cancellation decision **before** sending a prepare request.
+Only a validated prepared reply lets it recheck the player's life, current
+spendable points and withdrawal room. It then records a commit decision and
+updates spendable points without yielding. The sender retries settlement with
+the original identity, at delays of 1, 2, 4, 8, 16, then 30 seconds. Invalid,
+truncated or uncorrelated replies never authorize points changes.
+
+A failed/uncertain prepare leaves game points unchanged and queues cancellation.
+Cancellation can arrive first: its tombstone blocks the delayed prepare. Death,
+respawn, disconnect or match end before application also cancels. After a commit
+decision, retries only finish the ledger; they never change game points again.
+These points now belong to the current match, so an applied withdrawal is not
+automatically refunded if that match ends or its server crashes.
+
+Keep `tws-bank/t5/` or `tws-bank/t6/`, relative to the game process working
+directory, on persistent writable storage with a
+single process per server/endpoint. Journals contain transfer data, not secrets.
+On a map/process restart, saved decisions settle before new bank commands.
+Unreadable storage disables the bank; failed writes prevent points changes.
+Pending decisions remain until acknowledged, even on credential errors. Bank
+holds do not expire automatically, because expiring a saved commit can lose a
+deposit or create a free withdrawal. Restore connectivity/configuration and the
+original journal to resolve holds; never delete a pending journal to unblock
+the bank. If storage is permanently lost, reconcile against the retained
+request and ledger records before any manual repair.
+
+During removal, rebuild, host migration or launch cancellation, agent 1.9.1+
+stops the game and drains its journals **before** deleting runtime files or
+starting a replacement. It uses its host credential at
+`POST /api/agent/bank/settle`; EC checks the assigned host, immutable instance,
+unfinished teardown operation, server and original request identity. This
+endpoint permits only commit/cancel, never new deposits or withdrawals. Journal
+URLs must match the public game-bank endpoint or that host's legacy
+controller-derived bank endpoint; the agent never follows a journal URL.
+Corruption, an unrecognized endpoint, failed acknowledgment or an
+outage retains storage and leaves the operation unresolved for retry. An
+interrupted rebuild uses the existing rebuild recovery workflow to retry with
+a fresh operation after settlement. Do not manually erase its old directory.
+
+The journal provides recovery from map/process restart. It cannot make live
+game memory and SQLite one atomic transaction or preserve match points after a
+machine/storage failure. Back up persistent storage and verify crash behavior
+in the actual Windows/Plutonium deployment before rollout.
+
+## History and validation
+
+The **Players → Bank** page shows fleet-wide balances, funded accounts, completed
+transfers in the last 24 hours and pending transfers. Accounts can be searched by
+player name or GUID, sorted by balance or recent completed activity, and filtered
+to funded accounts, accounts at or above the configured limit, or pending transfers.
+Balances always remain shared across both games.
+
+Transactions can be filtered by player, game, server, direction and inclusive UTC
+dates. Pending transfers show oldest first, with an attention indicator after five
+minutes and details explaining the reservation. This indicator does not expire or
+cancel a transfer. Both views retain historical server identities and match IDs
+after retention; unavailable matches have no live link.
+
+Selecting a player opens a drawer with their balance, available points, pending
+transfer and paginated history. Filters and pagination are retained in the URL,
+including when following a match or player-profile link. The page refreshes every
+30 seconds while visible. Summary counts always cover the whole fleet. Bank is
+read-only for all browser roles; transfers continue to originate in game.
+
+Player profiles show the shared balance, pending status and three newest ledger
+entries, with a **View bank history** link to the drawer. Entries include signed
+amounts, resulting balances, server, time and match.
+Accounts, request IDs and ledger entries have no cascading match/server foreign
+keys and are not deleted by match retention or server deletion. Retained IDs
+also keep old retries idempotent. Expired match links become plain historical
+IDs. Include these tables in normal database backups.
+
+Run `npx vitest run tests/unit/bank.test.ts tests/unit/bank-overview.test.ts tests/client/bank-page.test.ts`, then the usual lint, type checks,
+tests and build. TWS supplies `node tools/check_bank.mjs` and
+`python3 tools/test_plugin_bank.py`. Live validation must cover both games:
+deposit/withdraw amounts and `all`, insufficient funds and caps, joining before
+telemetry, `.help`, score HUD updates, death/respawn/disconnect during requests,
+lost prepare/commit replies, controller outage, disk failures and map/process
+restart with pending work. Confirm earned score and leaderboard behavior, and
+verify that another server's credential and the telemetry credential fail.
+For the endpoint upgrade, also test a different enrolled controller address and
+a stale `tws_bank_url`, `.b` with an empty balance, `.w all` with no funds, and
+rebuild/removal with both legacy and public journals. Drop a settlement reply,
+restart the agent and verify the original IDs settle without duplicate ledger
+changes before the replacement starts or runtime storage is removed.

docs/guides/beta-test-plans.md

@@ -0,0 +1,449 @@
+# Beta test plans
+
+Create a plan in Betas, choose how many instances per game must pass, and deploy. The server's game script
+reports gameplay to Electric Cherry. The same reports update Fleet and any active
+beta on that server.
+
+## Files and automatic server selection
+
+A beta can bundle files for different games, including T5 and T6 together. Each
+file installs only on participants matching its game or map scope; an unscoped
+file applies to all participants. Every file needs at least one matching test
+participant. For example, a T5 script and a T6 script can share one beta without
+either script being installed on the other game's servers.
+
+Choose **Add all eligible files** to add the latest unpromoted version of every
+remaining eligible file at once. Files and exact versions you already selected
+stay unchanged.
+
+Setup defaults to automatic selection. Enter one **Instances per game** number,
+which defaults to 1 and applies to every game represented by the selected files
+and their matching servers. Every automatically selected instance must pass
+every requirement in the plan.
+
+When you choose **Start beta**, Electric Cherry chooses compatible servers with
+the fewest current players, while ensuring that every file is tested. Counts
+are exact: with T5 and T6 files selected, entering 2 selects two T5 and two T6
+participants, and all four must pass. Server choices are made at launch, so
+review shows the count for each game rather than a provisional list of server
+names.
+
+Automatic selection requires fresh player observations and available running
+servers. Servers held by another active beta or with pending deployment work
+are excluded, as are instances on unavailable, draining, or disabled hosts.
+Unknown or stale player counts are not treated as empty. If the
+requested counts cannot be met, including coverage of every selected map-scoped
+file, the beta does not start. File deployment rebuilds servers after waiting
+for confirmed empty lobbies and rebuild capacity.
+
+Open **Advanced options → Choose specific servers** for manual selection. The
+picker includes servers matching any file in the bundle; each file must have a
+matching selected server. Manual selection retains the configurable coverage
+rules below. Drafts preserve the selection mode and shared instance count, and
+older drafts with explicit server choices resume in manual mode.
+
+## Game reporter protocol
+
+Set `EC_TELEMETRY_TOKEN` on the Electric Cherry controller to a dedicated random
+secret of at least 32 characters (for example, generate 64 hexadecimal characters
+with `openssl rand -hex 32`). Configure the same secret in each game reporter or
+its trusted local relay. Use a production HTTPS URL or a reverse proxy that
+terminates TLS; browser login cookies do not authorize game telemetry.
+
+Send JSON to `POST /api/telemetry` with `Content-Type: application/json` and either
+`Authorization: Bearer <EC_TELEMETRY_TOKEN>` or
+`X-EC-Telemetry-Token: <EC_TELEMETRY_TOKEN>`. If both headers are present, they must
+contain the same secret. Keep the secret out of URLs, payloads and browser code,
+and use a different secret from host-agent credentials or account passwords.
+Request bodies are limited to 1 MiB (1,048,576 bytes), including requests without
+`Content-Length`. The limit applies to the UTF-8 JSON body before parsing.
+There are four events. Send them whether or not a beta is active.
+
+### Event fields
+
+| Event          | Fields                                                                                                                       |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| `game_start`   | `server`, `map`, `players` with each player's `name` and `guid`                                                              |
+| `round_change` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
+| `heartbeat`    | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats                |
+| `game_end`     | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
+
+`server` is the exact registered Electric Cherry server name. `map` is the
+current map. `players` is the full player array, including an empty array when
+there are no players. `server`, `map`, and each player's `name` and `guid` must be
+nonempty strings of at most 255 UTF-16 code units (JavaScript string length).
+Values over the limit reject the entire report; accepted values are preserved
+without trimming or truncation. Keep GUIDs as strings to preserve their precision
+and leading zeros. Round and stat values are nonnegative JSON integers.
+Each GUID must appear only once within a report's player array.
+
+Every event also accepts these optional top-level fields:
+
+| Field   | Meaning                                                                                             |
+| ------- | --------------------------------------------------------------------------------------------------- |
+| `match` | Reporter-generated game identity: a nonblank string of at most 64 UTF-16 code units.                |
+| `seq`   | Positive JSON safe integer, at most 9,007,199,254,740,991, identifying this report within the game. |
+
+Send both fields for reliable match boundaries, ordering, and retry handling.
+Choose a fresh `match` value for every game, including restarts on the same map,
+and keep it unchanged throughout that game. Match keys are scoped to the exact
+`server` name. Start `seq` at 1 and increment it once per event, including
+heartbeats. Retry a report with its original `match`, `seq`, and payload; do not
+allocate a new sequence number for a retry. Either field may be omitted for
+compatibility, but sequence-based deduplication requires both fields. Existing
+reporters without these fields remain valid.
+
+Every event also accepts an optional per-player `ping`: the game server's current
+latency measurement in milliseconds, as a nonnegative JSON integer (for example,
+`"ping": 42`). Send `null` or omit the field when unavailable; do not substitute
+zero for a missing measurement. Older reporters without this field remain valid.
+Fleet prefers telemetry ping and fills missing readings from fresh RCON status,
+matching players only by GUID. For T6, RCON's
+hexadecimal GUIDs are compared with telemetry's decimal GUIDs using exact integer
+conversion; stored GUIDs remain unchanged. T5 GUIDs are compared as reported.
+Occupied lobbies continue RCON polling every 30 seconds.
+RCON supplements only ping; telemetry still supplies roster membership and scores.
+RCON pings expire after 90 seconds and must match the current map and server
+lifecycle. Missing or ambiguous matches display `—`. Telemetry ping is retained
+and streamed with the latest report; it does not contribute to leaderboard statistics.
+
+### Game start
+
+Send when the game starts. Round and stats are not required.
+
+```json
+{
+    "event": "game_start",
+    "server": "t6_zombie_town_1",
+    "match": "town-20261012-001",
+    "seq": 1,
+    "map": "zm_transit",
+    "players": [
+        { "name": "Josh", "guid": "12345678901234567" },
+        { "name": "Alex", "guid": "23456789012345678" }
+    ]
+}
+```
+
+### Round change
+
+Send the new round and each player's current stats.
+
+```json
+{
+    "event": "round_change",
+    "server": "t6_zombie_town_1",
+    "match": "town-20261012-001",
+    "seq": 2,
+    "map": "zm_transit",
+    "round": 25,
+    "players": [
+        {
+            "name": "Josh",
+            "guid": "12345678901234567",
+            "kills": 420,
+            "headshots": 160,
+            "downs": 1,
+            "revives": 3,
+            "score": 32000
+        },
+        {
+            "name": "Alex",
+            "guid": "23456789012345678",
+            "kills": 380,
+            "headshots": 140,
+            "downs": 3,
+            "revives": 1,
+            "score": 28000
+        }
+    ]
+}
+```
+
+### Game end
+
+Send the final round and each player's final stats when the game ends.
+
+```json
+{
+    "event": "game_end",
+    "server": "t6_zombie_town_1",
+    "match": "town-20261012-001",
+    "seq": 3,
+    "map": "zm_transit",
+    "round": 26,
+    "players": [
+        {
+            "name": "Josh",
+            "guid": "12345678901234567",
+            "kills": 445,
+            "headshots": 170,
+            "downs": 2,
+            "revives": 3,
+            "score": 33500
+        },
+        {
+            "name": "Alex",
+            "guid": "23456789012345678",
+            "kills": 401,
+            "headshots": 146,
+            "downs": 4,
+            "revives": 1,
+            "score": 29750
+        }
+    ]
+}
+```
+
+### Responses and delivery
+
+Success returns HTTP 200 with `{ "ok": true, "telemetry": { ... } }`.
+The returned telemetry contains the event and an Electric Cherry `receivedAt`
+timestamp. A missing/invalid machine token returns 401; an unset or invalid
+controller `EC_TELEMETRY_TOKEN` returns 503. Authentication happens before payload
+validation. Authenticated bodies over 1 MiB return 413 before JSON parsing.
+Invalid JSON or fields return 400, an unknown/destroyed server 404, and a non-JSON
+content type 415.
+
+Send events in gameplay order directly to Electric Cherry. There is no discovery
+request, beta token, custom feature event, external helper, or separate beta
+acknowledgement. Basic payload checks happen in the receiver.
+
+With `match` and `seq`, repeated sequences are ignored even after an application
+restart. A lower sequence that arrives late can fill a missing-report gap without
+rolling back the current round, roster, or counters. A delayed `game_start` can
+record the starting roster without resetting the game's observed progression.
+A known older match key cannot replace a newer game's live state. When an unseen
+key starts a new game before the previous keyed game's `game_end` arrives,
+Matches labels the previous game **End report missing**. A later `game_end`
+completes its own match without changing the newer game. Missing sequence counts
+are diagnostic: they do not invent absent rounds, rosters, or an ending.
+Duplicate sequences, lower-sequence reports, and reports for known older keys do
+not advance beta requirements or replace the current scoreboard. A successful
+response acknowledges the submitted report; it does not mean that report became
+the current game state.
+
+Preserve delivery order across games as well: keep queued retries ahead of
+newer games' reports. An unseen key is always treated as the next game, even if
+its first report was delayed from an older game. `seq` cannot resolve this because
+it orders reports only within a game. A persistent per-server game counter or
+equivalent ordering metadata would require a protocol extension. See the
+[first-report ordering limitation](leaderboard.md#persistence-and-match-boundaries)
+for its effect on match history and the live scoreboard.
+
+Electric Cherry saves the latest accepted report, including player stats, and
+streams it to connected clients. A new start clears the previous round display;
+an end closes the match. Without a match key, legacy receipt-order and
+map/round heuristics remain in effect. Those reports cannot reliably distinguish
+delayed events from earlier games; send them once in order.
+
+Match timing is independent of player count: an empty `game_start` still records
+the start, and empty rosters or RCON polls do not erase it. Fleet can display a
+fresh reported round even when the start was missed; match duration stays unknown
+in that case. When match history identifies a new match from subsequent gameplay,
+Fleet clears the previous match's timing rather than carrying it into the new one.
+
+Reports also feed the persistent [Leaderboard](leaderboard.md), independently
+of beta activity. That page documents cumulative stat accounting, round credit,
+and match identity and ordering behavior.
+
+### Reporter deployment
+
+The game scripts are uploaded artifacts, not source maintained in this
+repository. Their reporter should emit the four payloads above and add the
+authentication header. Existing anonymous reporters must be updated before
+telemetry is accepted; if a reporter cannot set HTTP headers, configure a trusted
+host-local relay to add the header. This repository does not bundle that relay or
+assume a particular GSC HTTP implementation. Rotating the shared secret requires
+updating the controller environment and all reporters/relays together.
+
+### Telemetry diagnostics
+
+Open a server's **Diagnostics → Telemetry API requests** to inspect incoming
+`POST /api/telemetry` calls whose top-level `server` exactly matches its registered
+name. Each entry includes the Electric Cherry receipt time, HTTP result, error
+reason, event, map, round, player count, content type, and submitted JSON. Rejected
+reports are included, even when the content type is wrong but the body is readable
+JSON after authentication. Unauthorized requests are rejected before their bodies
+are read and are not retained in per-server diagnostics. Oversized bodies are
+rejected before parsing and are not retained either. Authentication headers
+and their secrets are never stored. Processing failures return 500 and record the internal error when diagnostic
+storage is available; failed database transactions do not update the scoreboard.
+
+The panel refreshes every five seconds while visible and has a manual refresh
+button. History is stored in SQLite starting with this version, with up to 200
+requests per server from the last seven days and a fleet-wide cap of 10,000
+requests. Old entries are pruned when new requests arrive, and expired entries
+are excluded from reads. Payload previews are limited to 16,384 characters and
+marked when truncated. Request headers other than content type are not stored.
+`GET /api/servers/{name}/telemetry-requests` returns the same recent history,
+newest first, with caching disabled.
+
+- **No entries:** no attributable requests are retained for this exact name.
+  Check the reporter's destination and server field. Calls that never reach
+  Electric Cherry, malformed JSON, and bodies without a usable server name
+  cannot appear in this server's log. Earlier requests are not backfilled.
+  Missing/invalid telemetry credentials and oversized bodies also produce no
+  retained request; inspect the reporter's HTTP result for 401, 413, or an
+  unconfigured-controller 503.
+- **400 / 415:** inspect the error and payload. One invalid player's stats or a
+  duplicate GUID rejects the entire report. These requests do not update the
+  scoreboard.
+- **404:** the reported server was not registered or had been destroyed at the
+  time of the request.
+- **200:** Electric Cherry processed the report. An empty `players` array
+  produces an empty scoreboard, including when sent with `game_end`. If a
+  populated accepted report is missing from the scoreboard, investigate the
+  live update/browser path.
+- **500:** Electric Cherry failed to process the request. Inspect the recorded
+  error and the application log.
+
+RCON's online status and player count are separate from gameplay telemetry, so
+an occupied server can still have no scoreboard report.
+
+## Requirements
+
+Requirements on a qualifying server are ANDed:
+
+| Requirement                 | Evidence                                                                                         |
+| --------------------------- | ------------------------------------------------------------------------------------------------ |
+| Reach round X               | A round change, heartbeat, or game end reports round >= X with the required player count         |
+| Complete qualifying matches | Game end at round 2 or later, optionally qualified by a higher round, player count, and duration |
+| Populated minutes           | Time between consecutive game reports with enough players at both ends                           |
+| Player-hours                | Populated minutes multiplied by the lower reported player count, divided by 60                   |
+| Reach player count          | Player array length in a game report                                                             |
+| Sustained population        | Consecutive report intervals above the player threshold within one game                          |
+| Continuous healthy minutes  | Consecutive fresh container/RCON observations                                                    |
+| Maximum health outages      | Transitions from observed healthy to unhealthy                                                   |
+| Successful recovery cycles  | Repeat the other requirements after each additional empty-server restart                         |
+| Tester checklist            | Tester name and observation entered on the beta detail page                                      |
+
+All gameplay requirements use the script reports. Match counts do not read game
+logs. The reported round and required player count must occur together in the
+same game; observations from different games cannot combine to qualify a match.
+The final game-end report can supply that qualifying round. Consecutive duplicate
+game ends do not add another completion.
+
+Match duration requires a game start and uses receipt times. Population time
+advances only when another report arrives in the same game; it uses the lower
+player count at the interval's endpoints. There is no heartbeat requirement or
+short interval limit between rounds. These are estimates between reported
+snapshots, not continuous measurements of player presence.
+
+Health checks and safe restart gates use the existing container/RCON monitoring.
+Health intervals spanning more than 60 seconds earn no healthy time.
+
+## Coverage, waves and outcomes
+
+Automatic selection requires all requested instances to pass. With manual server
+selection, coverage can require all selected servers, a quorum, one per selected map,
+one per selected game, or one per selected game/map combination. Coverage
+metadata is pinned when the beta opens. Every bundled file also needs at least
+one passing server that received it; a quorum cannot skip testing a game's file.
+
+Set wave size to 1 for a canary, a larger number for batches, or 0 for all
+selected servers. Later waves receive files after the previous waves pass.
+Health-outage budget violations on any tested server block promotion.
+
+The test window starts when the beta opens or a revision is submitted, including
+time waiting for empty servers. Missing evidence at the deadline produces **inconclusive**.
+Health monitoring continues after a pass until promotion.
+
+### Failure and rollback
+
+For failures, select hold as failed, pause for investigation, or automatic
+rollback. Rollback queues restoration only for files touched by the beta.
+Restoration rebuilds the server after waiting for an empty lobby and rebuild capacity.
+The manual option holds those changes in a rollback campaign until you select
+**Resume when empty**. Promotion is always manual.
+
+### Promotion
+
+Review the promotion preview before confirming. API clients fetch it with
+`GET /api/betas/:id/promote` and send its `previewToken` to the same route with
+`POST`. If the reviewed scope or configuration has changed, the request returns
+409 and requires a fresh preview.
+
+Confirming promotion creates a campaign without reading the existing files on
+additional servers. Each server waits for an empty lobby and rebuild capacity,
+then captures rollback snapshots before rebuilding with its queued files.
+Campaign history records snapshot capture, and file comparisons
+become available once those snapshots are confirmed. Tested servers retain their
+original pre-beta snapshots; their composed configuration is still checked at
+promotion to determine whether another activation is needed.
+
+Withdrawing changes that have not activated needs no snapshot and leaves those
+servers' files untouched. If activation was submitted but its snapshot has not
+been confirmed, reconcile the provisioning operation before rolling back.
+
+### Pause, resume and retry
+
+Pause holds evaluation and wave advancement. Resume retains completed progress and the
+current revision's deadline, but starts new timing intervals. A fresh attempt resets
+progress and restarts deployed servers when empty. Recovery cycles repeat
+gameplay milestones and checklist sign-offs while carrying health outages forward.
+
+Plans, progress, and evidence persist in SQLite. The detail page shows the most
+recent 200 evidence entries. On upgrading from the old reporter protocol,
+active progress resets. Unsupported checks in existing plans and saved presets
+become manual review items; they cannot silently pass.
+
+## Updating an active beta
+
+Choose **Update beta** on the beta detail page, upload replacements for existing
+bundle files, and review the changes and selected servers before choosing
+**Deploy revision**. Text files show a diff when both files are small enough;
+binary and large files show their checksums. Files must actually change.
+
+The beta keeps its ID, servers, scope and test policy. Each revision creates new
+file versions, clears current test progress and checklist sign-offs, repeats the
+configured waves, and starts a fresh testing window at submission. Servers rebuild
+with the requested files through the normal empty-server and capacity gates.
+Promotion requires the new revision to pass. Pause and retry retain the current revision's
+deadline; they do not create a revision.
+
+Revision history retains downloadable bundle files and previous test results,
+including the last 200 evidence entries shown at replacement. All underlying
+attempts and evidence remain in SQLite, separated by revision. Rejecting the beta
+restores the original pre-beta files, even after multiple revisions or while a
+new revision is waiting to deploy. Files originally absent are removed.
+
+Updates are available until promotion or rollback, including paused, failed and
+expired tests. An activation already in progress must settle first; interrupted
+file writes and deleted participants must be resolved before an update. Pending,
+unsubmitted activations are superseded. Adding/removing files, changing targets or
+changing the policy still requires a new beta.
+
+## API and storage
+
+`POST /api/betas` accepts `scripts`, `testPlan`, and either
+`instancesPerGame` (for example, `{ "t5": 2, "t6": 2 }`) for automatic selection
+or `servers` with explicit registered server names for manual selection. Submit
+only positive integer counts in `instancesPerGame`. The setup UI sends the same
+shared count for every applicable game; API clients can still supply different
+counts per game. Automatic selection sets the plan's coverage `scope` to `all`.
+Do not send both selection forms. File scopes determine which bundle files each
+participant receives in both modes.
+`GET /api/betas/:id` includes the plan, outcome, per-server progress, and
+recent evidence under `test`.
+
+`POST /api/betas/:id/test` accepts `pause`, `resume`, `retry`, or
+`sign-off` (with `server`, `requirementId`, and `note`).
+`GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are
+copied into a beta; later edits do not change an active plan.
+
+`POST /api/betas/:id/revisions` accepts multipart `expectedRevision`, optional
+`notes`, and replacement uploads named `file:<scriptId>`. Each script replacement
+also requires `release:<scriptId>` in `MAJOR.MINOR.PATCH` form. The update dialog
+accepts manual versions or `versions.json` to fill empty fields by exact filename.
+A changed script cannot reuse a previous release version. Missing or invalid
+versions reject the entire update without modifying beta history or deployments.
+Plugin release tags remain optional. See [script release versions](getting-started.md#script-release-versions).
+A stale revision is
+rejected. `GET /api/betas/:id` includes the current `revision` and `revisions`
+history; `GET /api/betas/:id/revisions/:revision/files/:scriptId` downloads an
+archived file (or the original pinned file before its first update).
+
+Internal progress records support attempts and recovery cycles. The reporter
+does not know about or supply these records. Legacy betas without a test plan
+also use `game_end` at round 2 or later for their match-completion gate.

docs/guides/collections.md

@@ -0,0 +1,381 @@
+# Deployment collections
+
+Open **Fleet → Collections** to manage reusable server lineups. Use
+**Active launches** to follow progress and **Launch history** to review completed
+launches, cancellations, and failures. Both views refresh automatically. A
+collection describes a whole lineup: for example, two Kino lobbies, one Moon lobby,
+and two Town lobbies. Counts are desired fleet totals: existing and queued
+servers count toward each target, and only missing servers are created.
+Collections save game/map selections and lobby counts. Every new server uses the
+latest promoted scripts, plugins, baseline, and matching configuration overlays
+when provisioning begins.
+
+## How lineup counts work
+
+For example, a collection targeting four Kino and two Der Riese lobbies, with
+two Kino already registered, creates two Kino and two Der Riese lobbies.
+The preview groups maps by game and shows **Target / Existing or queued /
+Create** for each map, with **Remove** in exact mode. Existing totals in exact
+mode include the listed removals, including those already awaiting cleanup.
+Matching is by game and exact map variant; T6 Town and Farm are distinct even
+though they share an engine map. Provisioning identities are counted once when
+they appear in both the server registry and active creation queue.
+
+Registered servers count even if stopped or unhealthy; conforming does not
+restart or reconfigure them. Failed creation jobs without a registered server,
+destroyed records, and servers awaiting removal do not count. The default
+**Fill missing servers** mode keeps excess servers and maps outside the collection.
+It fills deficits without changing existing servers' files.
+
+## Build and launch a collection
+
+1. Choose **Create collection** on **Fleet → Collections** and name the lineup.
+2. Add maps from Black Ops I or II and set the desired total lobbies for each.
+3. Choose **Save & preview** (or **Preview fleet changes** on a saved collection).
+   The dialog prepares its preview automatically. Both **Fill missing**
+   and **Match exactly** are visible; selecting a mode rechecks the
+   plan. Review the change summary and the map table grouped by game.
+4. Check the required key coverage for each game. Available keys count once
+   across the full lineup, and every new server needs a key. If a game is short,
+   open **Admin → Keys** to add keys or wait for assigned keys to be
+   released. Returning to the dialog rechecks the pools and fleet.
+5. Review the capacity summary. Expand its details for CPU, memory, ports, and
+   startup slots. Capacity estimates are advisory: queued launches can wait for
+   hosts or resources. Native host ports are assigned during provisioning.
+6. Expand **Server details** to inspect server IDs, launch order, retained servers,
+   and existing-server health. **Advanced settings** holds the runtime override;
+   **Setup details** shows configuration, managed files, and game-file upload
+   information.
+7. Use **Queue N servers** in the persistent footer, or review and confirm
+   the removal list before choosing **Create N · Remove M** when removals are
+   needed. Follow the collection progress page after submission. If no changes are needed,
+   choose **Done**; no operation is created.
+
+### Match collection exactly
+
+Exact mode applies to the entire registered and queued fleet across both games.
+The preview lists servers to keep, create, and remove. It keeps deployed lobbies
+before unstarted creations, then the lowest lobby numbers within each game/map
+variant. It removes excess lobbies and maps outside the collection. Retained
+servers keep their containers, files, ports, and configuration without a restart.
+The option is chosen per launch; opening another preview defaults to fill mode.
+
+The removal list stays visible in exact mode. Review it and check its confirmation
+before applying. Every refreshed preview clears that confirmation so it must be
+reviewed again. Removed servers lose their containers, files, and registrations;
+their keys return to the game pool only after confirmed removal.
+Unstarted creations are cancelled without booting a container. Running servers
+use the normal removal queue and require a fresh zero-player RCON response.
+Occupied or unreachable servers wait. Already pending removals are reused.
+
+Creation intents and removals are queued in one database transaction. A
+missing key or changed fleet rejects the entire request. Any fleet identity,
+map, port, or pending-removal change since an exact preview requires a new review.
+Servers added after submission are not included. Applying the same preview again
+returns the same collection without repeating operations.
+
+Keys assigned to pending removals stay reserved until cleanup is confirmed. An
+exact collection that replaces running servers still needs enough currently
+available game keys for every new launch. If the pool is full, complete the
+removals first, then refresh the preview to use the released keys.
+
+Native launches can be queued even when the current port range is full. Removed
+servers retain their ports until cleanup completes; provisioning assigns newly
+available ports as each creation starts. A failed or cancelled removal can leave
+creations waiting for capacity, while other eligible hosts can still be used.
+Legacy launches retain their reviewed port dependencies. Queued work and any
+assigned reservations survive app restarts. The progress page
+tracks creations and removals, including removal-only plans, and lists the servers
+that were retained. The existing operation detail page offers removal cancellation
+and cleanup recovery where applicable.
+
+### Collection limits and editing
+
+Collections support up to 40 distinct game/map selections, 1–15 lobbies per map,
+and 100 servers total. The host's configured port range and existing fleet may
+allow fewer. A collection can be saved even when there is insufficient capacity
+to launch it. Saving a collection does not capture or pin managed file versions.
+
+Use **Edit** on a collection card to change its name, description, maps, or lobby
+counts. Already reviewed launches keep their original lineup, including after
+collection edits or removal; their managed files still follow the latest promotions.
+Previously saved collections automatically use this policy without being resaved.
+Collections saved by the earlier single-server release load as one-server lineups
+and can be expanded directly. Any file snapshots in older collections are ignored
+when preparing new launches.
+
+### Preview and submission
+
+A preview runs automatically when the dialog opens, the mode or launch settings
+change, the user returns from managing keys, or the current preview expires.
+Refreshing also resets removal confirmation. The preview lasts 30 minutes;
+checks and displayed counts describe the fleet when it was prepared.
+
+A native collection preview reads a shared capacity snapshot, maintained from host
+heartbeats and fleet changes. It estimates available ports, CPU, memory, and startup
+slots without running placement for each server. Missing or insufficient capacity
+is a warning: provisioning can wait for it. The preview does not reserve resources.
+
+Apply validates the reviewed configuration, lobby IDs, keys, and fleet changes, saves all jobs in
+one database transaction, and returns the progress page ID. Native jobs initially
+have no host or port; each job performs its capacity checks asynchronously during
+provisioning. If a server lacks an available game key or its reviewed identity changed,
+no jobs or key reservations from that submission are committed. Legacy submissions
+also revalidate their reviewed ports. Refreshing the preview rechecks pool
+availability and resets removal confirmation.
+
+If a submission response is lost or cannot be confirmed, retry that same
+submission. The dialog keeps the reviewed preview while the outcome is uncertain
+instead of automatically replacing it. If the original request committed,
+submitting the same preview again returns the same collection, including after
+its original expiry or an application restart, without repeating operations.
+
+Reusing the collection with a new preview recomputes the shortfall, including
+servers queued by earlier launches. It does not add another full copy of the
+lineup. If the shortfall or reviewed identities change before submission, the request is rejected without changes and
+requires a fresh preview. Previously saved additive previews are checked against
+the current shortfall too.
+
+### Startup queue and progress
+
+All collections and individual launches share one startup queue, ordered by request
+time. By default, one server starts at a time. Its startup slot remains occupied
+until the host agent confirms container health, game status and the selected map
+are verified, and RCON reports a current Plutonium revision.
+Only then can the next queued server start. Set **Concurrent server startups** in **Admin → Settings** to
+a positive integer to allow more simultaneous startups. Each worker pass admits
+at most the available slots, even if an attempt fails immediately.
+
+Names, any assigned ports, and queue order survive application restarts. A server still booting
+or waiting for health keeps its slot until it passes verification or is removed.
+A lost startup response also keeps the slot because the container may already be
+running. A confirmed creation failure is recorded for review, allowing later
+queued servers to proceed on subsequent passes without undoing successful launches.
+
+The collection launch page at `/collections/launches/:id` separates ready,
+launching, queued, failed, and cancelled servers. Its breadcrumb returns to Collections. It
+highlights the current startup, shows each waiting server's queue position, and
+links to earlier launches when another collection occupies the available capacity.
+Every server links to its Provisioning operation timeline and recovery options,
+and each child operation links back to its collection launch. Individual operations
+remain visible in Provisioning activity and history.
+
+### Cancel a collection launch
+
+Operators and administrators can choose **Cancel collection launch** on the
+collection progress page while eligible work remains. Review and confirm the
+cleanup before proceeding. Cancellation stops unfinished creations, including
+eligible failed or interrupted launches, and removes their containers and instance
+files. This can disconnect players who joined before startup verification finished,
+even when player status is unknown. Shared game files are kept. Keys and ports
+remain reserved until removal is confirmed.
+
+Completed launches, including those that finished with a warning, and retained
+servers are kept. In exact mode, safe pending removals belonging to this launch
+are cancelled. Removals that have already started or are still needed by another
+collection may finish; cancellation does not restore servers already removed.
+
+The progress page shows **Cancellation requested** while work or cleanup remains,
+then **Collection launch cancelled**, or **Cancellation needs attention** if
+operations or cleanup failed. Cancelled operations are counted separately
+from failures. Each cancelled creation links to its cleanup operation for progress
+and recovery. A launch stays in **Active launches** until its work and cleanup
+have settled; failed cleanup remains marked as needing attention in launch history.
+Cancellation does not delete the saved collection, so it can be previewed again
+against the remaining fleet.
+
+## Create a single server
+
+1. Open **Create server** from the fleet or **Single server** from Provisioning.
+2. Choose a game and map. The dialog automatically checks the current promoted
+   setup, latest game files, available lobby, and game key pool as selections change.
+3. Review the readiness summary and resolve any displayed blockers. When no game
+   key is available, open **Admin → Keys** and add one or wait for a key
+   to be released. Returning to the launch dialog rechecks availability; **Retry checks**
+   also runs the checks again.
+4. If needed, expand **Advanced settings** for runtime and network overrides.
+   Expand **Setup details** to inspect the image, configuration, managed files,
+   game-file upload time, and internal server ID.
+5. Choose **Launch server** in the persistent footer and follow the operation to
+   verified health. **Cancel** closes the dialog without creating a server.
+
+The single-server dialog uses the current promoted setup. Create and launch
+collections separately from **Fleet → Collections**.
+
+Automatic checks reserve neither capacity nor a key. The reviewed preview lasts
+30 minutes. If another launch takes the lobby or key, submission stops; retry the
+checks before launching. Pool availability is checked again at submission, and
+keys are reserved atomically so concurrent launches cannot share a credential.
+
+Host selection, game-file versions, runtime configuration and lobby availability
+are checked before queueing. Provisioning assigns host capacity and physical
+ports when dispatching. The host agent performs container and RCON verification
+locally after launch. See [native hosts](native-hosts.md) for readiness checks.
+T6 variants retain their exact catalog slug and launch ruleset; RCON health checks
+verify the engine map, not the variant's start location.
+
+## Save and reuse a collection
+
+Choose **Create collection** under **Fleet → Collections**, give it a name and
+optional description, and add maps with their desired lobby counts. A collection
+can contain a single map with a target of one lobby. You can save it even when
+host configuration or available capacity blocks launching. Names are unique
+without regard to case.
+
+Each map in a collection retains the game, exact map variant, and desired lobby
+count. Image references, managed files, and composed configuration are resolved
+from the current setup for each new launch. Credentials, lobby IDs, and ports are
+not copied from the source server.
+
+Use **Conform fleet** on a collection card to review missing servers. Remove a
+collection from the library when it is no longer needed; its existing servers,
+reviewed previews, and deployment history remain intact.
+
+On a creation detail page, **Launch another server** opens the single-server
+dialog with the original game and map selected. You can change either selection
+before launching. Checks use the current promoted setup and latest game files,
+and provisioning assigns a fresh lobby and port. The recorded files remain
+visible as deployment history. Older deployments without a recorded recipe
+retain their history but cannot offer this action.
+
+## Promotions and deployment history
+
+A preview shows the currently promoted managed files, configuration, and image
+reference for the selected map. It does not pin those versions. New launches
+resolve the promoted setup again when provisioning begins, so
+promotions made after a preview or while a job is queued apply to that server.
+This applies to collections, single-server launches, and **Launch another server**.
+
+Previews and queued jobs store file metadata without copying script or plugin
+contents for every lobby. Provisioning captures the bytes when each job starts.
+Native host ranking, file reads, and command encoding run in a worker thread.
+The controller makes a short atomic reservation check after ranking, then
+rechecks capacity, promotions, credentials, and safety
+before submitting those commands. Agent delivery streams the stored payloads in
+small chunks, while progress polling reads metadata without loading file bytes.
+
+The files selected at provisioning, their hashes and version metadata, composed
+configuration, selected map, and image reference are recorded in SQLite for
+deployment history. These records do not override promotions on future launches.
+Existing API clients that omit a preview and jobs queued before this update
+also use the promoted setup at provisioning time.
+
+A submitted preview is bound to one deployment: retrying the same submission
+returns the original operation, including after an application restart or a lost
+HTTP response. The reviewed game/map, lobby ID, and key checks are preserved;
+native ports are reserved only during provisioning.
+Already running servers continue to receive updates through campaigns, betas,
+and drift reconciliation using the server's game/map scopes.
+Container tags can resolve to newer images; use a digest for a fixed image.
+Without a promoted baseline, Plutainer seeds its own defaults on first boot,
+which are not captured. Host, network, game-file volumes, destination directories,
+and RCON/Plutonium credentials use the environment and assigned game-pool key at
+launch time.
+The current main config filename and launch settings apply to new deployments.
+
+Recipes do not include the separately stored Plutonium or integration credentials.
+Managed files can themselves contain sensitive configuration, so protect database
+backups as you would the existing artifact library and saved-key database. Public
+recipe responses contain file metadata and hashes, not file contents.
+
+## Progress and interrupted operations
+
+Creation verifies container health, then RCON, then the selected engine map.
+The detail page explains the current condition, displays elapsed time, and keeps
+its activity history. A completed container start is not reported as a completed
+server launch until the health checks pass.
+
+For failed or interrupted creations, **Resume health verification** appears only
+when the original server is still registered on its assigned port and no later
+lifecycle operation has targeted that ID. Native commands must have completed
+before readiness verification can resume. If the container is unhealthy, the
+operation may continue its remaining automatic restart and rebuild recovery steps.
+
+When the host conclusively reports that the initial creation command failed,
+**Retry with rebuild** starts a new provisioning operation for the same lobby and
+port. It deletes the container and instance files and uses the latest promoted
+setup. Normal player safety, capacity, and protected work checks still apply.
+The original failure stays in history and links to the recovery operation.
+This retry is unavailable for unknown command results, replacement instances,
+cancelled creation cleanup, or a later operation on the same server.
+
+After an application restart, the worker reconciles the original host command
+before continuing eligible interrupted startups. Unknown command results retain
+their original command ID and occupied slots until the host confirms the outcome.
+Missing server registrations and later destruction attempts require inspection.
+
+### Recovering a completed rebuild generation
+
+A completed rebuild records the replacement generation before EC verifies health.
+If fresh host inventory disagrees, verification pauses with the controller and
+host generations in the operation's recovery message.
+
+For an existing mismatch, open the **completed rebuild's Provisioning detail**
+(even if that operation was later cancelled), then choose **Reconcile replacement
+generation**. EC requires a completed command for the original host and immutable
+instance ID, the command's requested replacement generation, and matching host
+inventory observed within the last 30 seconds. Wait for that host to reconnect
+and report fresh inventory if the action is unavailable. A server name alone is
+not sufficient evidence.
+
+Reconciliation preserves cancellation and submitted commands. Use **Resume health
+verification** when offered, or request a new lifecycle operation after resolving
+any remaining active operation. Fresh health checks, log reads, and new lifecycle
+commands use the replacement generation. EC requires fresh container and game
+evidence before reporting the server online. The agent continues to reject stale
+commands; an admin player-safety override does not bypass generation checks.
+
+## API
+
+Collections retain the existing `/api/blueprints` endpoints and `blueprintId` /
+`blueprintName` fields for compatibility. Saved data needs no migration. Old
+`/provisioning?view=blueprints` links redirect to `/collections`.
+Collection launch details live at `/collections/launches/:id`; old
+`/provisioning/collections/:id` and `/deployments/collections/:id` links redirect
+there while preserving query parameters.
+
+- `GET /api/blueprints`: list lineup summaries with game/map selections and counts.
+- `POST /api/blueprints`: save `{ name, description?, previewId }` or
+  `{ name, description?, deploymentId }` for a single-map target of one lobby, or
+  `{ name, description?, entries: [{ game, mapSlug, count }] }` for a collection.
+- `PUT /api/blueprints/:id`: edit `{ name, description?, entries }`. All entries
+  use the latest promoted setup when launched.
+- `DELETE /api/blueprints/:id`: remove a library entry.
+- `POST /api/deployment-collections/preview`: preview `{ blueprintId, mode? }`,
+  where mode is `fill` (default) or `exact`. Returns `mode`, `retainedServers`,
+  `removals` (name/reason), creation `servers`, and configured `launchConcurrency`.
+- `POST /api/deployment-collections`: atomically queue `{ previewId, keys? }`,
+  where `keys` maps preview server IDs to optional unused keys added to their
+  game's pool and assigned during launch. Omit entries covered by available
+  pool keys; these IDs do not permanently bind a key to a map or lobby.
+  A satisfied preview returns `{ ok: true, collectionId: null }` with HTTP 200,
+  without creating a collection, saving keys, or kicking the worker.
+- `GET /api/deployment-collections`: list collection progress summaries with
+  separate `queued`, `provisioning`, `verifying`, and `cancelled` counts,
+  `cleanupActive` and `cleanupFailed` counts, and `cancelled_at` when cancellation
+  was requested. Cancelled operations are excluded from `failed`.
+- `GET /api/deployment-collections/:id`: collection mode, retained server IDs,
+  per-operation progress with `action: create | destroy`,
+  each waiting server's global `queuePosition`, and the shared queue's concurrency,
+  active startups, and queued count. Includes `canCancel` and each creation's
+  `cancellation` cleanup ID, state, stage, and error when applicable.
+- `POST /api/deployment-collections/:id/cancel`: request cancellation and cleanup
+  of eligible unfinished work. Returns `{ ok: true }`, HTTP 404 for an unknown
+  launch, or HTTP 409 when the launch cannot be cancelled.
+- `POST /api/deployments/preview`: preview `{ game, mapSlug }` or `{ deploymentId }`
+  for a new single server using the latest promoted setup. All collections, including one-server
+  targets, must use the collection endpoint so existing servers count correctly.
+  Returns an ID, expiry, recipe summary, capacity, key
+  availability (never its value), and prerequisite checks.
+- `POST /api/servers`: existing request plus optional `previewId`; the game and
+  map must match that preview. `expectedName` remains supported.
+- `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
+  `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations.
+- `POST /api/deployments/:id/resume`: resume eligible creation health checks.
+- `POST /api/provisioning/:id/reconcile-generation`: reconcile a completed rebuild
+  using fresh inventory from its original host. Returns 409 when identity or
+  completion evidence cannot be validated; repeated application is safe.
+- `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible
+  failed creation and link it from the original operation.
+
+Collection launches begin processing immediately after all requests are queued.

docs/guides/discord.md

@@ -0,0 +1,173 @@
+# Discord companion
+
+Electric Cherry can publish live Black Ops I and Black Ops II server boards in
+Discord, answer commands privately, and send requested map availability alerts by
+DM. Configure it under **Admin → Discord**. Administrators can make changes;
+Super Viewers can inspect configuration, the saved preview, and delivery health.
+The integration is disabled by default. Once enabled, it publishes all current
+and newly created game servers unless an administrator excludes them.
+
+## Install the application
+
+1. Create an application in the [Discord Developer Portal](https://discord.com/developers/applications).
+   Copy its Application ID and Public Key, and create a bot token.
+2. Set the controller environment variables below. Keep the token private; the
+   settings page reports whether it is configured without displaying its value.
+   With Docker Compose, put the values in the deployment's `.env` file and recreate
+   the controller to apply them. Do not commit that file.
+
+    ```dotenv
+    EC_DISCORD_APPLICATION_ID=
+    EC_DISCORD_PUBLIC_KEY=
+    EC_DISCORD_BOT_TOKEN=
+    ```
+
+3. Install the application into your Discord server using the `bot` and
+   `applications.commands` scopes. Grant **View Channel**, **Send Messages**, and
+   **Read Message History** in each board channel. No Administrator permission or
+   privileged Gateway intents are needed. Channel overrides must also permit the
+   bot to read and write those channels.
+4. In the Developer Portal, set **Interactions Endpoint URL** to
+   `https://your-controller.example/api/discord/interactions`. Use your actual
+   public controller origin with a trusted HTTPS certificate. Discord verifies
+   the endpoint before accepting it. The controller checks Discord's request
+   signatures; this endpoint does not use the Electric Cherry login session.
+5. In Discord, enable Developer Mode, then copy the Discord server ID and the
+   board channel IDs. Open **Admin → Discord**, enter those IDs, and uncheck any
+   game servers you want to exclude. Either game may omit its board channel.
+6. Review the information-sharing switches, enable the companion, and save.
+   Select **Register commands** to install the commands in the saved Discord
+   server. Saving configuration and registering commands are separate actions.
+
+The deployment's reverse proxy must route the interaction endpoint to the
+controller and preserve Discord's signature headers and request body. No Gateway
+connection is required. Installation scopes and channel permissions follow
+Discord's [OAuth2 documentation](https://docs.discord.com/developers/topics/oauth2)
+and [permissions reference](https://docs.discord.com/developers/topics/permissions).
+See [HTTP interactions](https://docs.discord.com/developers/interactions/receiving-and-responding)
+for Discord's endpoint verification and response requirements.
+
+Use a dedicated Discord application for this controller. The command registration
+action updates that application's command set in the configured Discord server.
+
+## Choose what is published
+
+All current and newly created game servers are included by default. Uncheck a
+server to exclude it from boards, private server browsing, server details, and
+availability watches. Exclusions are retained if a game server is removed, so a
+server with the same name remains excluded if it returns. Check an excluded
+server again to remove that exclusion.
+
+Player names in server details and public connection addresses are separate,
+disabled-by-default options. Connection instructions use an explicitly configured
+native host public endpoint; internal host addresses and RCON credentials are not
+published. Capacity, player counts, map, round, and observation state can appear
+without either switch enabled.
+
+**Publish fleet leaderboards** is a separate decision. It exposes recorded player
+names and statistics from the whole fleet, including historical matches on
+excluded servers. Leave it disabled if those
+statistics should remain available only to Electric Cherry users.
+
+The settings page's preview uses the **saved settings**. Pending form edits do
+not change the preview or Discord. The preview remains available while the
+integration is disabled.
+
+## Boards and browsing
+
+Each configured game channel receives a persistent server board, split into
+additional messages when needed. The controller revisits delivery about every
+30 seconds and edits changed content. Stored message IDs survive controller
+restarts. Periodic existence checks recreate messages that were removed.
+
+Boards show observed occupancy, capacity and current round where available,
+with separate empty, in-progress, full, starting, offline, not-responding, and
+unknown states. Freshness is part of the observation. **Empty slots do not prove
+that a match accepts new players**; this release does not label games joinable.
+Capacity defaults to four players, matching the fleet UI, until a fresh reported
+capacity overrides it. A default capacity does not make stale player counts
+current or allow a stale observation to trigger an availability alert.
+
+Members can use the board controls or slash commands. Browsing, filters, details,
+leaderboards, and command confirmations are visible only to the member who
+requested them. They do not rearrange the shared board.
+
+| Command                         | Purpose                                                                                    |
+| ------------------------------- | ------------------------------------------------------------------------------------------ |
+| `/servers`                      | Browse published servers; filter by game, map and state, and page through results.         |
+| `/server name`                  | Inspect one published server, including optional roster names and connection instructions. |
+| `/leaderboard`                  | Browse recorded fleet rankings by game, map, metric, squad size and page, when enabled.    |
+| `/watch add game map condition` | Request a one-shot DM when a map has an empty lobby or observed space.                     |
+| `/watch list`                   | See your active watches and their IDs.                                                     |
+| `/watch cancel id`              | Cancel one of your own watches.                                                            |
+| `/help`                         | Explain the commands and their limits.                                                     |
+
+Commands are limited to the configured Discord server. Map options preserve the
+game's map variants. Leaderboard results reflect recorded gameplay, not a new
+claim that a match was won or that a player survived continuously.
+
+## Availability alerts
+
+A watch lasts up to 24 hours and sends one DM when its condition is satisfied.
+Each member may have up to ten active watches. Requesting the same map and
+condition again reuses the existing watch. Members can cancel only their own
+watches.
+
+An **empty** watch needs an observed empty lobby. A **space** watch needs fewer
+players than the current capacity, using the four-player default when no fresh
+capacity override is available. Neither condition promises that
+joining is allowed. Unknown, stale, and offline observations never satisfy a
+watch. The worker requires two qualifying observations at least 15 seconds apart
+before queueing a notification.
+
+Watches and pending notifications persist across controller restarts. Delivery
+handles transient failures and Discord rate limits. A permanent DM failure, such
+as a member blocking messages from the bot, ends that watch. Members should allow
+DMs from the bot before creating a watch and can inspect active watches with
+`/watch list`.
+
+## Delivery and troubleshooting
+
+The settings page shows the last attempt, last success, current error, active
+watches, pending notifications, and links to recorded board channels. Refreshing
+that page preserves pending edits. A failure keeps the last loaded information
+visible.
+
+- **Credentials need attention:** check all three environment variables and
+  restart the controller after changing them. A readiness indicator checks local
+  configuration; it does not establish that Discord has accepted the token.
+- **Discord rejects the interaction endpoint:** verify HTTPS routing, the
+  application's Public Key, and that the reverse proxy preserves signed
+  requests. Discord must reach the endpoint without an interactive login.
+- **Buttons time out without a private loading message:** inspect the controller
+  logs while clicking once (`docker compose logs --since=5m electric-cherry`).
+  A `[discord] Deferring interaction` entry means the signed request reached the
+  handler and it prepared a private acknowledgement before database work. A
+  rejection entry identifies signature, body, or application-ID validation
+  failures. An application-ID mismatch also returns a private explanation;
+  `EC_DISCORD_APPLICATION_ID` must identify the same application as the Public
+  Key and bot token. If no entry appears, check the proxy and saved Interactions
+  Endpoint URL. A verified endpoint and working boards alone do not confirm
+  that button clicks are reaching the running controller.
+- **Commands are missing:** save the correct Discord server ID, install the bot
+  with `applications.commands`, then select **Register commands**. Register again
+  after upgrading to a release that changes the command schema.
+- **Board delivery is denied:** check the bot's permissions and channel overrides.
+  Authorization failures pause retries. After correcting access, select **Save
+  and retry** in Electric Cherry.
+- **A board was deleted:** the controller's periodic check recreates it. Keep
+  Read Message History enabled so those checks can verify existing messages.
+- **Discord rate limits or temporary outages:** delivery honors retry delays and
+  retries transient failures with increasing delays. Fleet operations continue
+  independently.
+- **Old board messages remain after a channel change or disable:** the controller
+  attempts to remove old messages. If permissions were removed first, an
+  administrator may need to delete those messages in Discord.
+
+Disabling the companion stops new publication and notifications. Cleanup may
+still remove previously recorded board messages. Data already seen or copied by
+members cannot be recalled by changing publication settings.
+
+This release does not link Discord accounts to game identities, expose personal
+bank or AFK accounts, mutate servers or moderation state, or announce
+achievements. Those workflows require separate authorization and implementation.

docs/guides/getting-started.md

@@ -0,0 +1,164 @@
+# Getting started
+
+Electric Cherry runs a web controller and an agent on each game host. The
+controller stores fleet metadata in SQLite and uploaded game files on disk.
+Agents run Docker/Plutainer locally and connect outbound to the controller over
+HTTPS. Running game containers continue through controller or agent outages.
+
+## Requirements
+
+- A controller machine with Docker Compose and an HTTPS reverse proxy.
+- Debian 12 or 13 amd64 game hosts with storage for game files and instances.
+- A signed agent release and its public key published in the trusted repository.
+- Outbound HTTPS access from game hosts to the controller and public repository.
+- Your complete T5 or T6 game folders, including the DLC you intend to host.
+- Plutonium server keys for each game, with one unused key per new concurrent lobby.
+
+The controller does not need access to a Docker socket, game-host mounts, or
+inbound RCON. Host agents handle local game commands and file operations.
+
+## Start the controller
+
+From the repository root:
+
+```sh
+cp .env.example .env
+openssl rand -hex 32
+```
+
+Set these values in `.env`:
+
+| Variable             | Purpose                                                                                                          |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| `EC_PUBLIC_URL`      | The external HTTPS address of the dashboard.                                                                     |
+| `EC_SETUP_TOKEN`     | The generated secret, used to create the first administrator.                                                    |
+| `EC_DATA_DIR`        | A persistent directory for the database, game files, and beta archives.                                          |
+| `EC_TELEMETRY_TOKEN` | A separate secret of at least 32 characters for game reporters. Generate it independently with the same command. |
+
+For an existing installation, retain its original data directory. Compose
+defaults to `/root/zombies/.electric-cherry`; choose a dedicated directory for a
+new controller. See [.env.example](../../.env.example) for the other settings.
+
+Before building for game hosts, obtain the publisher's signed `static/agent`
+artifacts and matching source revision, or follow
+[Publishing signed releases](native-hosts.md#publishing-signed-releases)
+for your own build. Normal builds preserve signatures for identical artifacts;
+source changes require a new offline signature. An unsigned controller can run,
+but hosts reject its agent installation and updates.
+
+```sh
+docker compose up -d --build
+```
+
+Proxy the HTTPS address to controller port 3000. Enable gzip or Brotli compression
+at the HTTPS reverse proxy for HTML (`text/html`) and JSON (`application/json`),
+including server-rendered pages, page data, and API responses. The build already
+precompresses static JavaScript and CSS assets; dynamically generated responses
+need compression at the proxy.
+
+Configure `ADDRESS_HEADER` and `XFF_DEPTH` for your trusted proxy so login limits
+use each client's address. Leave `ADDRESS_HEADER` empty for direct connections;
+see the [proxy configuration guide](authentication.md#proxy-and-local-development)
+before enabling forwarded addresses.
+
+Open the dashboard and follow `/setup` to create the first administrator. Then
+remove `EC_SETUP_TOKEN` from `.env` and run `docker compose up -d` to recreate the
+controller. Setup stays closed once an account exists.
+
+Add other users under **Admin → Users**. The
+[access guide](authentication.md) explains roles, proxy configuration, and
+password recovery.
+
+## Prepare a game host
+
+1. Open **Releases → Game files** and upload the complete local folder for T5 or
+   T6, including DLC. Uploads can run before any host is enrolled. The latest
+   verified upload becomes that game's default files.
+2. Ensure the release publisher has [published its public key and signed release](native-hosts.md#publishing-signed-releases).
+   Hosts fetch the key automatically during installation and before every update
+   check; there is no manual key-copy step.
+3. Open **Fleet → Hosts → Add Host**, configure the host's game-port range and
+   bind address, and run the generated enrollment command. It downloads the
+   installer from the fixed public repository URL before running it. Check that
+   the installer source is the known repository, independent of the controller.
+4. Wait for automatic setup to prepare the Plutainer runtime and available game
+   files. The host page reports download, verification, and readiness progress.
+
+The installer sets up the Python agent as a systemd service and enables automatic
+agent updates. It installs Docker if needed. Multiple instances share read-only
+base game files; each instance has its own writable data. See
+[Native hosts](native-hosts.md) for storage layout, network access, and recovery.
+Existing hosts using updater protocol 1 need a
+[trusted installer migration](native-hosts.md#automatic-agent-updates) before
+controller-independent release verification takes effect.
+
+## Launch your first server
+
+1. Save Plutonium server keys in the T5 or T6 pool under **Admin → Keys**.
+2. Open **Create server** and choose a game and map. Readiness checks run
+   automatically. For a reusable lineup, use **Fleet → Collections** instead.
+3. Resolve any launch blockers and choose **Launch server**. Provisioning queues
+   the work, then assigns a host and port when capacity is available.
+4. Follow the operation until readiness checks pass, then verify that players
+   can find and join the server.
+
+Keys are shared across maps within their game. A launch assigns one available key
+until the server is removed; stopped servers keep their keys. Optional key labels
+can record the hostname chosen in Plutonium without binding the key to a map or
+lobby. Electric Cherry does not change the hostname registered with Plutonium.
+
+New launches use the latest verified game files and promoted configuration,
+scripts, and plugins. Uploading newer game files does not restart existing
+servers. Disruptive operations normally wait for a fresh empty lobby or a
+verified stopped container. A rebuild removes the instance's writable files.
+See [Provisioning](provisioning.md) and [Collections](collections.md) for details.
+
+## Script release versions
+
+Every new script upload needs a release version in `MAJOR.MINOR.PATCH` form,
+for example `1.2.0`. Enter a version for each file in the upload drawer, script
+page, or beta replacement dialog. Versions belong to individual scripts; a batch
+can contain different versions.
+
+To fill empty fields automatically, choose `versions.json` from the same source
+checkout as your scripts. Cherry matches exact, case-sensitive filenames (such as
+`tws_t5_hud.gsc`) and leaves entered values intact. Review the values before
+uploading. A manifest identifies source releases; it does not verify file contents.
+
+Changing a script requires a new release version. Reuploading identical bytes with
+the same version reuses the saved release; different bytes with that version are
+rejected. Older release versions can be imported. Cherry keeps its upload number
+separately, displaying labels such as `1.2.0 (upload 7)`.
+
+Historical scripts without a release label display `Unversioned (upload N)` and
+remain available for testing, promotion, and rollback. Cherry does not infer old
+versions from today's manifest or rewrite historical records. Uploading the same
+bytes with an explicit version creates a separate labeled record when needed.
+Plugins retain optional publisher tags; configurations retain automatic upload
+numbers.
+
+API clients must include multipart `releaseVersion` when creating a script with
+`POST /api/scripts` or uploading with `POST /api/scripts/:id/versions`. Missing or
+invalid versions return HTTP 400; conflicting contents return HTTP 409. The
+version upload response reports `reused: true` for an existing identical release.
+
+## Connect game telemetry
+
+Configure your game reporter or plugin to send `EC_TELEMETRY_TOKEN` in an
+authorization header to `POST /api/telemetry`. This credential is separate from
+browser login, setup, and host enrollment. Ingestion is disabled until the
+controller has a valid telemetry token.
+
+The [game reporter protocol](beta-test-plans.md#game-reporter-protocol) defines
+events and payloads for gameplay statistics and beta-test evidence. Electric
+Cherry does not include the game reporter scripts themselves.
+
+## Maintain the installation
+
+Back up the complete controller data directory, including SQLite metadata and
+published game files. Preserve that directory across upgrades. Review
+[Maintenance](maintenance.md) for scheduled work and
+[Fleet settings](settings.md) for concurrency and retention controls.
+
+For lobby autoscaling, start with the documented
+[pilot procedure](autoscaling.md#pilot) before enabling live actions.

docs/guides/leaderboard.md

@@ -0,0 +1,442 @@
+# Leaderboard
+
+Open **Leaderboard** in the main navigation to compare players across the fleet,
+with one entry per GUID combining Black Ops I (T5) and Black Ops II (T6). Game, map,
+and starting squad-size filters limit which matches contribute to each player's statistics and
+rank. Choose a ranking, search a name or GUID, and share the page URL with its
+filters intact. Results refresh while the page is visible. The page defaults to
+**All maps** for a fleet-wide comparison; choose a map to narrow the results. A URL
+with no `map` parameter or with `map=` selects all maps. **Players** selects Solo, Duo, Trio or
+Four players using the recorded starting roster; unknown starting counts appear
+only under All squad sizes. The JSON API also defaults to all maps.
+
+## Qualifying matches
+
+Matches only appear in server/global history once they reach round 2.
+Leaderboards, summaries, map options, profile statistics and player history include only
+completed matches with a received `game_end`. Active, interrupted, and
+**End report missing** matches do not contribute. A player must also be observed
+at round 2 or later; players who left before a completed game's end retain their
+recorded stats.
+This also excludes existing round 1 records. Early telemetry is retained so
+qualifying games keep their start time, roster and cumulative counters.
+
+## Player profiles and match results
+
+### Profiles and filters
+
+Players have a profile as soon as they are first reported, even before they have
+qualifying match history. These profiles show their identity, an explanation of
+when statistics become available, and dashes for personal bests with no recorded
+match. The page fills in on refresh after a qualifying game ends. Unknown player
+GUIDs still return 404.
+
+Click a player's name to open their shareable profile at
+`/leaderboard/players/[guid]`. Profiles combine Black Ops I (T5) and Black Ops II
+(T6) records for the same GUID. Existing `/leaderboard/players/[game]/[guid]`
+links redirect to the combined profile. Profiles show all ten lifetime statistics,
+headshots as a percentage of kills, kills per down, average kills and observed
+rounds per game, and completion rate. Each map has its own complete set of
+records; each personal best shows its rank among players on that map in that game,
+across all squad sizes. Equal bests share a rank, with the next rank skipping tied
+positions (for example, 1, 1, 3). Map records and match history identify their game. Match history shows
+individual counters, server, date and completion credit in pages of 25 matches
+across both games. Map names link back to that map's leaderboard in its game.
+Profiles update every 30 seconds while visible; the freshness indicator offers Retry if an update fails. Ratios with a zero denominator
+display a dash. Lifetime statistics and per-map records always cover all recorded
+maps across both games.
+
+Opening a player from the leaderboard preserves the originating game, map, squad size,
+metric, search and page in the **Leaderboard** return link. The profile's
+**Filters** controls initialize from that game/map/squad/metric and show
+the player's rank within that competition, independent of the original search.
+These controls also filter match history; lifetime totals stay unchanged.
+History pagination retains the selection and return link. Map links preserve
+the selected ranking metric. A player without qualifying matches in a selection
+can still view their lifetime profile and choose another scope.
+
+On phones, per-map records and match history use compact cards with the map,
+game, record values and match identity up front. Record values link directly to
+their completed match, where the full evidence is available. Desktop profiles
+use compact tables; both views use the same history filters
+and pagination. History dates say **First observed**, since a missing start
+report can make the first recorded observation later than the actual game start.
+
+### Record evidence and match results
+
+Highest-round, best-game-kill and highest-score records link to the completed
+match that earned them on the leaderboard and in lifetime/per-map profile stats.
+Each source names its match and map. Open the match to see completion time,
+server, whether its start was recorded, and player participation evidence.
+Presence is not proof of continuous survival. Equal personal bests link to the
+earliest recorded completion, then the lowest match ID when times tie.
+
+Completed matches link to their own results page at `/matches/[id]` from global
+Match History, server history, and player profiles. The page shows the recorded
+map and server, final round, start/end times, duration when the start was observed,
+player counts, and the full saved roster. Players missing from the final report
+retain their earlier counters; players with no stats display dashes. Counters are
+the highest reported values within the match, not necessarily its final scoreboard.
+
+Each match shows one score column. If any player has a recorded total (including
+zero), **Total score** shows each player's highest recorded cumulative points
+earned in that match. Players absent from the final report retain their latest
+recorded total. A dash in this column means no total was recorded for that player;
+their points balance is never substituted. Historical matches where all totals
+are missing retain **Highest reported score**, the highest recorded points
+balance, not total points earned or necessarily the final score. This choice is
+based on recorded data, not the match date. It does not change leaderboard or
+profile score rankings, which still use balances.
+
+Active, interrupted, and **End report missing** matches remain listed
+without results links. Matches can filter each status and show known gaps in the
+reporter's sequence; a delayed report can reduce that missing-report count.
+Match pages remain available after the original server is removed.
+
+### Player identity
+
+The leaderboard uses the existing telemetry events; no reporter change is
+required. The leaderboard and profiles identify a player by **GUID**, displaying
+the latest reported name across both games, regardless of game or map filters.
+Different names for the same GUID do not split their leaderboard entry or
+profile; identical names with different GUIDs remain separate. Stored player
+records remain keyed by **game + GUID** for telemetry ingestion; combining their
+identity requires no rekeying. The record-source upgrade automatically adds
+nullable participation-start fields; existing rows keep an unknown start instead
+of inferring one. GUIDs stay strings.
+
+## Recorded statistics
+
+Telemetry stat reports (`round_change`, `heartbeat`, and `game_end`) may include
+`score_total`, a nonnegative integer read from the game's cumulative match score.
+`score` remains the spendable balance. Updated T5/T6 reporters capture game end
+at the first end signal, before stock intermission replaces the balance with the
+total. `game_start` remains identity-only. Deploy the receiver before the updated
+reporters so it retains the new field.
+
+The receiver accepts older reports without `score_total`. Latest telemetry
+preserves the field when supplied; match history retains the highest observed
+total independently of the balance and exposes it as `scoreTotal` on each player
+in the match-detail API. Missing totals and pre-upgrade history are unknown
+(`null`), not zero, and are never inferred from balances. Totals are per match;
+repeated snapshots are not added together. Existing score rankings continue to
+use the highest reported balance.
+
+| Statistic                              | Meaning                                                                                                                                                                                                                                                                                                                     |
+| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Highest round                          | Highest round reported with that player present, including players who joined mid-game.                                                                                                                                                                                                                                     |
+| Total kills, headshots, revives, downs | Sum of each player's highest reported counter in each game. Round snapshots are cumulative, so they are never added together within a game.                                                                                                                                                                                 |
+| Total rounds                           | Round progression during observed participation, added across games. A player observed at game start and then round 8 receives seven rounds. A player first seen at round 8 gets a highest-round record of 8, but zero total rounds; reaching round 9 then adds one.                                                        |
+| Games played                           | Completed games in which a stats report at round 2 or later contains the player, excluding players who left in the same round they joined. Staying through game end counts even without a round change. Legacy participation with an unknown joining round still counts. Start-only and round 1 participation do not count. |
+| Games completed                        | Games whose `game_end` roster contains the player. Disconnected players retain earlier stats without receiving completion credit.                                                                                                                                                                                           |
+| Best game kills                        | Highest kill count recorded in any one game.                                                                                                                                                                                                                                                                                |
+| Highest reported score                 | Highest observed score balance. Spending points can lower the score, so this is not cumulative points earned.                                                                                                                                                                                                               |
+
+Same-round departures retain their recorded stats, personal bests, and match
+history; only the games-played count excludes them. This also changes the game
+count used for efficiency qualification and per-game averages.
+
+Every ranking sorts highest first, including downs (a count, not an efficiency
+rating). Equal values share a rank; name and GUID provide a stable display order.
+Searching preserves the player's rank for the selected game/map filters, or
+across both games when unfiltered. Summary cards describe the same selection,
+regardless of the search text. Player counts include each qualifying GUID once,
+including when they have records in both games. Summary rounds are
+player-rounds: four players progressing from game start to round 8 contribute 28.
+Highest-round records always use the round reached in a completed match,
+including the final round and players who joined mid-game.
+
+### Efficiency rankings
+
+Efficiency rankings are also available for **Headshots / kills**, **Kills per
+down**, **Average kills**, **Average rounds**, and **Completion rate**. Each
+requires at least five completed games in the selected game/map scope. Headshot
+rate also requires 100 kills; kills per down requires five downs. The board shows
+the numerator, denominator and game count beside each ratio. Ratios are computed
+from aggregate counters, not an average of individual match percentages.
+Completion rate is the share of those ended games whose final report contained
+the player; it is not a win rate. Games without a received `game_end`, including
+**End report missing** games, do not enter any denominator. Zero-denominator ratios
+remain unknown and are not ranked.
+
+Qualification is applied before ranking, search and pagination. Summary cards
+still describe all players in the game/map selection, including those below the
+threshold. Profile metrics show their values and whether the player qualifies
+for the combined lifetime board; a scoped board can have a smaller sample.
+
+### Participation limits
+
+Round totals require the player to appear in consecutive observations. If a
+report omits them, their return starts a new participation interval and the gap
+is not credited. The payload cannot tell whether a present player was alive,
+spectating, or absent between reports. Mid-game snapshots still contribute their
+reported kills and other counters, even when earlier participation is unknown.
+
+## Persistence and match boundaries
+
+Match and player records live in SQLite alongside the rest of Electric Cherry.
+They persist across app restarts and server deletion, and are not subject to
+campaign history retention. The latest scoreboard and leaderboard update in the
+same transaction. Each match keeps player counters even after someone leaves.
+
+A `game_start` opens a match; `game_end` completes it. Reporters can send an optional
+`match` key and per-game `seq` on every event; see the
+[reporter protocol](beta-test-plans.md#event-fields) for field limits and retry rules.
+A key identifies one game on one server, including when the start report was
+missed. An unseen key opens a new game and labels the previous active keyed game
+**End report missing**. This records missing evidence; it does not establish
+whether the game finished or was interrupted. A late `game_end` for a previously
+seen key can still complete its own match; reports for known older keys do not
+replace the newer game's live state.
+
+Keys are opaque, and `seq` orders reports only within one game. Electric Cherry
+therefore orders games by their first processed report. If game `new` is already
+reporting when the first report for game `old` arrives, the unseen `old` key
+supersedes `new` and marks it **End report missing**. Later `new` heartbeats update
+its history but cannot restore it as the live game. If `old` arrived as a
+heartbeat, it can remain active even after `new` ends. Reporters should preserve
+delivery order across games, keeping queued retries ahead of newer games' reports.
+Reliably ordering unseen keys would require additional reporter metadata, such
+as a persistent, increasing per-server game counter; the current protocol has no
+such field.
+
+With both fields present, each sequence is counted once, including across app
+restarts. Lower sequences fill missing-report gaps without rolling back newer
+rounds, counters, or rosters. A delayed `game_start` may fill the recorded starting
+roster without resetting progression. The history API exposes the highest
+sequence minus the count of unique received sequences as `missingReports`, or
+`null` when sequencing is unavailable. A missing report count does not grant
+credit for unobserved rounds or complete a game without `game_end`.
+
+Without a match key, the legacy heuristics remain: a changed map, a lower round,
+or a round report after a completed game can establish a new match when the
+start was missed. Superseded active matches remain interrupted. Repeated starts
+before any stats belong to the same match. Identical consecutive payloads
+(ignoring receipt time and roster order) do not add records. Repeated snapshots
+in a match cannot increase counters or rounds twice.
+
+The first upgrade imports the latest stored full scoreboard from each server
+once. That provides the available counters and highest round, but not discarded
+earlier games or unobserved round progression. Old round-only snapshots have no
+player identity and cannot be imported. Tracking continues from future reports.
+
+T6 selections sharing an engine map, such as Town Survival and TranZit, remain
+separate using the server's registered map variant. Unknown variants remain
+explicitly identified instead of being labeled as a known selection. A match
+retains the game/map identity recorded at ingestion even if the server is later
+reconfigured or removed.
+
+Legacy reports without match identity cannot reliably distinguish a late report
+from an older game from a new game. Likewise, identical starts before any stats
+are indistinguishable from retries. Send those reports once in order. Within a
+match, counter decreases use the previous high-water mark; if a reconnect resets
+a player's counters, subsequent activity may be undercounted until those counters
+pass their earlier maxima. Match keys and sequence numbers do not identify
+player counter resets.
+
+## Read API
+
+This reference describes the current repository implementation. A deployed
+instance must run the corresponding code to provide the same behavior. The
+sources are the [HTTP handler](../../src/routes/api/leaderboard/+server.ts),
+[response types](../../src/lib/leaderboard.ts),
+[query implementation](../../src/lib/server/fleet/leaderboard.ts), and
+[authentication guard](../../src/lib/server/auth/guard.ts).
+
+### Authentication and requests
+
+`GET /api/leaderboard` is a public JSON endpoint; `HEAD` is public too. A tool
+does not need an account, session cookie, API key or authorization header to
+read it. Reads also work before administrator setup and with an expired session
+or a session awaiting a password change. This exception applies only to reads
+of this exact API route. Leaderboard/profile HTML pages, match-history APIs and
+management routes still require a session; see [Users and access](authentication.md).
+
+For example:
+
+```sh
+EC_BASE_URL='https://ec.example.com'
+curl --fail-with-body --silent --show-error \
+  --get "$EC_BASE_URL/api/leaderboard" \
+  --data-urlencode 'game=t6' \
+  --data-urlencode 'metric=kills' \
+  --data-urlencode 'limit=100' \
+  --data-urlencode 'offset=0'
+```
+
+HTTPS is required. Local HTTP development requires `EC_AUTH_ALLOW_HTTP=true`
+and a loopback hostname. The read request does not require an `Origin` header.
+This endpoint does not add cross-origin browser access (CORS) headers; the
+example above works for command-line or server-side clients. Successful responses use
+`Content-Type: application/json`, `Cache-Control: no-store` and `Vary: Cookie`.
+
+### Query parameters
+
+All parameters are optional. Names and enumerated values are case-sensitive.
+Use URL encoding for map keys and search text.
+
+| Parameter | Default         | Accepted values and behavior                                                                                                                                                                                                                              |
+| --------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `game`    | Both games      | Exactly `t5` or `t6`. An empty value is invalid.                                                                                                                                                                                                          |
+| `map`     | All maps        | Opaque `maps[].key`, at most 1024 characters. Empty means no map filter. An unknown key or a game/map mismatch returns an empty selection, not an error.                                                                                                  |
+| `players` | All squad sizes | Exactly `1`, `2`, `3`, or `4`, matching the recorded starting player count. Unknown counts contribute only when this filter is absent.                                                                                                                    |
+| `metric`  | `highestRound`  | One of the metric keys below. An empty or unknown value is invalid.                                                                                                                                                                                       |
+| `q`       | No search       | At most 100 characters before trimming. Matches a literal substring of the current name or GUID, ignoring ASCII letter case. Surrounding whitespace is trimmed; whitespace-only input means no search. `%` and `_` are literal characters, not wildcards. |
+| `limit`   | `50`            | Integer from 1 to 100.                                                                                                                                                                                                                                    |
+| `offset`  | `0`             | Integer from 0 to 1,000,000; counts matching entries to skip, not ranks or pages.                                                                                                                                                                         |
+
+`limit` and `offset` accept decimal digits only. Empty strings, signs, spaces,
+fractions and exponent notation are rejected rather than clamped. String
+length limits use JavaScript string length (UTF-16 code units). Unrecognized
+query parameters are ignored; if a parameter is repeated, its first value wins.
+
+Metric keys are `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives`,
+`downs`, `gamesPlayed`, `gamesCompleted`, `bestKills`, `bestScore`, `headshotRate`,
+`killsPerDown`, `averageKills`, `averageRounds`, and `completionRate`.
+
+### Success response
+
+HTTP `200` returns the following top-level fields directly, with no `ok` or
+`data` wrapper:
+
+| Field             | Type        | Meaning                                                                                                                           |
+| ----------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| `entries`         | Entry array | Requested page, ordered by the selected metric descending.                                                                        |
+| `maps`            | Map array   | Available completed-match map options, filtered only by `game`, independent of `map`, `q`, metric qualification and pagination.   |
+| `total`           | number      | Number of matching GUIDs after metric qualification and search, before pagination.                                                |
+| `limit`, `offset` | number      | Accepted pagination values.                                                                                                       |
+| `metric`          | string      | Selected metric key, including the default when omitted.                                                                          |
+| `summary`         | object      | Totals for all qualifying completed participation in the game/map scope, independent of `q`, metric qualification and pagination. |
+
+Each entry combines one GUID's qualifying participation across the selected
+games/maps. It contains all of these fields regardless of the selected metric:
+
+| Entry field                                                                                                                           | Type           | Meaning                                                                                                                |
+| ------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `guid`                                                                                                                                | string         | Player identity. Preserve it as a string, including leading zeros.                                                     |
+| `name`                                                                                                                                | string         | Latest reported name across both games, even if that report is outside the selected scope or from an unfinished match. |
+| `rank`                                                                                                                                | number         | Rank within the scope after metric qualification, before search/pagination. Ties share rank and leave gaps: 1, 1, 3.   |
+| `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives`, `downs`, `gamesPlayed`, `gamesCompleted`, `bestKills`, `bestScore` | number         | The ten counters defined under [Recorded statistics](#recorded-statistics).                                            |
+| `headshotRate`, `killsPerDown`, `averageKills`, `averageRounds`, `completionRate`                                                     | number or null | Ratios defined below; `null` if the denominator is zero.                                                               |
+| `lastSeen`                                                                                                                            | string         | Latest player observation among the qualifying participation in this scope, not the latest global identity update.     |
+| `records`                                                                                                                             | object         | Personal-best evidence keyed by `highestRound`, `bestKills` and `bestScore`.                                           |
+
+Entries have no `game` field. Equal metric values are displayed in name order
+(ASCII case-insensitive, then exact name), then GUID order. `q` is a substring
+search, not an exact-GUID lookup; a tool looking up one GUID must check equality
+on returned `guid` values and account for pagination.
+
+| Ratio            | Calculation                    | Requirement to rank by this metric    |
+| ---------------- | ------------------------------ | ------------------------------------- |
+| `headshotRate`   | `headshots / kills`            | `gamesPlayed >= 5` and `kills >= 100` |
+| `killsPerDown`   | `kills / downs`                | `gamesPlayed >= 5` and `downs >= 5`   |
+| `averageKills`   | `kills / gamesPlayed`          | `gamesPlayed >= 5`                    |
+| `averageRounds`  | `roundsSurvived / gamesPlayed` | `gamesPlayed >= 5`                    |
+| `completionRate` | `gamesCompleted / gamesPlayed` | `gamesPlayed >= 5`                    |
+
+These are unrounded ratios, not formatted strings. For `headshotRate` and
+`completionRate`, `0.5` means 50%. Requirements use the selected game/map scope
+and apply only to the metric being ranked. Entries on a counter leaderboard
+still expose ratio values even when they would not qualify for a ratio ranking.
+`gamesPlayed` counts ended matches with qualifying participation, including
+matches the player left early; it does not require completion credit.
+
+Each map option has string fields `key`, `game`, `map` and `name`. `map` is the
+engine map identifier; `name` is the display name. Pass `key` as the request's
+`map` parameter, preferably together with the option's `game`. Options come from
+completed matches reaching round 2, so an option can exist without any qualifying
+players. Removed servers' maps remain available.
+
+| Summary field             | Type           | Meaning                                                                                                               |
+| ------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------- |
+| `players`                 | number         | Unique qualifying GUIDs, combined across games.                                                                       |
+| `matches`                 | number         | Distinct completed matches with qualifying player participation; not the sum of `gamesPlayed`.                        |
+| `kills`, `roundsSurvived` | number         | Sums across qualifying players; rounds are player-rounds.                                                             |
+| `highestRound`            | number         | Highest qualifying player's recorded round.                                                                           |
+| `firstTrackedAt`          | string or null | Earliest match `startedAt` among those matches.                                                                       |
+| `updatedAt`               | string or null | Latest match report receipt time among those matches; not a response-generation time or a complete data-change token. |
+
+For an empty scope, numeric summary fields are zero and both timestamps are
+`null`. A search with no results or an offset beyond `total` returns
+`entries: []` while preserving the scope's summary and available maps. An
+efficiency board can likewise have no entries while `summary.players > 0`.
+
+Each `records` value has the following fields. Evidence uses the same game/map
+scope as the entry, and ties choose the earliest completion, then lowest match
+ID. The three records can refer to different matches.
+
+| Evidence field                        | Type            | Meaning                                                                                              |
+| ------------------------------------- | --------------- | ---------------------------------------------------------------------------------------------------- |
+| `matchId`, `value`                    | number          | Completed match ID and that personal-best value. Link to `/matches/{matchId}` for its results page.  |
+| `game`, `mapKey`, `mapName`, `server` | string          | Recorded game, opaque map key, map display name and server name.                                     |
+| `startedAt`                           | string          | First recorded match observation; only the game start when `startObserved` is true.                  |
+| `endedAt`                             | string or null  | Recorded completion time; can be unknown in older stored records.                                    |
+| `startObserved`                       | boolean         | Whether a `game_start` was received for the match.                                                   |
+| `presentAtStart`                      | boolean or null | Whether the player was first observed at the recorded start; `null` for unknown older participation. |
+| `firstObservedRound`                  | number or null  | Round of the player's first observation, or unknown. This does not prove continuous survival.        |
+
+Timestamps reflect controller receipt/observation times, normally ISO 8601 UTC
+strings such as `2026-10-05T12:00:00.000Z`, not reporter gameplay-clock times.
+
+### Errors, pagination and freshness
+
+Invalid query values return HTTP `400` with
+`{ "ok": false, "kind": "bad-request", "error": "..." }`. Public reads do not
+return `401` for missing/expired sessions or `403` for a required password change.
+Sending host/enrollment bearer credentials still returns `403` because those
+credentials are restricted to the host-agent API; omit them. Disallowed HTTP transport
+returns `426` with an `error` string. Report-service failures can produce `5xx`;
+check status and content type before assuming the success schema or a uniform
+error body.
+
+For successive pages, retain the same filters and increase `offset` by `limit`
+until `offset + entries.length >= total` or the page is empty. There is no cursor,
+snapshot token, date-range filter or incremental `since` parameter. Each report
+reads a consistent database snapshot, but separate requests may see newly ended
+matches or name changes. A multi-page crawl can therefore miss or repeat players
+as their ordering changes; deduplicate by GUID and refresh as needed. The offset
+cap also limits how far a tool can page through a very large selection.
+
+Reports run in a worker and use caches keyed by database revision, invalidated
+by writes to match, participation and player-identity tables. There is no fixed
+TTL wait before newly committed leaderboard data becomes eligible for a fresh
+report. Identical requests already in flight may share a response. Neither
+`summary.updatedAt` nor an entry's `lastSeen` should be used as a cache revision:
+an identity rename can change results without changing those timestamps.
+
+### Related routes and current limits
+
+`GET /api/matches` and `GET /api/servers/{name}/matches` return JSON match
+summaries as `{ "matches": [...], "total": number, "page": number, "limit": number, "maps": [...] }`.
+These routes still require session authentication. Both accept `limit` (default
+25, range 1–10,000), `page`, `game`, `map`, `q` (server name), `days` (0, 7, 30, or
+90), `status`, `sort` (`started`, `round`, `duration`, or `players`), and `direction`
+(`asc` or `desc`). Results default to newest first by first observation, then
+match ID, and include unfinished matches that reached round 2. Status values are
+`active`, `completed`, `interrupted`, and `end-report-missing`; omit the filter for
+all statuses. `reporterMatchKey` is the supplied match key or `null`;
+`missingReports` is the known sequence-gap count or `null`. Their
+[ServerMatch fields](../../src/lib/match-history.ts) differ from leaderboard entries
+and do not include player counters or a roster. Unknown server names return an
+empty list. URL-encode server names in paths.
+
+`/leaderboard/players/{guid}` and `/matches/{id}` are HTML pages backed by server
+loaders. There is currently no dedicated JSON endpoint at
+`/api/leaderboard/players/{guid}` or `/api/matches/{id}` for complete profiles,
+player match history or match rosters. A tool needing those records will need
+an API extension; the leaderboard endpoint supplies aggregate stats and
+personal-best evidence only.
+
+## Query performance
+
+On a cache miss, leaderboard reads aggregate the selected completed participation
+for rankings and summary totals. The ranked scope and map options are reused
+across search/pagination requests at the same database revision; filtering and
+pagination operate on that ranked list. Empty and out-of-range pages still
+return accurate totals. Record evidence is fetched only for requested entries.
+Exact player-rank reads use the same ranked scope, including its summaries and
+map list. Report work runs in a dedicated worker with revision-keyed response
+caching as described above.
+
+See the [historical SQL benchmark](../research/leaderboard-query-benchmark.md)
+for the earlier optimization measurements and methodology.

docs/guides/maintenance.md

@@ -0,0 +1,200 @@
+# Maintenance
+
+Open **Activity → Maintenance** to keep existing servers healthy and current.
+Maintenance owns observations, policies, schedules, requests, and verification of
+maintenance outcomes. Provisioning owns lifecycle execution, player checks,
+conflicts, rolling capacity, container recovery, and persistent operation history.
+Campaigns retain managed-file deployment evidence; Collections define lineups.
+
+## Views
+
+- **Overview** shows active work, attention, upcoming restarts, and Plutonium version
+  coverage. Filter by capability or server. **Check now** refreshes health, drift,
+  the production release, and running revisions. Failed checks are grouped by
+  cause with links to affected servers; raw paths and responses stay in
+  **Technical details**. Identical revision states share one summary row. Expand
+  it for each server’s check and pending restart, or use **Retry version check**
+  when the running revision is unverified.
+- **Policies & schedules** controls health recovery, configuration drift, automatic
+  Plutonium updates, and the optional daily restart schedule independently.
+- **Activity** shows retained history scoped to maintenance and recovery.
+  Filter retained findings, update results, and scheduled restarts in one timeline.
+  Follow a finding for evidence or an operation for execution progress. Use
+  Overview’s **Active work & attention** to cancel waiting work or retry failed
+  verification after inspection.
+
+Fleet and server details link their revision indicators to Maintenance. Failed
+runtime maintenance appears in Fleet attention, even when the underlying restart
+succeeded. Provisioning and repair Campaigns link back to the requesting work.
+
+## Policies and pause
+
+Health recovery and drift repair inherit the previous Watchdog enabled setting.
+Automatic Plutonium updates are **disabled by default**. Existing daily schedules
+retain their enabled state, timezone, and next occurrence.
+
+**Pause all maintenance** holds new remediation, scheduled runs, and unsubmitted
+maintenance operations. Scans and result verification continue; submitted
+operations finish recovery. Campaigns, betas, and manually requested Provisioning
+operations continue independently. Resuming keeps each policy's saved setting.
+
+Disabling health recovery or drift repair holds queued work for that capability
+and prevents further automatic attempts, including health escalation. Disabling
+automatic updates holds queued automatic update requests. Explicitly requested
+updates remain operator-authorized work. Disabling a daily schedule prevents
+future runs but does not cancel requests it already created.
+
+All capabilities respect intentionally stopped servers. Start them explicitly
+through Provisioning before expecting maintenance to act on them.
+
+While Provisioning owns a server, Maintenance skips new health/drift observations,
+version probes, automatic updates, and scheduled restart requests for that server.
+Queued, waiting, running, and recovering operations count, including failed
+operations that retain recovery capacity. Other servers continue normally. Existing
+maintenance tracks its own operation to completion; revision verification waits
+until any later operation releases the server.
+
+## Plutonium updates
+
+The published revision comes from the official production updater manifest.
+Running revisions come from RCON `version` after checking container readiness.
+Checks run hourly while the application is running. Failed release checks retain
+the cached release with an error. Unsupported replies, stale checks, unhealthy
+containers, and evidence from before a lifecycle submission remain unverified.
+A missing revision is never treated as current or as an automatic update target.
+
+**Update on restart** requests ordinary Provisioning restarts for verified outdated
+servers. **Update outdated servers on restart** applies to the filtered list.
+Automatic updates, when enabled, apply to verified outdated servers across the
+fleet, with one attempt per server and published revision. Existing pending
+maintenance is reused. Occupied servers and servers with conflicting beta or
+campaign work wait without consuming a restart slot.
+
+Plutainer's startup updater must be enabled (`PLUTAINER_AUTO_UPDATE`). Electric
+Cherry does not download binaries, pull a container image, or pin the runtime to
+a particular revision. The observed published revision is recorded as the minimum
+expected result; the actual revision is queried after recovery.
+
+A healthy restart and a successful update are separate outcomes. If the server
+recovers on an older or unknown revision, the restart is successful in Provisioning
+and the update needs attention in Maintenance. Update verification never triggers
+a rebuild or an automatic restart loop. Inspect the updater configuration and
+server logs, then use **Retry after inspection** to create a new attempt. Ambiguous
+or failed lifecycle requests require inspection of the existing Provisioning
+operation and its retained recovery slot first.
+
+Update attempts persist as suppression evidence across app restarts and history
+retention. Cancelling an automatic attempt suppresses another automatic attempt for
+that same revision; an operator can request an update explicitly.
+
+## Scheduled restarts
+
+Daily restarts are independent of update availability. At the saved queue time,
+Maintenance requests a restart for each server whose desired state is running.
+These requests have no revision requirement. Servers owned by Provisioning are
+skipped for that occurrence; the schedule advances normally. Requests already
+queued can wait for players, campaign work, or beta work to finish.
+
+Saving a schedule uses the selected timezone and follows daylight saving changes.
+Skipped times run at the first valid minute after the gap; repeated times run once
+on their first occurrence. Legacy UTC schedules keep their timing until saved.
+Changing the browser display timezone does not change the saved schedule.
+
+Schedules survive application restarts. Missed days, including while globally
+paused, become one run on resume. Pending work is deduplicated per server. The
+worker starts on the first application request, so keep the app running and
+reachable. Finished scheduled and legacy maintenance activity is retained for
+30 days by default (configurable in Admin → Settings); update attempt records remain durable.
+
+## Health recovery and configuration drift
+
+Fleet keeps successful game and running-container reports fresh for 90 seconds.
+Two consecutive failed monitoring checks mark RCON health stale; a first timeout
+or send failure remains visible in diagnostics without changing the health badge.
+Configuration and invalid-reply errors, including authentication failures, take
+effect immediately, as do failed manual probes. Successful RCON checks reset the
+failure count; skipped checks do not count. Without fresh telemetry, RCON evidence
+older than 120 seconds is offline. Explicit stopped/unhealthy container states
+and lifecycle changes still take precedence.
+
+The status monitor collects container and game health every 30 seconds. Once per
+minute, the watchdog evaluates that completed evidence without issuing its own
+routine health probes. Samples expire after 90 seconds; missing or stale evidence
+leaves health unknown and cannot justify recovery. Samples retain their original
+timestamps and instance identity, and each failure sample counts at most once.
+A monitoring gap or changed instance resets the sustained failure window. After
+an application restart, routine watchdog checks wait for a new status-monitor pass.
+**Check now** and recovery verification still request fresh evidence, and
+Provisioning retains its checks before executing recovery actions.
+
+Three observations over at least two minutes are required before automatic
+health recovery; starting or restarting containers get five minutes. A confirmed
+missing container goes directly to rebuild after this same sustained detection
+window. It cannot contain players, so recovery skips the player-count check.
+Offline hosts, unknown container state, and missing or stale observations wait
+for fresh evidence. Repeated manual checks do not accelerate this
+window. A successful routine or manual game-status probe between scans resets
+the failure streak. Missing RCON configuration, unresolved ports, and invalid
+replies alone remain diagnostic findings and cannot authorize health recovery.
+
+A health finding for an existing container gets one restart, then one rebuild if
+recovery fails after five minutes and fresh health checks still justify recovery.
+Missing-container recovery gets one direct rebuild. Both paths respect maintenance
+policy and pause settings, intentionally stopped servers, capacity limits, and
+protected beta or campaign work. If current container and
+game checks succeed before an automatic health restart or rebuild is submitted,
+the obsolete operation is cancelled and the finding is resolved. Unknown health
+holds recovery for another check. Drift repairs and requested lifecycle operations
+retain their own eligibility rules.
+
+Escalation transfers only that finding's restart slot. A failed rebuild
+requires operator attention. **Rebuild deletes all instance files** and uses the
+latest promoted setup, the same server identity and port, and the saved server key.
+Missing keys block teardown. Submitted rebuild verification continues while paused.
+
+Drift includes missing promoted files, changed bytes, and extra files in managed
+script, config, and plugin directories. Config comparison includes the composed
+baseline, overlays, selected map, and the RCON password assigned to the current
+instance at launch. Launches and beta config writes use the same password rendering
+as drift checks, including Plutainer's startup newline handling. Unmanaged default config is exempt until
+a baseline is promoted. Missing mounts are scan errors, not empty directories.
+Routine drift scans run every five minutes per server, reducing file traffic
+while health observations continue every minute. **Check now**, a replacement
+registration, or a new lifecycle submission bypasses the drift interval. Failed
+file scans retry on the next observer pass; post-rebuild verification still reads
+fresh files.
+
+For older instances whose only difference is Plutainer's expected password rewrite,
+a successful scan reconciles the recorded config hash without changing server
+files or rebuilding. The entire file must match the expected runtime config;
+unexpected passwords and other config edits still count as drift. Current launch
+credentials remain available through history cleanup while the instance is live.
+If that launch history was already expired, the check reports unavailable evidence
+instead of guessing a password or requesting a rebuild.
+
+Drift repair creates a Campaign capturing observed differences and requests a
+Provisioning rebuild. The Campaign observes that rebuild without requesting
+another restart. Its detection evidence and the actual launch recipe remain
+available even if promotions change while the work waits. Histories survive
+instance removal. Active beta and campaign work defers new recovery.
+
+**Dismiss** suppresses health/drift attempts until the condition clears. **Retry
+recovery** begins a fresh bounded attempt after inspection. Existing operations
+must finish or be cancelled first. Fresh verification resolves the finding.
+
+## Compatibility and API
+
+Existing job, incident, and operation IDs are retained. Upgrade migration adds
+request metadata without replaying commands; existing pending requests keep their
+original verification requirements. `/watchdog` and `/watchdog/:id` redirect to
+Maintenance, preserving query parameters. Old Watchdog and Plutonium APIs remain
+compatible. The previous Watchdog enabled switch controls health and drift only.
+
+- `GET /api/maintenance`: policies, recovery findings, release/version coverage,
+  schedule, and recent runtime activity with linked operation stages.
+- `POST /api/maintenance`: actions `scan`, `pause` (`paused` boolean), `policy`
+  (`policy`: `health`, `drift`, or `updates`; `enabled` boolean), `update` (optional
+  `servers` array), `schedule` (`enabled`, `time`, `timeZone`), `cancel` or `retry`
+  (`id`, the numeric runtime maintenance activity ID).
+- `/api/maintenance/:id`: GET recovery evidence and attempts; POST `retry` or
+  `dismiss` for a health/drift finding.

docs/guides/metrics.md

@@ -0,0 +1,275 @@
+# Metrics
+
+Metrics is under **Analytics** at `/metrics` (keyboard shortcut: `g m`).
+Choose the past 24 hours, 7 days, or 30 days. The page refreshes every minute.
+
+## Overview and filters
+
+**Lobbies running** shows the average and peak number of recorded lobbies in the
+selected scope. The average divides captured lobby-hours, including unknown
+status, by captured wall time. Empty spares are deliberate supply for new groups:
+the card shows their hours and share of all captured lobby-hours, rather than
+treating them as failed utilization. Unknown status can include lobbies starting
+or unavailable, so this count does not guarantee every lobby was ready to join.
+
+**Player-hours** integrates reported player counts over time. **No empty lobby**
+shows the share of known map time when no tracked lobby reported zero players.
+Each game/map is measured separately, then its durations are pooled. This is not
+elapsed fleet time or a count of rejected joins. A partly occupied Zombies lobby
+does not count as an empty spare, even if player slots remain.
+
+**Last collected** is the newest recorded batch, including unknown checks;
+**Report refreshed** is when this report was calculated. Collection stops when no
+servers are registered. Earliest retained observations do not prove continuous
+history or the original installation date.
+
+Background refresh keeps the last successful report visible on failure and offers
+Retry. Returning to a visible tab refreshes immediately. Game and map filters apply
+to the whole report, including comparisons, signals, and the heatmap. Map choices
+preserve the exact recorded game/map pair and include retained occupancy, match,
+and autoscaling Provisioning history. Range and scope are shareable through the URL; refreshes preserve table
+controls and chart selection.
+
+## Autoscaling operations
+
+The **Autoscaling** section follows the selected period, game, and map and links
+to the policies. **Lobbies added** and **Lobbies removed** count successful create
+and remove operations, including completions with warnings. Failed and cancelled
+operations appear separately. Counts come from retained Provisioning records
+linked to autoscaling, independently of occupancy samples; manual operations and
+changes in sampled lobby counts do not contribute.
+
+An operation belongs to the period containing its finish time, including both
+endpoints. A request made before the period still counts if it finishes within it;
+pending operations and finishes outside the period do not count. **Request to
+ready** shows the average and maximum request-to-finish duration of successful
+creates with valid timestamps, including queue time. Each create has equal weight.
+Without valid completed creates, these timings are unavailable, not zero. Verified
+readiness does not establish browser visibility or successful external joins.
+
+Scope uses the operation's recorded server name, which encodes its game and map
+variant. Game-wide policies do not combine different variants when filtering by
+map. Friendly map names, slugs, and unambiguous canonical map names are supported;
+an ambiguous engine label such as `zm_transit` is not assigned to a variant.
+History survives server removal and policy deletion, subject to Provisioning
+retention. Missing or purged operations are not estimated. The policy card's
+31-day spare-deficit measurements remain separate from this section.
+
+## Matches and groups
+
+**Matches started** counts retained matches whose recorded start falls within the
+selected period, including both endpoints. A missing start event uses the first
+recorded match observation, which may be later than the actual start. Active,
+completed, and interrupted matches all count, including matches that never reach
+round 2 and matches on removed servers. These counts do not use the leaderboard's
+completed-match or round qualification rules.
+
+**Average group size** is the average number of distinct recorded player GUIDs per
+match. A player counts once across all reports for that match, including late
+joins and players who left before its end. Each match has equal weight in the
+fleet average; the average is not an average of map averages. This measures
+recorded participation over the match's lifetime, not its starting squad or peak
+simultaneous occupancy. Active matches can change as more players are recorded.
+
+**Solo games** counts matches with exactly one distinct recorded player. Its share
+uses only matches with at least one recorded player. Matches with no recorded
+players still count as started, but their group size is unknown and they are
+excluded from both the average and solo-share denominator. If no matches have
+recorded players, those two values are unavailable rather than zero.
+
+The map table includes these same match statistics. A map with match history in
+the selected period remains visible even without occupancy samples; its match
+statistics do not establish lobby-hours, availability, or an autoscaling shortage.
+Match statistics cover the selected period only and have no previous-period
+comparison.
+
+## Players over time
+
+The player chart uses the lobby chart's periods: hourly for 24 hours, and UTC-aligned
+24-hour periods for 7 or 30 days. Labels use the report timezone. **Unique players**
+counts distinct player GUIDs in every period overlapped by their recorded match
+participation, from first to last sighting. Multiple matches in one period count
+once, and the same GUID across Black Ops I and II is one player. Matches that began
+before the range still contribute participation inside it. These are recorded
+participants, not concurrent player counts or the lobby chart's player-hours.
+
+**New players** counts a GUID at its earliest recorded sighting across both games.
+Game and Map filters limit unique players to matching retained matches; new players
+count only when their original sighting was in a matching match. A regular's first
+visit to another map or game does not make them new again. Period boundaries belong
+to the following period; a sighting exactly at the range's end counts in the final
+period. Range totals count distinct GUIDs across the entire window independently:
+daily unique counts cannot be added to get the range total.
+
+Periods before the earliest retained match are gaps, not zero players. This history
+boundary is fleet-wide, so filtering to a quiet map does not hide recorded zeroes.
+The chart notes limited history and marks periods clipped by the selected range or
+the retained-history boundary with hollow points. Totals cover available history
+only. Select a point, use Left/Right or Home/End, or open **Inspect a period** for
+the exact interval and counts. **View exact time data** provides the same values
+in a keyboard-accessible table, with missing history shown as a dash.
+
+History begins when leaderboard recording started; earlier play cannot be recovered.
+Players already playing then count as new when first recorded. Participation spans
+do not prove continuous presence between reports, and unrecorded matches are absent.
+Match retention removes participation, which can shorten or leave holes in this
+history. The durable first-seen player identities survive retention, so removing an
+old match cannot make a returning player new again. The original map cannot be
+recovered from an expired match. The chart remains available without occupancy samples.
+
+## Timeline and tables
+
+The timeline defaults to **Average lobbies running**, split into **With players**,
+**Empty spare**, and **Unknown**. Each bucket divides those lobby-hours by its
+captured wall time, so changing lobby counts remain visible. Collection gaps are
+excluded from that denominator. The average-lobby axis uses whole-number ticks;
+period details preserve fractional averages. Switch to the share view to compare proportions,
+or hours to compare absolute lobby time. Periods without captured time remain gaps.
+
+The timeline has labelled axes and marks partial edge periods. Select a bar or use
+Previous/Next to inspect its exact interval and values. The time table supplies
+the same figures for comparison and assistive technology. Zero captured time is
+distinct from an observed empty lobby. Short durations use seconds or minutes so
+a real observation is not rounded to zero.
+
+The map table supports numeric sorting and local search. **Lobbies (avg · peak)**
+shows the time-weighted average and peak concurrent lobby count for each recorded
+game/map pair, with empty lobby-hours beneath. Sort this column by average count.
+The average divides all captured lobby-hours, including unknown occupancy, by
+that map's captured wall time; collection gaps and periods without recorded lobbies
+for that map are excluded. Peak is the largest captured concurrent count, not the
+number of distinct lobby names. Empty lobby-hours sum time with zero reported
+players across the map's lobbies. Unknown occupancy is not counted as empty;
+empty time is unavailable when no occupancy is known. Lobby counts are unavailable
+without captured time. Per-lobby match and activity history is available on server pages.
+
+The map table's
+**No empty lobby** column uses occupancy alone and retains the known map-time
+denominator in its details. Unknown map availability stays distinct from zero
+time without a spare. Evidence can be limited by short observations or unknown
+occupancy, never by missing player limits.
+
+## Comparisons and activity
+
+Previous-period comparisons use the preceding equally long window with identical
+filters. Average lobby counts compare recorded supply, while player-hours changes
+are relative percentages. A previous observed zero can show **New**. Missing
+evidence remains unavailable. Partial retained history is labelled; a 30-day
+comparison usually has little previous history because default retention is
+31 days. Collection gaps are excluded, and comparisons do not prove uninterrupted
+collection or adjust player-hours for fleet size.
+
+The activity heatmap groups captured time by weekday and hour in its displayed
+timezone. Color represents occupied / known lobby time. Exact cell values include
+unknown time and contributing dates. No-data cells differ from observed zero;
+repeated DST hours contribute their actual elapsed time. Occupancy share describes
+how the fleet was used, but can change simply because a policy adds or removes
+spares. Read it alongside player-hours, lobby counts, and empty-lobby availability.
+
+## Autoscaling signals
+
+Signals compare each map's observations with its **current** applicable policy.
+Game-wide policies apply independently to each supported map variant. Each signal
+links to the policy on **Fleet → Autoscaling**; a map without a policy gets a link
+to add one. Signals are advisory and never change a policy or a lobby.
+
+A global pause appears once above the signals. Historical shortage signals remain
+visible while the fleet is paused, including suggestions to review the maximum
+or spare target. Review these findings and current settings before resuming.
+An individual policy's own pause or disabled state still prompts a policy review.
+
+| Recorded condition                                                                       | Suggested action                                                 |
+| ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------- |
+| No empty lobby for at least 2% of known map time, mostly at or above the current maximum | Review raising the policy maximum.                               |
+| No empty lobby for at least 2% of known map time, mostly below the current maximum       | Review the spare target and check creation time or blockers.     |
+| An applicable policy is off or paused                                                    | Review the policy and its pause or activation conditions.        |
+| No applicable policy and known occupancy is available                                    | Consider adding an autoscaling policy to keep spares available.  |
+| Occupancy unavailable                                                                    | Check telemetry/RCON reporting and wait for known player counts. |
+| No shortage signal                                                                       | Keep watching empty-lobby availability.                          |
+
+Evidence includes no-empty map time and how it splits above/below the current
+maximum. Less than one hour of known map availability, less than 80% known map
+availability, or less than 80% known lobby time makes advice tentative. Player limits are not needed. These
+thresholds are advisory rules, not demand forecasts.
+
+Policy settings may have changed during the selected period. Comparing historical
+counts against today's maximum does not establish which settings applied then or
+whether autoscaling was enabled. No reported empty lobby also does not prove
+failed joins or unmet demand. Review provisioning delays, blockers, host resources,
+and game-key availability before changing targets.
+
+## Definitions and collection
+
+- **With players:** a successful observation reported at least one player.
+- **Empty spare:** an observation reported zero players.
+- **Unknown:** failed checks, provisioning, or otherwise unavailable occupancy.
+- **Player-hours:** reported players × observed time. Bots count if the server includes them.
+- **Empty lobby available:** at least one lobby on the game/map reported zero players.
+- **No empty lobby:** at least one lobby reported occupancy, and none reported zero. Mixed positive and unknown counts mean no empty lobby was reported; an unknown lobby might actually have been empty.
+- **Unknown map availability:** every lobby on the game/map had unknown occupancy. This interval is excluded from the known-map denominator and is never counted as a spare shortage.
+
+Observations come from the existing status worker. Each complete collection stays
+valid until the next collection, up to the 90-second freshness limit. Longer gaps
+are excluded, not counted as unknown or zero lobbies. Captured wall time includes
+intervals with at least one recorded lobby in the selected scope. Unknown lobby
+time and unknown map time use different units: several unknown lobbies on one map
+contribute several lobby-hours but only one interval of unknown map availability.
+
+Charts split time at UTC hour/day boundaries; partial periods remain partial.
+Map variants use the configured variant when it matches the reported map. Player
+capacity may be retained in raw samples, but no metric or signal requires
+`sv_maxclients` or an assumed four-player limit.
+
+Occupancy collection begins after this feature is installed; earlier occupancy cannot be
+reconstructed. Samples default to 31 days of retention; **Admin → Settings** can
+change this period. Cleanup runs during collection and in the background. Samples
+survive removal of a server and contain no player identities.
+
+Match statistics use existing saved match history, independently of occupancy
+samples and raw telemetry diagnostics. **Match history retention** under
+**Admin → Settings → Matches & players** defaults to indefinite. Expiring a
+completed or interrupted match also removes its player participation and changes
+these statistics; active matches are protected. Increasing retention cannot
+restore expired matches. Saved match history survives removal of a server.
+
+## API
+
+`GET /api/metrics` supplies the same report as the page and disables HTTP response
+caching. Query parameters are `days` (1, 7, or 30), `game`, `map` (requires game),
+and `timeZone` (an IANA zone). Invalid ranges and zones fall back to the defaults;
+invalid scope syntax is normalized. Fleet collection timestamps remain separate
+from selected-scope evidence timestamps. Collection boundaries are determined
+before scope filtering, so samples from other groups still end a batch interval.
+
+`running` contains `average`, `peak`, and `capturedHours`; timeline points include
+their own `capturedHours`. Report-level `matches` and each map row's `matches`
+contain `started`, `withRecordedPlayers`, `averageGroupSize`, `solo`, and
+`soloShare`. `withRecordedPlayers` is the denominator for the group average and
+solo share; `soloShare` is a percentage from 0 to 100. When that denominator is
+zero, `averageGroupSize` and `soloShare` are `null`, `solo` is zero, and `started`
+retains the recorded number of starts. Match scope uses the exact saved `game` and `map_name`
+labels, and its time window uses the saved `started_at` timestamp. Occupancy
+collection and evidence timestamps remain independent of match history.
+
+`players` contains the separately deduplicated range totals `uniquePlayers` and
+`newPlayers`, plus `historyStartAt` (the earliest retained match, in epoch
+milliseconds). Every `points` entry also has `uniquePlayers`, `newPlayers`, and
+`playerHistoryPartial`. Counts before retained history are `null`; known periods
+with no scoped participation are zero. Player charts treat either `partial` or
+`playerHistoryPartial` as partial. Participation timestamp changes and durable
+first-seen identity changes invalidate the report cache.
+
+Map rows expose `noEmpty`, `availabilityObserved`, and
+`availabilityUnknown` durations, plus current `policy` context and
+`noEmptyAtMaximum` / `noEmptyBelowMaximum` durations. `pooledAvailability` combines
+map durations and coverage. `autoscalingPaused` reports the current global pause
+separately from each policy's own `pausedReason`. Policy edits and global pause changes invalidate
+cached metrics reports, as do changes to the match and player participation data
+used by the report.
+
+`autoscaling` contains `added`, `removed`, `failed`, `cancelled`, `readinessSamples`,
+`readinessMeanSeconds`, and `readinessMaxSeconds`. Counts use `finished_at` in the
+selected window; readiness uses `finished_at - started_at` for successful creates
+with nonnegative, valid durations. Both timing values are `null` when there are
+no readiness samples. Changes to Provisioning outcomes or autoscaling source
+links invalidate cached reports, including when no new occupancy is collected.

docs/guides/moderation.md

@@ -0,0 +1,255 @@
+# Player moderation
+
+**Players → Moderation** provides player search, warnings, private messages,
+kicks, announcements, bans, mutes, case records and private history. Restrictions
+can cover a server, a game or the fleet. Saving a restriction records the policy;
+the coverage table shows whether game runtimes have synchronized it.
+
+## Access
+
+Administrators manage all moderation controls and configure integrations.
+Moderators can read private history, create notes/cases, send live actions, and
+issue server/game restrictions lasting at most seven days. They can revoke their
+own server/game restrictions originally lasting at most seven days. Applying or
+revoking permanent, longer or fleet restrictions, and revoking another staff
+member's restriction, require Administrator access. These limits also apply after
+an administrator becomes a Moderator. Super Viewers can inspect
+moderation but cannot change it. Operators and Viewers cannot read private
+moderation data. Moderator does not grant fleet operation, script, plugin,
+credential or account-management privileges.
+
+Assign Moderator through **Admin → Users**. Existing users retain their roles;
+changing a role revokes that user's sessions. Each service rechecks current
+authorization, including immediately before delivering a queued live action.
+
+## Player identity and restrictions
+
+Select a player from Moderation, a live scoreboard or a player profile. Search
+matches known aliases as well as GUIDs. A display name never identifies a
+restriction target. Manually entered GUIDs can have case records and restrictions
+before gameplay is observed; their first/last seen dates remain **Not recorded**.
+
+IDs remain strings. Numeric game IDs are normalized without losing precision;
+T6 RCON hexadecimal IDs have a separate conversion rule. Historical chat links
+search for the raw log identity because its encoding is not assumed. Confirm a
+known player identity before applying a restriction. Fleet restrictions require
+the same numeric account ID to have been observed in both games. Otherwise use
+game scope.
+
+Every restriction has a player-visible reason, optional private note, scope,
+expiry, creator and revision. Reasons and live messages must fit in 240 UTF-8
+bytes. The native adapter replaces console control characters in displayed text.
+Overlapping restrictions remain independent: revoking one mute does not cancel
+another active mute. Revocation requires a reason and the current revision;
+concurrent changes require refreshing before another decision.
+
+Server scope follows a logical server registration across instance replacement
+or rebuild. Deleting and later registering the same server name creates a new
+registration, which does not inherit the old server-scoped restriction.
+Game/fleet policies apply to every connected compatible runtime in that scope,
+including newly provisioned servers. A new runtime must synchronize before it
+has a complete policy; provisioning readiness alone does not prove moderation
+coverage. The initial limit is 1,000 active restrictions across the controller.
+
+## Enable the game integration
+
+The controller requires `EC_PUBLIC_URL` to be a public HTTPS origin. The host
+agent must advertise `moderation.configure` (agent 1.11.0+). Install the matching
+TWS moderation script 1.0.0, common 1.2.0+, and TWS native plugin 1.3.0+ from
+`tws-zombies-scripts` for each game. New native plugins require a full
+game-server process restart. Use the normal release/beta workflow and validate
+one T5 Zombies and one T6 Zombies instance before fleet rollout.
+
+New launches/rebuilds on capable agents receive a private moderation config when
+`EC_PUBLIC_URL` is configured. Existing instances can use **Enable integration**
+in Moderation. This queues a typed operation that writes the fixed private config
+and loads its dvars, without restarting gameplay. Existing startup persistence
+uses the verified private bank config; older instances without that bootstrap
+must be rebuilt. Installed moderation scripts retry missing configuration.
+
+The private file `configs/electric_cherry_moderation.cfg` contains the HTTPS
+endpoint, generation and a separate per-instance credential. It is excluded from
+shared artifacts, file reads, drift scans, logs and operation receipts. Game
+credentials are distinct from bank, telemetry and host credentials. They only
+authorize the active immutable instance generation; retirement/rebuild revokes
+the previous generation's access. Older agents retain ordinary server launches,
+but cannot provide this integration.
+
+Coverage states describe runtime communication:
+
+- **Policy current:** a fresh runtime supports both bans and mutes and has
+  acknowledged the current policy revision.
+- **Pending:** the runtime is connected but has not acknowledged the latest revision
+  or does not support both bans and mutes.
+- **Offline:** no runtime synchronization has arrived for more than 15 seconds.
+- **Unavailable:** this instance generation has not connected.
+
+Policy current is not proof that an engine mute command took effect. The runtime
+reports supported capabilities; unsupported live actions stay unavailable.
+Native kick/mute command behavior and voice/text suppression require testing on
+the deployed Plutonium build. A disconnected host cannot receive a new ban or
+revocation until it reconnects.
+
+## Live actions and offline enforcement
+
+Live actions expire after 30 seconds and target the runtime's exact player
+connection. A reconnect, reused slot, changed generation or revoked moderator
+access prevents stale dispatch. Browser retries preserve the request identity.
+The native adapter journals one-time actions before dispatch and never executes
+the same action ID twice, including after restart.
+
+Queued actions can become delivered, confirmed, failed, expired or unknown.
+An expired undelivered action did not run. Unknown means delivery occurred but
+the result could not be established; do not assume the action failed. A durable
+late receipt can resolve an unknown outcome. Confirmation of a message means
+server dispatch, not proof the recipient read it. Kick confirmation requires an
+observed disconnect.
+
+The native runtime caches the complete applicable policy on persistent storage,
+scoped to the server, endpoint, generation and credential fingerprint. Cached
+bans/mutes continue through controller outages and expire using the host clock.
+Keep host time synchronized. A ban disconnects a player after connecting; it is
+not a pre-authentication admission filter. The shared connection handler and a
+500 ms reconciliation loop enforce it, retrying still-connected banned players
+every five seconds. No native `ban.txt` entries are rewritten.
+
+Preserve `tws-moderation/t5/` or `tws-moderation/t6/` in the game working directory.
+The cache includes an increasing runtime session counter that prevents an older
+process from replacing the active player roster. Losing or rolling back this
+cache requires a generation rebuild with a new credential before synchronization
+can resume; re-enabling the same integration does not reset the counter.
+Corrupt caches and failed durable writes stop new actions and require operator
+investigation. A controller restored to an older policy revision cannot silently
+replace a newer cached revision; review the restore and integration before
+resetting its cache or credentials.
+
+## Cases, evidence and review
+
+Create a case for a report, investigation or appeal. Each case has a dedicated,
+shareable page with the player identity, evidence, staff notes, linked restrictions,
+and an activity timeline. These pages require moderation access. Give a case a
+priority and assign it to an eligible administrator or moderator. Filter the case
+queue by status, priority, or assignment; active cases appear first, ordered by
+priority and most recent update.
+
+Use **Change status** to move a case between **Open**, **In progress**, and
+**Closed**, recording a reason for each transition. Assignment is separate:
+a case can have an owner at any stage. To resume a closed case, change its status
+to Open or In progress. Evidence, assessments, actions, and earlier decisions
+remain in its history. Editing case details does not change its status.
+
+Existing closed outcomes are migrated to Closed. Previously escalated cases
+become In progress. Their previous statuses are recorded in history; existing
+decision reasons, action references, assignments, and follow-ups are preserved.
+
+Use **Edit assessment** to keep a concise current summary, list the applicable
+rules, and record a finding for each concern: hate speech, harassment, threats,
+safeguarding, spam, cheating, or another concern. Each finding is **Substantiated**,
+**Inconclusive**, or **Unsubstantiated**, with a short rationale. Findings describe
+the evidence; case status tracks whether work remains. Record the decision and
+its rationale in the assessment summary.
+Every saved assessment retains its author, timestamp, and complete previous
+revision. Assessments can be revised after closure without reopening the case.
+
+Before opening a case, review existing cases for the same game and GUID. A case
+also shows same-player candidates; a shared archived match is a useful potential
+duplicate signal, not a finding that two reports are identical. Link cases as
+**Same incident**, **Appeal of**, or **Duplicate of**, recording why they belong
+together. Same-incident links may include different players; appeals and duplicates
+must concern the same game and player. Links appear from both cases with their
+direction made explicit. Self-links and circular appeal/duplicate chains are
+rejected. Removing a mistaken relationship requires a reason and preserves its
+history. Relationship changes do not close or reopen either case.
+
+Schedule a follow-up using a date and time, or **Review when a linked mute expires**.
+Custom date entry uses your browser's local time; displayed dates follow your time
+zone preference. Reviews remain scheduled after a case closes or reopens. Use
+**Overdue reviews** to find due work across open and closed cases, and clear the
+follow-up with a reason when the review is complete.
+
+In-app case updates notify the assignee and case creator when another staff member
+adds evidence or a staff note. Assignment changes notify the previous and new assignees
+and the creator, excluding the person making the change. **Your unread updates**
+is personal to your account. Open a case to review and mark existing updates
+read; notifications arriving afterward stay unread. Queue and case pages refresh
+every 15 seconds while visible. These are in-app notifications, with no email or
+external messaging delivery.
+
+Apply a ban or mute directly from an active case. The case fixes the target game
+and GUID and records the resulting restriction alongside the evidence. The same
+role, scope, and duration limits apply as elsewhere in Moderation. Applying a
+restriction leaves the case active so staff can finish reviewing it. Changing case status
+records workflow progress; it does not send a warning, create a restriction, or
+revoke one. Linked restrictions can be revoked separately with
+the required permission and reason, including after a case closes. Reopening a
+case also leaves its restrictions in place. Check enforcement coverage after
+applying or revoking a restriction.
+
+Use **Link existing action** to associate a warning, kick, private message, ban, or
+mute already recorded for this player with the case. This preserves the original
+action, author, time, delivery state, and expiry; it does not issue another action
+or extend a restriction. The same action can support more than one related case.
+Linking requires the current case revision and a reason, and appears in its timeline.
+Linked records show what actually happened, including failed or uncertain delivery.
+They remain visible when a case closes; status changes do not require choosing
+or re-entering an action.
+
+Staff notes are explicitly labeled as notes. In match chat, select up to 100
+messages and choose **Attach to case** to open a case or add them to an existing
+case for the selected subject. Select at least one message from that subject;
+messages from other participants can be selected alongside it. Each attachment
+preserves selected messages and, by default, three surrounding messages on either side, with
+names, GUIDs, original timing, and highlighted subject messages. The selection
+preview shows the excerpt before saving. Server verification uses stable archived
+message identifiers; repeated text remains separate occurrences, and attaching
+the same selected messages again does not duplicate evidence. Exact-text
+attachments remain available and include every matching occurrence, without the
+former ten-occurrence limit.
+
+Snapshots remain readable after source-match cleanup. Unknown or estimated
+timestamps remain such; the snapshot does not turn them into exact observations.
+Unarchived or already deleted chat cannot be attached as verified evidence.
+
+Open **Chat archive** from Moderation to include all recorded matches, including
+round-one games, interrupted games, and games missing an end report. Archive
+access requires moderation read permission. Filter by server, game, map, status,
+or date and open a match to select its preserved messages. Incomplete matches
+show their observed state rather than final results. Ordinary match history and
+leaderboard qualification are unchanged.
+
+Both public (`say`) and team (`sayteam`) log messages are captured. Team messages
+carry a **Team chat** label in chat and saved excerpts. Collection runs without a live
+viewer and checks surviving logs once after startup, including idle servers.
+Archive coverage still depends on the source log and accepted match telemetry:
+logs that were deleted or rotated before collection, or old clock segments that
+cannot safely be associated with a match, cannot be reconstructed. The collector
+retains a bounded log tail (up to 2,000 lines / 256 KiB); it is not a full historical
+log import. Already saved case excerpts survive later source cleanup.
+
+Case decisions, edits, and restrictions check the current revision; refresh before
+retrying a decision if another staff member has changed the case. Evidence,
+restrictions and append-only moderation events survive source-match cleanup and
+server deletion. They currently have no automatic retention cleanup.
+Authentication audit cleanup does not remove moderation history. Correct decisions
+by reopening, adding notes, or revoking restrictions; there is no history-erasing
+delete endpoint.
+
+Leaderboard disqualification, bank balance changes, automatic escalation and a
+public report/appeal portal are separate features. This workflow supports staff
+review of reports and appeals without exposing private evidence publicly.
+
+## Development and acceptance
+
+Controller tests cover permission boundaries, migrations, idempotency, exact
+instance credentials, stale connections, expired actions, late receipts,
+restriction expiry/revocation, policy coverage and evidence preservation. Agent
+tests cover private configuration delivery and secret redaction. Companion
+native tests exercise real policy/journal code and adapters with engine shims.
+
+Before deploying, build the x86 native plugins on Windows and validate both
+games: duplicate names, account reconnects, slot reuse, kick reasons, voice/text
+mute/unmute, overlapping restrictions, map/process restart, new server policy
+sync, expiry, revocation, controller outage and persistent-storage failure.
+Portable tests cannot establish the live Plutonium ABI or engine behavior.
+Package and offline-sign the agent release before publishing it; local builds
+remove any stale manifest signature and do not sign or deploy the release.

docs/guides/plugin-releases.md

@@ -40,7 +40,7 @@ GitHub release is imported from its detail page.
 ## Limits
 
 - Public repositories on github.com only; no GitHub credentials are needed.
-- Direct `.dll` release assets up to 64 MB; archives and source-code downloads
+- Direct `.dll` release assets up to 32 MiB; archives and source-code downloads
   are not extracted. Other formats can still use the manual upload workflow.
 - Release tags have the same 80-character limit as manually entered versions.
 - GitHub rate limits, missing releases, download failures and checksum failures

docs/guides/provisioning.md

@@ -0,0 +1,340 @@
+# Provisioning
+
+Open **Activity → Provisioning** for server queues, bulk cancellation, lifecycle
+actions, and retained lifecycle history. **Fleet → Collections** owns reusable lineups and collection launch
+progress; their individual operations remain visible in Provisioning.
+Provisioning owns the queues, safety checks, execution, and
+recovery for all six actions. Create a server using **Create server**. For an existing
+server, use its visible **Restart server** or **Start server** action, or open
+**More actions**. Only actions supported by current host and container evidence
+are shown. Confirmations name the server and its current players and round;
+disruptive actions still wait for a fresh empty-lobby check. **Manage server**
+on the operations page offers the same actions for a selected server.
+
+Provisioning badges show the recorded lifecycle state consistently on the list
+and job details: **Queued**, **Waiting**, **Running**, **Validating**, **Completed**,
+**Completed with warnings**, **Failed**, **Interrupted**, or **Cancelled**.
+**Validating** is the display name for `awaiting-health`. The list's Status column
+shows only the state badge. Open **Details** or the individual job to inspect
+execution progress and waiting reasons. File preparation can occur
+while a job is Queued or Waiting; Running begins when the host command is submitted
+and includes waiting for its acknowledgment. These progress details do not change
+the badge, status counts, or filters.
+
+The **Active** tab shows Validating jobs first, then Running, Waiting, and Queued.
+Within each state, older requests appear first, with recording order breaking
+timestamp ties. **History** shows the newest requests first. **Needs attention**
+filters unresolved Failed, Interrupted, and Completed with warnings outcomes;
+their individual state badges remain visible. This display order does not change
+execution priority or capacity admission.
+
+**Create server** opens one dialog with game and map choices followed by automatic
+readiness checks. Blockers explain what needs attention before launch. Runtime
+and network overrides are under **Advanced settings**; image, configuration,
+managed files, upload time, and internal server ID are under **Setup details**.
+The footer keeps **Cancel** and **Launch server** available while reviewing the
+setup. Create and launch reusable lineups separately in **Fleet → Collections**.
+
+Collection launches prepare their fleet preview automatically and show fill and
+exact modes together. Review changes and game key coverage, then use the
+persistent footer to queue work. Capacity estimates are advisory; jobs can wait
+for hosts and resources. Exact mode covers both games and requires confirmation
+of its explicit removal list, which resets whenever the preview is refreshed.
+See [Collections](collections.md) for the review details and safe submission
+retries when a response is lost.
+
+| Action  | Behavior                                                                                                                                                                                                  |
+| ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Create  | Create a new instance using the latest promoted files and verify container health, game status, the selected map, and the Plutonium revision.                                                             |
+| Start   | Start the existing container and verify container health and RCON.                                                                                                                                        |
+| Restart | Wait until empty, restart the existing container, and verify container health and RCON.                                                                                                                   |
+| Rebuild | Wait until empty, replace the instance through the host agent, preserving its server identity and port while advancing its generation. Verify container health, game status, map, and Plutonium revision. |
+| Stop    | Wait until empty, stop the container, and verify that the host agent reports it stopped. Keep the instance for a later Start.                                                                             |
+| Remove  | Wait until empty, remove the container and its files through the host agent, then retire the instance and registration.                                                                                   |
+
+## Safety and verification
+
+### Beta ownership and rebuilds
+
+Servers linked to a running or ready-to-promote beta cannot be removed or
+rebuilt. Promote or roll back the beta first, including paused or failed test
+plans. Fleet removal and exact collection launches queue nothing if any server
+selected for removal belongs to an active beta. Provisioning checks again before
+teardown; already queued removals wait while their server belongs to an active
+beta. Opening a beta also requires cancelling or finishing any pending removal
+or rebuild on its selected servers.
+
+Rebuild requires confirmation because **no instance data is preserved**. Its fresh
+files come from the latest promoted setup when creation begins, including changes
+promoted while waiting. Its assigned key in **Keys** must exist before teardown
+and stays reserved throughout the rebuild. Blueprint definitions, promoted
+artifacts, saved keys, and historical activity belong to Electric Cherry, and are
+not instance files.
+
+Rebuild reserves the original ID and port throughout teardown and creation, even
+when a lower lobby is available. It shares the existing creation queue and startup
+limit. The agent records each operation durably, and the controller reconciles its
+receipt after reconnecting or restarting. An interrupted fresh startup can resume
+health verification without repeating the operation. Inspect failures before
+making another request.
+
+### Health and runtime checks
+
+If the host agent explicitly reports an unhealthy container during Create or
+Rebuild verification, that operation forces one restart, then forces one rebuild
+if the container remains unhealthy. Recovery preserves the selected launch files,
+credentials, server identity, and port. If both attempts leave it unhealthy, the
+operation fails for inspection instead of repeating recovery. Starting, restarting,
+and unknown container states continue waiting for readiness. This requires host
+agent 1.8.1 or newer.
+
+Create and Rebuild finish only after RCON reports a runtime revision at least as
+new as the current Plutonium production release. The worker checks the production
+manifest and records the runtime revision for fleet and server version status as
+part of provisioning. An unavailable release or unknown runtime revision keeps the
+job awaiting verification and holds its startup slot while the worker retries.
+A confirmed older runtime fails the operation with its expected and observed
+revisions; it does not trigger an automatic restart. Check Plutainer's startup
+updater configuration and server logs before retrying. Telemetry can verify game
+status and map, but cannot verify the Plutonium revision.
+
+Start and Restart share the rolling restart limit with campaigns, betas, and
+maintenance. A successful agent receipt is followed by actual container and game-status checks. Failed or ambiguous restart requests retain their recovery
+slot and are not automatically replayed. Stop requires an explicit stopped container state; an
+unreachable host agent is not evidence that a stop succeeded.
+
+### Game key pools
+
+Each create reserves one unused Plutonium key for its game before the launch is
+queued. T5 and T6 have independent pools; any map or lobby within that game can
+use a free key. Stopping a server keeps the reservation, and a rebuild keeps the
+same key. Confirmed removal releases it for another launch. Failed or ambiguous
+creation keeps the assignment while cleanup may still be required.
+
+An unstarted creation cancelled before dispatch releases its key immediately.
+Administrators can add and remove available keys in **Admin → Keys**;
+assigned keys cannot be removed. Existing keys are migrated into their game
+pool, retaining assignments for existing servers and pending work. If a legacy
+rebuild has an uncertain result and used a different key, both possible keys
+remain reserved until that server is removed. Labels are for identification
+only and do not change the hostname registered with Plutonium.
+
+### Player checks and administrator overrides
+
+Disruptive operations require a fresh zero-player result from RCON or accepted
+telemetry, unless the host agent positively reports a stopped container. Fresh occupied
+telemetry vetoes a conflicting empty RCON reading. Reports must be received within
+90 seconds; recovery verification also excludes reports from before the submitted
+operation. Telemetry can verify the game's activity and selected map after the host agent
+reports healthy. Runtime actions wait for conflicting
+campaign, beta, maintenance, or provisioning work. Duplicate pending requests for
+the same action return the existing operation; a different action returns a
+conflict. Requests and events survive application restarts.
+
+Rebuild and recovery within Create or Rebuild treat an explicitly unhealthy
+container as empty without requiring a player count. The agent checks health
+again before disruption; if the container has recovered, normal player checks
+apply. This exception does not apply to unknown health or other lifecycle actions.
+
+For a failed container stuck waiting for player status, an administrator can open
+its queued or waiting **Restart**, **Stop**, **Rebuild**, or **Destroy** operation
+and choose **Admin override…**. The confirmation modal explains the interruption
+and deletion risks. Confirming bypasses player-count and empty-server checks in
+both the controller and host agent for that operation only. The approval persists
+across restarts and records the administrator and time in the operation history.
+The host agent must support overrides (version 1.4.15 or newer); older agents show
+an update requirement. Submitted commands, host migrations, and completed operations
+cannot be overridden. Capacity, conflicting work, instance identity, file validation,
+and post-action health checks still apply.
+
+## Campaigns, betas and maintenance
+
+Campaign and beta file changes submit durable rebuild requests to
+Provisioning and observe the resulting operation by ID. Their pages link to that
+operation, and its history links back to the requesting work. Provisioning owns
+the empty-server check, conflict handling, capacity limits, remote command, and
+health verification. The callers retain file deployment, beta evidence, and
+maintenance outcome verification. Jobs remain visible while waiting for players,
+other work, or integration configuration.
+
+File rebuilds require host agent 1.9.2 or later. They capture rollback snapshots
+before replacing the instance, and install the requested files over the promoted
+setup. Queued file activations wait for the agent update when necessary.
+Maintenance jobs and beta recovery cycles without file changes still use restarts.
+
+[Maintenance](maintenance.md) also submits recovery operations and links each attempt
+to its incident. A failed Maintenance restart can hand its reserved slot to a rebuild
+after five minutes; unrelated recovery slots remain held. Drift campaigns follow
+their linked rebuild without requesting another restart. Their file evidence and
+history remain available after the instance is removed.
+
+Each campaign activation has its own durable operation and file selection. A failed or interrupted request keeps
+its original ID and never automatically replays. An explicit beta retry starts a
+new attempt, preserving prior history; paused or closed beta work cannot start a
+queued restart. Recovery of an already submitted restart continues even if its
+requester closes. Upgrades adopt existing restart intent for observation only.
+
+## Operation details
+
+Each job page puts its recorded outcome, blocking condition, and available next
+action before launch files. Execution is grouped into phases, with separate
+milestones for remote deletion, local cleanup, and replacement creation during a
+rebuild. Copy the operation ID or link when investigating or handing off a job.
+Related work links back to the requesting campaign, beta, maintenance item, or
+collection. A completed provisioning job does not imply that its requester has
+finished its own verification.
+
+Optional timestamps appear only when recorded. **Launch files** appears for Create
+and Rebuild, or when an older operation has a recorded recipe. Start, Restart,
+and Stop use the existing instance without selecting new launch files.
+
+While a job is active, **Controller activity** lists current work and elapsed
+time. Background scans run independently of provisioning.
+
+### Verification evidence
+
+New operations record structured verification evidence alongside their activity:
+the check result, source, evaluation time, and expected/observed map or Plutonium
+revision when relevant. Create and Rebuild show **Verify Plutonium revision**, with
+separate evidence for the production manifest and the running revision from RCON.
+Game status can be verified by RCON or accepted fresh telemetry. Engine-map checks
+do not independently verify a T6 start location. Passed checks describe their
+recorded observation; later checks can fail or become unknown. Completion describes
+what was verified at that time, while **View server** opens the current instance.
+Older history retains its recorded stages and outcomes; unavailable evidence is
+not inferred from stage wording.
+
+Submission attempted, command accepted, and verified completion are separate
+milestones. A failed restart may still have ongoing recovery observation and hold
+restart capacity. Its original failure remains in history even if recovery later
+succeeds and releases the slot. Capacity and request order are snapshots, not
+guarantees of the next job admitted or an estimated completion time. Blocked
+requests can be bypassed by eligible work.
+
+### Recovery actions and launch files
+
+Recovery controls use the server's current eligibility checks. **Resume health
+verification** observes the existing instance without submitting another create or
+restart. **Resume cleanup** retries local cleanup after confirmed remote deletion,
+including eligible historical deleted identities; replacement or newer-operation
+guards still apply. **Resume rebuild cleanup** completes cleanup before creating
+the replacement in its reserved slot. Resuming the same operation preserves its
+earlier failure and resume events. Uncertain commands have no generic Retry action.
+
+Cancelling an operation does not stop a command already delivered to the host.
+The confirmation explains that remaining work and health checks stop, while
+changes already made remain. **Cancel creation and remove server** also deletes
+the launch's container and instance files, even if players have joined or their
+status is unknown. Shared game files are kept, and the key returns to its game
+pool after confirmed removal.
+
+A later rebuild waits for that command to settle. Its waiting message includes
+the host command's state and last reported error; **View cancelled operation**
+opens the original command details. If an interrupted rebuild removed its old
+container before creating the replacement, the host can confirm failure after
+verifying that both containers are absent and the instance identity is unchanged.
+The original diagnostic remains available, and the queued rebuild can continue.
+Uncertain inspections keep it waiting; the old command is never replayed merely
+because its receipt was lost.
+
+Create and rebuild capture a setup when queued and select the latest promoted
+files again when creation begins. The **Launch files** card shows one selection
+status beside the capture time: **Captured when queued**, **Selected when creation
+began**, or **Recorded setup; selection time unknown** for older records. A queued
+capture does not pin versions; an attempted selection is not proof that every
+file was installed. Expand the managed file list to inspect versions, paths,
+checksums, and the recipe fingerprint.
+
+### History and refresh behavior
+
+Five recent events remain visible. Expand full history to search, change order,
+and load older pages. A fixed history watermark keeps the viewed pages stable as
+events arrive; **Jump to latest** incorporates new activity. The detail view
+fetches a bounded recent-event window, while full history loads on demand.
+
+The page refreshes every five seconds while visible, including terminal jobs whose
+recovery or available actions may still change. **Page updated** describes the
+last successful detail fetch; it is separate from the worker's evaluation and
+telemetry receipt times. A refresh failure keeps the last snapshot visible and
+does not erase an acknowledged action result. These timestamps are not worker
+heartbeat or completion-time promises.
+
+## Routes and compatibility
+
+- `/provisioning`: active operations and history filtered by action.
+- `/provisioning/:id`: operation stages, errors, timestamps, and activity.
+- `/collections`: saved lineups, active launches, and launch history.
+- `/collections/launches/:id`: collection progress, with links to each operation.
+- `GET /api/provisioning`: lifecycle history (`deployments` response property).
+- `POST /api/provisioning`: queue `{ "server": "t5_zombie_theater_1", "action": "start" }`.
+  Supported actions are `start`, `restart`, `rebuild`, and `stop`. Returns HTTP 202
+  with `deploymentId`. Create continues through `POST /api/servers`; Destroy uses
+  `DELETE /api/servers/:name`.
+- `/api/provisioning/:id`, `/:id/resume`, `/:id/cancel`, `/preview`, and
+  `/house-party` preserve the existing operation, recovery, removal cancellation,
+  launch preview, and fleet removal APIs.
+- `POST /api/provisioning/:id/cancel` cancels any active create, start, restart,
+  rebuild, stop, or destroy job, including health verification. It releases that
+  job's recovery capacity and prevents further steps; completed effects are not
+  undone. Undelivered host commands are withdrawn. Commands already delivered
+  retain their receipts, and replacements wait for those mutations to settle.
+  Cancelling is idempotent. Lists and details expose `canCancel`.
+- `POST /api/provisioning/:id/cancel-creation` retains the explicit native creation
+  cleanup action: cancel the launch and remove its partial instance. Cancelling
+  a job alone retains its existing instance/files for inspection or later cleanup.
+- Beta rollback cancels conflicting provisioning jobs on its original servers
+  before queuing independent restoration restarts. It does not wait for the
+  superseded restart's health verification. Native activation receipts must still
+  provide durable before images; rollback reconciles completed receipts itself.
+  Manual rollback keeps the replacement jobs held until the campaign is resumed.
+- `GET /api/provisioning/:id?events=recent` returns the same detail with five recent
+  events plus activity totals/latest ID and a failure excerpt. Omitting the option
+  preserves the full legacy event response. Progress, capacity, named related work,
+  and normalized warning messages are additive fields; the original warning string
+  and lifecycle fields remain available.
+- `GET /api/provisioning/:id/events` reads history with `order=newest|oldest`,
+  literal-text `query`, `limit` (default 25, maximum 100), an exclusive event-ID
+  `cursor`, and an upper-ID `watermark`. Keep the returned watermark when loading
+  more and use `nextCursor` until it is null. Begin without a watermark to include
+  newly arrived events. These reads do not contact host agents or probe RCON.
+
+Old `/deployments` pages redirect to their `/provisioning` equivalents, preserving
+query parameters. Collection launch links at `/provisioning/collections/:id` or
+`/deployments/collections/:id` redirect to `/collections/launches/:id` and preserve
+query parameters. `/api/deployments` remains compatible, including old campaign
+detail endpoints. No history or IDs are renamed or discarded.
+
+The standalone Activity log has been retired. Use a server's **History** tab for
+its combined history, **Activity → Maintenance → Activity** for maintenance and
+recovery, **Releases → Campaigns** or **Releases → Betas → History** for release
+history, and **Fleet → Hosts → Operation history** for retained host operations.
+Host operation history remains available after a host is deleted.
+
+Legacy `/activity` and `/operations` bookmarks redirect to Provisioning history
+by default. Server-scoped bookmarks open the server's History tab; host-scoped
+bookmarks open host operation history. Source-only bookmarks open the matching
+Campaigns, Betas, Maintenance, host operations, or autoscaling provisioning
+history. Shared history filters are retained when opening server or Maintenance
+history; pagination is reset when switching to a different kind of history.
+`GET /api/activity` and the shared history views remain available. Historical
+records, event collection, and retention are unchanged.
+
+Start and Stop use durable `instance.start` and `instance.stop` host-agent
+operations. Rebuild uses `instance.rebuild` and replaces the instance files.
+See [native host management](native-hosts.md) for command delivery and recovery.
+
+## Concurrency limits
+
+Admin → Settings groups Concurrency separately from history and storage settings,
+with one Save changes action for all pending edits. Existing fleet restart, startup
+and removal limits continue to apply. Additional per-host restart, startup and removal limits, a combined
+lifecycle limit, a rebuild limit, and a limit on hosts preparing game files default
+to 0 (no additional cap), preserving existing throughput on upgrade.
+
+Lifecycle work counts once toward the combined limit, including launch preparation
+and unresolved delivered commands. Rebuilds include drain migrations and retain
+capacity between source removal and destination creation. Per-host recovery slots
+remain occupied until health verification. Lowering limits does not cancel work.
+Game-file preparation is limited when commands are delivered; downloads and imports
+share the cap, including manual requests. Replayed or interrupted commands retain
+the same host slot. Runtime image pulls and per-host file transfer counts are unchanged.

docs/guides/settings.md

@@ -0,0 +1,149 @@
+# Fleet settings
+
+Admin → Settings groups fleet limits and retention by the work they affect.
+Search by name or description, or choose a category in the section list. Each row
+shows its default, effect, and whether it has changed. Switching categories or
+searching keeps your pending edits.
+
+The sticky **Save changes** bar saves all pending changes together. **Discard**
+restores the last saved values. Settings persist in SQLite, apply without restarting
+the controller, and are writable only by administrators. Upgrades preserve existing
+values and add new defaults.
+
+Use **Forever** for history with no age expiry and **No limit** for optional
+concurrency or telemetry count caps. **No age expiry** on artifact versions still
+allows the version-count policy to remove old unused versions. **No delay** applies
+to the game-file grace period; **No minimum** applies to minimum retained counts.
+Turning these modes off restores the last finite value. Numeric fields show the
+unit beside the value.
+
+## Concurrency
+
+The existing fleet restart, startup and removal limits default to 1. Additional
+limits default to 0 (no additional cap):
+
+- Restarts per host, including starts and rebuild recovery.
+- Startups per host, including creation and rebuilds.
+- Removals per host, including migration source removal.
+- Combined lifecycle work across create/start/stop/restart/rebuild/destroy.
+- Rebuilds across the fleet, including host drain migrations.
+- Hosts downloading or verifying game files simultaneously, including manual imports.
+
+Limits are checked again before command submission after asynchronous preflight.
+Launch-file preparation reserves capacity. A lifecycle workflow counts once in the
+combined pool; unresolved commands and recovery keep capacity occupied. Migrations
+keep rebuild capacity through source removal and destination creation. Lower limits
+do not cancel admitted work. Game-file preparation slots are assigned on agent
+command delivery and survive interrupted transfers. Runtime pulls, the four file
+transfers per host and the two health probes per host remain unchanged.
+
+## Retention
+
+For age settings, 0 means no expiry. An explicit version-count policy can still
+expire unused artifact versions when their age policy is disabled. Game-file grace
+is a delay: 0 adds no delay. A positive age and version count together preserve the
+minimum count as well as versions younger than the age threshold.
+
+| History or storage                             | Default                                         |
+| ---------------------------------------------- | ----------------------------------------------- |
+| Campaigns                                      | 7 days                                          |
+| Successful routine operations                  | 1 day                                           |
+| Provisioning workflows                         | Indefinite                                      |
+| Agent actions                                  | Indefinite                                      |
+| Explicitly resolved failures                   | Indefinite                                      |
+| Non-update maintenance jobs                    | 30 days                                         |
+| Resolved recovery incidents                    | Indefinite                                      |
+| Host resource history                          | 7 days                                          |
+| Instance resource samples                      | 1 day                                           |
+| Fleet occupancy metrics                        | 31 days                                         |
+| Telemetry diagnostics                          | 7 days; 200 per server; 10,000 overall          |
+| Completed/interrupted matches                  | Indefinite                                      |
+| Closed betas and test evidence                 | Indefinite                                      |
+| Campaign file bodies and closed beta snapshots | 30 days                                         |
+| Completed launch and command file bodies       | 30 days                                         |
+| Unused artifact versions                       | No automatic expiry; no minimum count           |
+| Unused game files                              | No grace period; keep latest 1 version per game |
+| Incomplete browser uploads                     | Indefinite                                      |
+| Authentication audit events                    | Indefinite                                      |
+| Minimum history count                          | 0 additional records                            |
+
+Telemetry age and count policies are independent; 0 disables each individual cap.
+Diagnostic responses remain bounded to 10,000 rows when the per-server retention
+cap is disabled. Metric and capacity chart time ranges are unchanged; increasing
+storage retention does not automatically add new chart ranges. Capacity learning
+continues to use its 24-hour window independently of longer sample retention.
+
+### Cleanup and dependencies
+
+Background cleanup runs in bounded batches. New age policies for previously
+permanent history default to indefinite. Pending work, unresolved failures,
+recovery dependencies and explicitly protected records are never removed merely
+because they are old. Referenced artifact versions are retained, including versions
+referenced by historical campaigns and betas. Collection workflows with unfinished
+members, cancellation links and recovery slots can keep provisioning history longer
+than its configured age.
+
+Update attempts and dismissed watchdog incidents remain durable suppression evidence.
+Host setup, credential/decommission operations and referenced agent commands are
+protected. Expired unreferenced action history drops payloads, results and events
+but keeps compact terminal receipts and request fingerprints. Reusing an idempotency
+key cannot execute an expired command again. Agent mutation journals remain intact;
+this policy controls controller history, not the agent's execution recovery journal.
+
+### File bodies and match history
+
+Expiring campaign file bodies preserves file paths, hashes and summary records but
+makes file comparisons and rollback unavailable. The detail page identifies this
+state and rollback requests fail explicitly. Active beta rollback images stay
+protected. Expiring matches removes their player participation rows and changes
+leaderboard/player totals; lifetime aggregates are not retained separately.
+
+Completed launch recipes and command receipts also retain their metadata when
+their file bodies expire. Unfinished workflows, unreconciled commands, unresolved
+failures and explicit holds preserve the bytes they need. Shared file bytes are
+deleted only after their final reference expires; library versions can keep a file
+available longer. Previously saved retention choices are preserved on upgrade.
+
+### Database cleanup
+
+The database upgrade normalizes retention clocks to UTC and enables incremental
+vacuum. An existing database without a vacuum page map is rebuilt once at startup;
+allow time for this rebuild and free disk space up to twice the database size.
+The rebuild uses disk-backed temporary storage and a 16 MiB page cache instead
+of holding a temporary copy of the database in RAM. Container scratch files use
+`/data/.sqlite-tmp`; `[database]` startup logs identify the current migration.
+Normal cleanup releases unused file blobs in bounded batches and incrementally
+returns free pages to disk after larger prunes.
+
+### Game files and incomplete uploads
+
+Game-file minimums apply per game. Grace starts when a version is superseded.
+Live, stopped, rebuilding and provisioning instances pin their files; unresolved
+operations and offline host copies also prevent controller deletion. Controller
+cleanup still waits for every host to report its copy absent. Incomplete browser
+uploads expire based on their last activity, including resume checks; active chunk
+writes and publication hold locks against cleanup. Metadata remains after files
+expire. Raising retention cannot restore previously deleted bytes.
+
+### Preview and record protection
+
+Expand **Cleanup preview and protected records** to preview the values currently
+entered without saving them. Preview lists eligible records and known payload
+sizes; SQLite file space may be reused rather than immediately returned to the OS.
+Policies can overlap, so payload sizes should not be added together. Unknown row,
+index, upload and agent-disk overhead is not included.
+
+Use a record's ID from its detail URL or the preview to protect it. Protection can
+be removed in the same panel. Protecting a campaign or beta also protects its file
+bodies; protecting those bodies prevents deletion of their parent history.
+Game-file and incomplete-upload protections apply to storage cleanup.
+
+The minimum history count preserves recent records per history category. It does
+not apply to telemetry caps, metric/resource samples or game-file storage, which
+have their own policies. Explicit holds protect individual history/storage records,
+not individual resource samples or telemetry diagnostics.
+
+Failed campaigns, provisioning workflows and agent commands require explicit
+resolution before the resolved-failure period applies. Resolution does not bypass
+recovery/dependency checks. Resolving a failed campaign stops automatic campaign
+retry requests; it does not cancel commands already submitted.

docs/leaderboard.md

@@ -1,94 +0,0 @@
-# Leaderboard
-
-Open **Leaderboard** in the main navigation to compare players across the fleet,
-or filter by game and map. Choose a ranking, search a name or GUID, and share the
-page URL with its filters intact. Results refresh while the page is visible.
-Matches only appear in server/global history and leaderboard map options once
-they reach round 2. Player records only contribute to leaderboards, summaries,
-profiles and player history once that player is observed at round 2 or later.
-This also excludes existing round 1 records. Early telemetry is retained so
-qualifying games keep their start time, roster and cumulative counters.
-
-Click a player's name to open their shareable profile at
-`/leaderboard/players/[game]/[guid]`. Profiles show all ten lifetime statistics,
-headshots as a percentage of kills, kills per down, average kills and observed
-rounds per game, and completion rate. Each map has its own complete set of
-records; match history shows individual counters, server, date and completion
-credit in pages of 25 games. Map names link back to that map's leaderboard.
-Use **Refresh stats** to load the latest reports. Ratios with a zero denominator
-display a dash. Profiles always cover all recorded maps for that game and player.
-
-The leaderboard uses the existing three telemetry events; no reporter change is
-required. Each player is identified by **game + GUID**, with their latest reported
-name displayed. Different names for the same GUID do not split their records;
-identical names with different GUIDs remain separate. GUIDs stay strings.
-
-## Recorded statistics
-
-| Statistic | Meaning |
-| --- | --- |
-| Highest round | Highest round reported with that player present, including players who joined mid-game. |
-| Total kills, headshots, revives, downs | Sum of each player's highest reported counter in each game. Round snapshots are cumulative, so they are never added together within a game. |
-| Total rounds | Round progression during observed participation, added across games. A player observed at game start and then round 8 receives seven rounds. A player first seen at round 8 gets a highest-round record of 8, but zero total rounds; reaching round 9 then adds one. |
-| Games played | Games in which a stats report at round 2 or later contains the player. Start-only and round 1 participation do not count. |
-| Games completed | Games whose `game_end` roster contains the player. Disconnected players retain earlier stats without receiving completion credit. |
-| Best game kills | Highest kill count recorded in any one game. |
-| Highest reported score | Highest observed score balance. Spending points can lower the score, so this is not cumulative points earned. |
-
-Every ranking sorts highest first, including downs (a count, not an efficiency
-rating). Equal values share a rank; name and GUID provide a stable display order.
-Searching preserves the player's rank within the selected game/map. Summary
-cards describe that game/map, regardless of the search text. Summary rounds are
-player-rounds: four players progressing from game start to round 8 contribute 28.
-Highest-round records always use the round reached, including the final round
-and players who joined mid-game.
-
-Round totals require the player to appear in consecutive observations. If a
-report omits them, their return starts a new participation interval and the gap
-is not credited. The payload cannot tell whether a present player was alive,
-spectating, or absent between reports. Mid-game snapshots still contribute their
-reported kills and other counters, even when earlier participation is unknown.
-
-## Persistence and match boundaries
-
-Match and player records live in SQLite alongside the rest of Electric Cherry.
-They persist across app restarts and server deletion, and are not subject to
-campaign history retention. The latest scoreboard and leaderboard update in the
-same transaction. Each match keeps player counters even after someone leaves.
-
-A `game_start` opens a match; `game_end` completes it. A changed map, a lower round,
-or a round report after a completed game can also establish a new match when the
-start was missed. Superseded active matches remain in history as interrupted.
-Repeated starts before any stats belong to the same match. Identical consecutive
-payloads (ignoring receipt time and roster order) do not add records. Repeated
-snapshots in a match cannot increase counters or rounds twice.
-
-The first upgrade imports the latest stored full scoreboard from each server
-once. That provides the available counters and highest round, but not discarded
-earlier games or unobserved round progression. Old round-only snapshots have no
-player identity and cannot be imported. Tracking continues from future reports.
-
-T6 selections sharing an engine map, such as Town Survival and TranZit, remain
-separate using the server's registered map variant. Unknown variants remain
-explicitly identified instead of being labeled as a known selection. A match
-retains the game/map identity recorded at ingestion even if the server is later
-reconfigured or removed.
-
-The current protocol sends events in gameplay order and has no match ID, event
-ID, or source sequence. A late report from an older game cannot be reliably
-distinguished from a new game. Likewise, identical starts before any stats are
-indistinguishable from retries. Send reports once in order. Within a match,
-counter decreases use the previous high-water mark; if a reconnect resets a
-player's counters, subsequent activity may be undercounted until those counters
-pass their earlier maxima. Exact replay handling and counter-reset accounting
-would require additional reporter identity/ordering information.
-
-## Read API
-
-`GET /api/leaderboard` accepts optional `game` (`t5` or `t6`), `map` (the opaque
-key returned in `maps`), `metric` (default `highestRound`), `q` (name/GUID search),
-`limit` (1–100, default 50), and `offset` (default 0). It returns ranked `entries`,
-available `maps`, the matching player `total`, pagination values, and `summary`.
-
-Metric keys are `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives`,
-`downs`, `gamesPlayed`, `gamesCompleted`, `bestKills`, and `bestScore`.

docs/metrics.md

@@ -1,16 +0,0 @@
-# Metrics
-
-Metrics is a primary navigation tab at `/metrics` (keyboard shortcut: `g m`).
-Choose the past 24 hours, 7 days, or 30 days. The page refreshes every minute.
-
-- Utilized means a successful status response reported at least one player.
-- Idle means a successful response reported zero players.
-- Unknown covers failed checks, provisioning, and unavailable containers; it is excluded from utilization percentages.
-- Player-hours integrate the reported player count over observed time. Bots are included if the server counts them as players.
-- All lobbies full measures map wall time when every tracked lobby in that game/map group has a known capacity and is full. Missing capacity or status makes that interval unknown. This is a capacity-pressure indicator, not a count of rejected joins or unmet demand.
-
-Observations are recorded by the existing status worker. Each remains valid until the next collection, up to the 45-second freshness limit. Longer manager outages are excluded. Charts split time at UTC hour/day boundaries; partial periods remain partial. Map variants use the configured variant when it matches the reported map.
-
-Capacity comes from `sv_maxclients` / `sv_maxClients` in the status response. When unavailable, the UI displays unknown rather than assuming a four-player limit. Capacity coverage is the map time for which every tracked lobby reported both occupancy and capacity.
-
-Collection begins after this feature is installed; previous occupancy cannot be reconstructed. Samples are retained for 31 days and pruned during collection. They survive removal of a server and contain no player identities. The additive database table is created automatically on startup.

docs/plans/lobby-autoscaling-plan.md

@@ -0,0 +1,161 @@
+# Lobby autoscaling implementation plan
+
+Status: Approved and implemented October 6, 2026. Controller support and agent 1.7.0 are complete; production pilot measurements and live rollout remain pending. See [usage and the pilot procedure](../guides/autoscaling.md).
+
+The design below preserves the approved scope and subsequent implementation
+record. Use the linked guide for current policy behavior, including game-wide
+policies and shared game key pools added after the initial design.
+
+Introduce an opt-in policy for each game and map variant that keeps a small number of empty, ready lobbies available. When the last spare fills, request another lobby through Provisioning. Remove surplus lobbies only after sustained, confirmed inactivity. This reduces idle hosting while preserving room for new groups, including when existing Zombies sessions have free player slots but are already well into a game.
+
+Start with Create/Destroy on existing native hosts. Run the evaluator in dry run for two weeks, then enable creation for a small pilot before enabling removal. Host acquisition, VM shutdown, Stop/Start pooling, scheduled demand prediction, and waking servers from incoming packets are outside the first release.
+
+Update October 7, 2026: Dry run and Create only have been removed. Current policies support Off and On, defaulting to Off. Existing policies in removed modes switch to Off on upgrade. Action cooldowns have also been removed; policies retain the continuous empty duration before removal and one outstanding operation at a time. Policies now save directly with server-side validation; no preview or confirmation token is required. The original implementation stages below record the initial design; follow the current [usage and pilot procedure](../guides/autoscaling.md) for operation.
+
+## Existing foundations and required extensions
+
+| Foundation            | Reuse and required work                                                                                                                                                                                                                                                                                        |
+| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Background worker     | Add a requester to [pipeline/worker.ts](../../src/lib/server/pipeline/worker.ts), which already reconciles background work every 15 seconds independently of Provisioning. Keep host execution inside Provisioning.                                                                                            |
+| Occupancy evidence    | Reuse [status evidence](../../src/lib/server/fleet/status-evidence.ts) and shared freshness rules. Monitoring runs on a 30-second cadence; accepted evidence currently expires after 90 seconds. Positive occupancy vetoes an empty claim. Unknown is never zero.                                              |
+| Safe creation         | Use the durable creation intent and placement path in [provisioning/native.ts](../../src/lib/server/provisioning/native.ts). Preserve CPU, memory, port, host eligibility, library, and startup concurrency checks. A blocked create remains queued.                                                           |
+| Safe removal          | Reuse the controller occupancy checks, instance generation checks, host-local check, and cleanup checkpoints. Add autoscaling-specific admission and cancellation behavior before enabling automatic destruction.                                                                                              |
+| Requester eligibility | [provisioning/worker.ts](../../src/lib/server/provisioning/worker.ts) currently gives Create and Destroy separate paths that bypass the runtime requester eligibility check. Autoscaling ownership and policy checks must cover both paths, including delayed execution. A source label alone is insufficient. |
+| Saved keys            | [server-keys.ts](../../src/lib/server/fleet/server-keys.ts) stores keys in game pools. Each live or pending launch reserves a key until cleanup confirms it is safe to reuse. Allocate lobby identities numbered 1–20 independently of the key label.                                                          |
+| Collections           | [collections.ts](../../src/lib/server/fleet/collections.ts) already counts queued creations and launches promoted recipes. Exact matching can remove lobbies across the fleet, including maps omitted from the collection. Add ownership checks to preview and submission.                                     |
+| Hosting insights      | [metrics-insights.ts](../../src/lib/metrics-insights.ts) provides historical suggestions for hosting trials. Use these to help choose policies; the live evaluator needs current empty-lobby evidence and must not execute historical suggestions directly.                                                    |
+
+Create already uses the promoted launch recipe and capacity-aware placement. Start retains existing files and has recovery concurrency controls, but would need equivalent CPU and memory admission before becoming an autoscaling strategy. The proposed scope follows option 4 in [the idle CPU research](../research/t5-t6-idle-cpu-research.md).
+
+## Policy and ownership
+
+Use one policy per `(game, mapSlug)`, preserving configured variants such as T6 start locations. Do not group ambiguous legacy records by engine map alone. Resolve or exclude ambiguous identities before activation.
+
+| Setting                       | Proposed initial behavior                                                                                                                    |
+| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
+| Mode                          | Off, Dry run, Create only, or Create and destroy. Default Off.                                                                               |
+| Minimum running lobbies       | Default 1; a floor for desired ready capacity, subject to available keys and host capacity.                                                  |
+| Maximum lobbies               | Required explicit limit, at most 20. Count occupied lobbies, registered identities, reservations, and unresolved creations toward admission. |
+| Empty spare target            | Default 1. Count empty ready lobbies, not unfilled player slots.                                                                             |
+| Empty duration before removal | Default 15 minutes; adjustable after the pilot.                                                                                              |
+| Action cooldown               | Default 5 minutes between new policy actions, measured from durable admission. Cancellation and safety checks do not wait for cooldown.      |
+| Launch settings               | Reuse normal map recipes and automatic placement on eligible active hosts. Use current promoted files when provisioning begins.              |
+
+Validate integer counts with `1 <= minimum <= maximum <= 20` and `1 <= spare target <= maximum`. Show the game pool’s saved and available counts independently of each map’s maximum. Policies can be saved with fewer keys than the maximum, or with no keys yet. Exhaustion holds growth across every map sharing that game, while eligible surplus removal can still return capacity. Game-wide targets are per map, but game keys are shared across all maps and launch paths.
+
+Live activation explicitly hands management of the game's map variant to the policy and adopts its eligible existing native lobbies. Show those lobbies, exclusions, current keys, and the first proposed action in the activation preview. Do not activate across unresolved provisioning, collection launches, or ambiguous identity ownership. Dry run observes without acquiring ownership or reserving names, ports, or capacity.
+
+While a policy is live, reject collection launches that would create or remove its lobbies. In exact mode, check the entire removal set, including managed maps absent from the collection. Apply the same check to manual Create, Start, Stop, Rebuild, and Destroy requests that compete with the policy. Explain which policy must be paused, with a link. Existing operator cancellation remains available; cancelling an autoscaling job pauses the policy so it cannot immediately recreate that intent.
+
+Pause the entire policy's decisions while a member is in an active beta, campaign, maintenance operation, host drain, or unresolved recovery. This conservative first-release rule avoids treating temporarily unavailable lobbies as new demand. Those workflows can proceed through their existing gates; autoscaling resumes only after they settle and fresh observations are available. Repeat these checks during Provisioning admission because a conflict can begin after evaluation.
+
+## Counting and decisions
+
+Build a bounded snapshot per map from current identities, evidence, and operations. Deduplicate a registered lobby and its creation job by identity. Classify each member as occupied and ready, empty and ready, creating, removing, stopped, unknown or unhealthy, or protected by other work. Queued and starting lobbies reserve capacity but are not usable spares.
+
+The first release holds new scaling actions whenever an existing member has unknown occupancy, is stopped or unhealthy, or has unresolved lifecycle work. Record the blocker and leave recovery to existing maintenance and operator flows. A confirmed empty map with no registered members can bootstrap from its minimum; it does not need occupancy observations for nonexistent servers.
+
+When the snapshot is complete and eligible, calculate:
+
+```text
+desired running lobbies = min(maximum, max(minimum, occupied ready lobbies + spare target))
+```
+
+This is a bounded target, not a promise that capacity is immediately available. If occupied lobbies consume the maximum, keep them running and report that the spare target cannot be met. Reducing the maximum below the existing count never authorizes removing an occupied lobby.
+
+Evaluate in this order:
+
+1. Reconcile existing autoscaling intent and any unsettled host commands. Cancel an eligible queued removal if players arrive, evidence becomes unknown, the policy changes, a conflict appears, or removal would breach the minimum or spare target.
+2. Apply mode, ownership, evidence, conflict, failure, and cooldown gates. Never submit a second policy action while a prior one remains unresolved. Count external reservations as well as policy jobs against the maximum.
+3. If the ready count is below the desired count, propose one create using the lowest available lobby identity and any free key from that game. Recheck identity availability, game key availability, and the policy revision atomically when persisting the request. Admission rechecks recognize the pending launch’s own held key. Capacity waiting must reuse this request rather than add another each tick.
+4. If the ready count exceeds the desired count, select one continuously empty eligible lobby whose removal preserves both the minimum and spare target. Prefer the longest-confirmed empty lobby, with lobby number as a stable tie-breaker. Create only mode records this opportunity without executing it.
+5. Otherwise record a stable no-action reason. Deduplicate unchanged decisions so the 15-second timer does not flood history.
+
+For example, with minimum 1, maximum 4, and one spare, two occupied lobbies plus one empty lobby need no action. If that empty lobby fills, request a fourth. When only one lobby remains occupied and three are empty, the target becomes two; remove surplus empty lobbies one at a time after their empty duration and action cooldown permit it.
+
+Track continuous emptiness against the exact instance generation, using fresh observation timestamps. Re-reading the same sample does not extend the observation history. A positive count, unknown interval, lifecycle transition, or identity replacement resets eligibility. After a controller outage that breaks observation continuity, require a new full empty interval; downtime must not age a lobby into removal.
+
+## Removal and cancellation contract
+
+Automatic removals always use normal safety gates and never set a safety override. Revalidate policy mode and revision, ownership, membership, remaining spare capacity, conflicts, and occupancy immediately before claiming and dispatching removal. Preserve the agent's fresh local occupancy check immediately before mutation.
+
+The existing [cancelRemoval](../../src/lib/server/provisioning/history.ts) transaction competes with the worker while a removal is waiting. It cannot undo removal after the worker has claimed it. General cancellation also cannot withdraw an already delivered host command. The agent can hold an occupied command and retry later, so cancelling only the controller row would leave stale destructive intent.
+
+Before enabling Create and destroy, extend the controller/agent contract for autoscaling removals: an occupied or unknown host-local check must invalidate that autoscaling removal attempt rather than retain it for eventual execution. A later attempt requires a new controller evaluation and the full empty interval. Carry policy revision and a bounded authorization expiry, and require renewed controller authorization before an agent retry after delay or restart. Pausing or superseding a policy must invalidate authorization that has not begun mutation. Test these races with the real agent journal behavior; do not assume a controller cancellation implies host acknowledgement.
+
+If mutation has already begun, show that cancellation was too late and let the existing operation reconcile its result. Do not release its identity, key slot, or port until removal and cleanup are confirmed. A cancelled, failed, or ambiguous job must not cause a replacement command to overtake an unsettled delivered command.
+
+Pause on failed or ambiguous automatic operations and link to Provisioning recovery. Capacity waits are normal waiting states, not failures. After explicit recovery, resume from a new snapshot without replaying the old command. This bounds retries and avoids repeatedly creating failing lobbies with the same key or recipe.
+
+## Storage and integration
+
+Add shared policy and decision types plus a server-side evaluator under `src/lib/server/autoscaling/`. Separate a deterministic planner from database reads and Provisioning submission so dry run and live modes evaluate the same rules.
+
+Persist policies with their revision, mode, settings, and timestamps; adopted instance membership; observation continuity; next-action time; and decisions linked to exact Provisioning operation IDs. Store the relevant evidence source and age, counts, intended action, blocker, and policy revision. Do not include keys or full player rosters in decision history.
+
+Use database transactions and uniqueness constraints to enforce one policy per game/map, one unresolved autoscaling intent per policy, and unique lobby allocation across manual launches, collections, and autoscaling. Persist the decision and its operation link together. Process-local in-flight guards prevent overlapping passes, while durable constraints prevent duplicate work after a crash. Add indexes for policy membership, unresolved decisions, and recent history; prune decision history on the existing retention schedule with a proposed 31-day window.
+
+Extend Provisioning source types, admission, history, and backlinks with an autoscaling source containing policy and decision identity. Keep creation recipe selection, placement, operation execution, and cleanup inside the existing services. Add no separate host command executor or per-map RCON polling loop.
+
+Expose authenticated read and update routes using the existing fleet-management authorization rules. Use a revision check for edits and activation so stale forms cannot overwrite newer policy settings. Off and Dry run submit no host mutations. Moving out of a live mode withdraws cancellable policy work, reports unsettled commands, and releases exclusive ownership only when competing mutations can safely proceed.
+
+## Operator experience and measurements
+
+Add an Autoscaling view linked from Fleet, Collections, and Metrics. Show each map's mode, occupied and empty ready counts, pending work, minimum, maximum, spare target, game key availability, last evaluation, next eligible action, and current blocker. Detail includes the policy editor, affected lobbies, recent decisions, Provisioning links, and pause/resume controls. Dry run actions must be visibly hypothetical.
+
+Activation and edits preview the resulting target and any proposed removals. Pause stops new decisions immediately and shows which queued actions were withdrawn and which delivered operations remain unresolved. Do not claim the fleet was restored when a policy is disabled.
+
+Record empty-spare deficit minutes, unknown-status minutes, empty lobby-hours, action frequency, capacity/key blocks, cancellations, failures, and request-to-ready durations. Use Provisioning milestones for queue time and readiness, but measure browser visibility and actual successful joins separately in the pilot: a healthy container or RCON response alone does not establish player joinability.
+
+Compare equivalent weekly periods and record game/runtime versions, player-hours, host CPU and memory, and provisioning latency. “Time with players” will naturally rise when idle lobbies leave its denominator; it does not by itself prove increased demand or CPU savings. Dry run estimates are conditional on the observed fleet, not a simulation of how players would redistribute after removal. Use measured creation and join latency to tune spare targets before broad rollout.
+
+## Delivery sequence and acceptance
+
+Complete and validate each phase in a separate commit before starting the next.
+
+1. **Policy model and planner.** Add additive schema changes, identity-aware snapshots, policy validation, decision types, and the pure evaluator. Keep all modes non-mutating until admission integration exists. Acceptance: deterministic fixtures cover the target formula, variant separation, incomplete evidence, game pool exhaustion and reuse, bounds, and continuous empty timing.
+2. **Dry run and visibility.** Register the requester, persist bounded decision history, and add the policy UI/API with dry run enabled. Acceptance: repeated ticks and controller restarts produce understandable decisions without creating Provisioning jobs or reserving identities, ports, or hosts. Begin the two-week observation period on representative maps.
+3. **Ownership and live creation.** Add transactional intent, Provisioning source eligibility for Create/Destroy, collection/manual conflict checks, and Create only mode. Acceptance: no duplicate creation under concurrent ticks or submissions; no growth beyond the maximum; no capacity bypass; blocked creates remain single durable jobs. Pilot one or two maps after reviewing dry run evidence.
+4. **Safe removal.** Implement continuous-empty selection, admission rechecks, queued cancellation, and the agent authorization contract above. Acceptance: positive/unknown occupancy blocks destruction; a held command cannot remove a lobby later without renewed eligibility; policy pause, revision changes, and new protected work invalidate pending intent. Enable Create and destroy only after these checks pass.
+5. **Pilot review and rollout.** Compare CPU and empty lobby-hours alongside spare availability, successful joins, failures, and creation latency. Expand one map at a time only if savings are measurable, ordinary gameplay is preserved, and loss of spare availability stays within the pilot's agreed tolerance. Document the tuned settings and operating procedure.
+
+Required behavioral scenarios across these phases:
+
+| Scenario                                                                     | Expected result                                                                                                                                             |
+| ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Last spare fills, including a high-round partially filled session            | One create when eligible; free player slots do not count as spares.                                                                                         |
+| Create is queued, starting, or awaiting verification                         | Reservation counts once; it is not advertised as ready and no duplicate is queued.                                                                          |
+| Every eligible host is full or unavailable                                   | Existing job waits; other policies and worker tasks continue.                                                                                               |
+| Game key pool is empty or exhausted                                          | Hold growth with a clear reason; other games remain independent and eligible surplus removal can proceed.                                                   |
+| A launch is queued, stopped, failed, or uncertain                            | Its key stays reserved until confirmed cleanup; no other launch can use that key concurrently.                                                              |
+| Occupancy evidence conflicts, expires, or arrives from an old lifecycle      | Positive current occupancy wins; invalid evidence cannot establish emptiness.                                                                               |
+| Players join before claim, after dispatch, or while the agent holds removal  | Queued work is withdrawn where possible; agent safety prevents occupied destruction; held autoscaling intent cannot execute later without a fresh decision. |
+| Process restarts before or after intent submission                           | Existing operation is recovered by durable identity without replay or duplicate admission.                                                                  |
+| Collection exact match omits a managed map                                   | Preview and submission report the ownership conflict, including stale-preview races.                                                                        |
+| Beta, campaign, maintenance, host drain, or recovery begins after evaluation | Admission rechecks block scaling and existing work retains its lifecycle ownership.                                                                         |
+| Operator pauses, changes limits, or cancels a policy job                     | Unstarted intent is invalidated; unsettled commands stay visible and keep reservations until reconciled.                                                    |
+| A name is reused after confirmed deletion                                    | Old observations, cooldown decisions, and commands cannot apply to the replacement generation.                                                              |
+
+Run targeted Vitest coverage for planner, collections, provisioning, cancellation, telemetry, and routes, plus Python agent tests for the removal contract. Before each implementation phase is committed, run the relevant checks; before live release, run `npm run check`, `npm test`, `npm run test:agent`, and `npm run build`. Verify the UI with keyboard navigation and narrow layouts, including stale evidence, blocked capacity, and partial cancellation states.
+
+## Rollback
+
+Provide a global pause as well as per-policy pause. It stops new decisions and withdraws unstarted work using the same cancellation contract. Keep the controller and agent running to settle delivered operations and cleanup; do not delete policies or reservations to bypass recovery.
+
+After outstanding operations settle, release map ownership. Operators can restore a fixed lineup through Collections using normal capacity and empty-server gates. Turning autoscaling off does not recreate removed lobbies or remove lobbies it created. Retain decision and Provisioning history for review, and keep the additive schema compatible with all policies Off.
+
+## Implementation and validation record
+
+The implementation adds the pure evaluator and persistent policy model, dry-run requester and UI, exclusive map ownership with capacity-aware creation, and revocable removal authorization. It creates no production policies. The authorization claim is the final cancellation boundary; agents invalidate an occupied or expired attempt and require a new decision rather than retrying it later. The local claim window is five seconds, measured from the start of the authorization request, with another occupancy check before mutation.
+
+Validation includes planner and snapshot tests, duplicate-admission and stale-preview tests, collection exact-match conflicts, operator cancellation, authorization expiry and host identity, controller/agent occupancy races, and agent restart recovery. Agent packaging publishes version 1.7.0 with the new capability.
+
+The full controller suite reported 1,934 passing tests and two existing failures in `server-detail-api.test.ts` (plugin classification) and `watchdog.test.ts` (an occupied-telemetry RCON expectation). Both failures reproduce independently at the pre-implementation commit `a22461a`. They are not autoscaling regressions. Targeted controller tests, all 228 Python agent tests, type checking, and the production build passed.
+
+An isolated Firefox review exercised policy creation, preview, save, keyboard focus, and 1440, 390, and 320 pixel layouts. A narrow-layout heading issue found during review was corrected. The preview used a temporary database and no production host credentials. Production CPU savings, external joinability, spare targets, and acceptable deficit durations still require the two-week observation period and operator-run pilot.
+
+Game-wide policy follow-up: selecting every map applies the targets independently to each catalog map and variant. Game-wide policies serialize actions across maps and cannot overlap live map-specific ownership. Existing map policies remain compatible.
+
+Game key pool follow-up: keys are shared by all launch paths for a game. Lobby names and key labels are independent. Policy maxima no longer require dedicated key coverage on every map, and policies can be saved before adding keys. Snapshots account for reservations on other maps, planner creation waits when the pool is exhausted, and admission accepts its own reserved key. Keys are released only after confirmed cleanup.
+
+Follow-up validation: 40 autoscaling tests and 118 related provisioning, collection, and cancellation tests passed, along with type checking and the production build. The earlier isolated Firefox review covered keyboard navigation and 1440, 390, and 320 pixel layouts. Its per-map key coverage checks described the previous behavior, superseded by game key pools.

docs/plans/player-achievements-and-visual-refresh-plan.md

@@ -0,0 +1,296 @@
+# Player achievements and visual refresh implementation plan
+
+Status: Proposed October 7, 2026. Implementation has not started.
+
+Make Electric Cherry more rewarding to revisit by showing recent player
+achievements, making match results easy to share, and giving maps a recognizable
+visual identity. Deliver player achievements first, then refresh the presentation
+across Fleet, Autoscaling, and operations. Each workstream should be useful on
+its own.
+
+## Intended experience
+
+- A player can compare this week's or this month's results, open the match that
+  earned a record, and see how it improved on their previous best.
+- An operator can export a readable match or player result card without manually
+  assembling a screenshot.
+- Maps and games are recognizable at a glance. Player counts, rounds, health,
+  and required actions have a clear visual order.
+- Routine states look calm. Failures, stale observations, and safety blockers
+  remain visible without opening technical details.
+
+## Scope and defaults
+
+| Area                    | First release                                                                                                                                                                                     |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Leaderboard periods     | All time, This week, and This month, with previous/next period navigation. Keep All time as the default.                                                                                          |
+| Calendar                | Weeks start Monday at 00:00 UTC; months start on the first at 00:00 UTC. Show the boundaries explicitly. Personal timezone preferences format timestamps without changing competition membership. |
+| Personal bests          | Highest round, best game kills, and highest reported score, compared within the same game and map variant.                                                                                        |
+| Sharing                 | A preview and PNG download for a match summary or one player's result, plus an authenticated match link. Native device sharing is an optional enhancement with download/copy fallbacks.           |
+| Visual direction        | Existing dark surfaces, electric-blue accent, typography, and brand mark, extended with original SVG map emblems and consistent game/status treatments.                                           |
+| Initial visual coverage | Fleet, server detail, Autoscaling, provisioning and host operation history, player highlights, and match/share cards.                                                                             |
+
+Season rewards, player-account linking, public profile/match pages, notification
+subscriptions, rank-change tracking, custom themes, and photographic map artwork
+are outside this release. These features need separate product decisions.
+
+## Existing foundations
+
+The [leaderboard guide](../guides/leaderboard.md) defines the current qualification
+rules, player identity, record evidence, and access model. Completed matches and
+player participation persist independently of live servers and operational
+history retention. Reuse these records rather than requiring reporter changes.
+
+The leaderboard already has game/map/metric filters, shareable filter URLs,
+player profiles, source-match links, and refresh behavior that protects someone
+inspecting a row. Fleet already has search, pins, grouping, cards/table views,
+and a live match spotlight. Preserve those behaviors.
+
+Extend the [shared UI patterns](../development/ui-patterns.md), semantic tokens,
+`ServerIdentity`, `StatusBadge`, and existing disclosure controls. The visual
+work should refine their use across pages.
+
+## Player achievements
+
+### Calendar rankings
+
+Add validated `period=all|week|month` and optional `start=YYYY-MM-DD` query
+parameters. An omitted start selects the current period; an explicit start must
+be a Monday for a week or the first day for a month. Reject malformed dates,
+misaligned boundaries, and a start supplied with All time. Disable navigation
+into future periods. Existing URLs without these parameters retain their behavior.
+
+Return resolved period metadata, including inclusive start, exclusive end, and
+the fixed UTC timezone. Use the same resolved boundaries for rankings, summary
+totals, eligibility thresholds, exact-player rank, selected profile history,
+and record evidence. Apply a period to a match's completion time, so a match
+crossing midnight belongs entirely to the period in which it completed.
+
+Retain the existing rules: only completed games reaching round 2 with qualifying
+player participation contribute; ties share rank; search preserves actual rank;
+GUIDs combine across T5/T6; map variants remain distinct. Efficiency thresholds
+apply to matches inside the selected period. Keep known map options available
+when a selected period has no results.
+
+On the profile, label lifetime totals separately from the selected competition
+and filtered match history. Carry period/start through profile links, record
+links, pagination, and return navigation. Reset pagination when the period
+changes. A copy-link action should resolve the current period to an explicit
+start so the shared link still identifies the same competition next week.
+
+Add an additive nullable `first_completed_at` field to historical matches. Set it
+once when a match first becomes completed. The current ingestion path can update
+`ended_at` when a changed final report arrives; a correction must not move a
+match into another competition. Preserve the existing `ended_at` behavior.
+Backfill completed rows from valid `ended_at` values as the best available
+historical estimate. Unknown completion times remain eligible for All time but
+are excluded from dated competitions, with that limitation explained in report
+help.
+
+Create any index on the new field after the additive column migration. Startup
+currently executes the base schema before adding columns to existing databases.
+Use a dedicated, idempotent migration that normalizes valid values to ISO UTC;
+existing timestamp-normalization markers must not cause upgrades to skip it.
+Evaluate a partial completion-time index for completed matches using query plans
+and the existing synthetic benchmark before selecting the final index. Period
+membership stays stable after correction, while corrected counters can update
+the rankings for a past period.
+
+Resolve the period before report submission, and include its bounds in all
+in-flight, worker-response, and ranked-scope cache keys. A week or month must
+roll over without requiring a database write. Extend evidence requests with the
+same bounds so a period's record never links to an equal result from another
+period. Preserve the current all-time evidence tie-breaking behavior.
+
+Primary changes:
+
+- [Leaderboard types](../../src/lib/leaderboard.ts), API validation, page and
+  profile loaders, and [navigation context](../../src/lib/leaderboard-navigation.ts).
+- [Ranking queries](../../src/lib/server/fleet/leaderboard.ts),
+  [record evidence](../../src/lib/server/fleet/leaderboard-records.ts), player
+  profile queries, and report worker/cache inputs.
+- Database schema/migration and
+  [completion ingestion](../../src/lib/server/fleet/leaderboard-ingest.ts).
+
+### Personal best highlights
+
+Show a compact Recent personal bests section on player profiles and annotate
+the relevant results on completed match pages. Use copy such as “Highest round
+on Town Survival: 31, up from 27.” Identify whose record it is; dashboard accounts
+are not currently linked to player GUIDs.
+
+A milestone is a strict improvement over all earlier qualifying results for the
+same GUID, game, map key, and metric. Order by `first_completed_at`, then match ID
+for deterministic simultaneous completions. An equal result is not a new record.
+A first qualifying result is labeled First recorded result rather than an
+improvement. A later record does not remove the earlier match's “Personal best
+at the time” annotation. Current record evidence remains a separate concept.
+
+Derive milestones in the report worker from retained history for the requested
+player or bounded match roster. Compare against the complete earlier history
+before applying the displayed date range or result limit. Do not make the first
+match of each week a false personal best by limiting the comparison to that week.
+Batch the requested player/map scopes to avoid one query per metric or row.
+
+Use undated historical values as a conservative comparison floor and do not
+emit dated milestones for them. Explain in methodology that incomplete older
+timestamps can suppress a highlight. Recompute derived milestones after counter
+corrections using the existing report revision invalidation. Duplicate reports
+must not create duplicate highlights. No new event journal, notification history,
+or full-history startup backfill is needed.
+
+Initially show at most five grouped match highlights on a profile, with a link
+to its existing match history. Follow the selected game/map and competition
+period for displayed highlights while comparing against all earlier history.
+Group multiple improved metrics from one match.
+Keep the existing qualification caveats near the evidence: highest round can
+include late joins, score is the highest observed balance, and first observation
+does not prove continuous participation. Add no confetti or repeating animation.
+
+Add a dedicated milestone query/presentation module alongside the existing
+leaderboard records and pass only the needed results to profile and match pages.
+Use explicit report-worker job handling for these reads rather than expanding
+the worker's current player-report fallback implicitly.
+
+### Shareable match results
+
+Add Share result to completed match pages, and a player-specific action to each
+eligible result. Open an accessible dialog with two layouts: Match summary and
+Player result. The preview and exported image use the same bounded presentation
+model.
+
+The summary includes map, game, final round, completion date, and known duration.
+The player layout adds display name, recorded player statistics, and applicable
+personal-best improvements. Use the player's observed highest round for their
+result, which may differ from the match's final round. Include relevant late-join
+or missing-start context. Omit unknown values rather than rendering them as zero.
+
+Render a 1200 × 630 PNG locally using bundled fonts and assets. Share-card layout
+code must wrap or deliberately truncate long names without dropping essential
+qualifications. Provide semantic HTML for the preview; the canvas is an export
+implementation detail. Wait for fonts/assets before export and show a retryable
+error if generation fails. Native sharing, when supported, must run only from an
+explicit user action.
+
+Construct an explicit export model. Exclude chat, GUIDs, internal instance IDs,
+addresses, keys, and raw diagnostics; the current match detail object includes
+chat and must not be exported wholesale. Offer a plain match link labeled as
+requiring Electric Cherry access. Do not change route permissions or introduce
+an unauthenticated image endpoint. Downloads remain useful to recipients without
+a dashboard account.
+
+Implement the first version with the current brand styles. Apply the new map
+emblems during the visual workstream without changing result semantics.
+
+## Visual refresh
+
+### Map and game identity
+
+Add a presentation resolver keyed by game and selected map slug, backed by the
+existing [map catalog](../../src/lib/map-aliases.ts). Use original small SVG
+emblems: a theater marquee for Kino, a rocket for Ascension, a moon for Moon,
+prison bars for Mob, and excavation/gears for Origins. Complete the catalog with
+the same simple line or silhouette style and a neutral custom-map fallback.
+
+Maps sharing an environment may share artwork, while supported Survival
+variants retain explicit names. Historical views resolve identity from
+their stored game/map key and variant, never from a replacement live server.
+An ambiguous engine map must use a fallback rather than inventing a variant.
+Extend `MatchDetail`, `PlayerMatch`, and their queries with the stored map key;
+they currently expose friendly map names without that identity. Do not infer
+historical variants from display names.
+
+Introduce `MapEmblem` and a shared game label/badge. Keep Black Ops I/II readable
+without color, using distinct I/II marks where space is limited. Put larger
+emblems on map headings and result cards, smaller ones on lobby cards, and keep
+table mode compact. Artwork is decorative when adjacent text names the map.
+
+Use locally shipped, dimensioned SVG assets or components. Keep the current
+fonts and dark/electric-blue palette. Game decoration must not reuse danger or
+warning tones. Add only the semantic tokens needed for identity and achievement
+styling, and document their use.
+
+### Information order and status consistency
+
+| Surface                          | Presentation changes                                                                                                                                                                                                                                |
+| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Fleet cards                      | Map/lobby/game first; health, player count, and round next; roster and match duration below; lower-priority timing and technical identity in supporting details. Preserve compact empty cards, search, pins, table mode, and the match spotlight.   |
+| Server detail                    | Add map/game identity to the shared header with health. Keep player counts, round, and timing in the Live content so Configuration and Diagnostics remain focused on their task.                                                                    |
+| Autoscaling                      | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Off/On setting, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. |
+| Provisioning and host operations | Use consistent labels and status components while preserving action, target, result, elapsed time, and specific wait reasons. Put long IDs, source paths, and low-level metadata behind explicit details/copy controls.                             |
+| Player and match results         | Reuse map identity and achievement treatment across profile highlights, match results, and downloaded cards. Distinguish a current record from a record earned at that time.                                                                        |
+
+Align status presentation by meaning, while preserving domain-specific labels.
+For example, `StatusPip` currently colors starting/provisioning amber while shared
+badges use an informational treatment. Normal queued work and waiting for players
+should not look like failures. Keep the specialized provisioning status helpers
+that explain why an operation is waiting.
+
+Do not collapse freshness warnings, unknown player counts, failed operations,
+safety blockers, pending destructive work, or unsettled commands. Raw identities
+must remain reachable through keyboard-accessible details and copy controls.
+Preserve navigation context, permission checks, confirmations, and existing
+polling/store behavior.
+
+## Delivery sequence
+
+Complete, validate, and commit each phase before starting the next. Finish the
+player-achievement workstream before starting the visual refresh. Commit any
+unrelated fix separately.
+
+| Phase                                 | Reviewable result                                                                                                     | Acceptance gate                                                                                                                                                                                           |
+| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| A1 Completion time and period queries | Stable completion field/migration, period parsing, scoped ranking/evidence/profile queries, and cache keys.           | Boundary, correction, qualification, and all-time compatibility tests pass; query plans and representative timing are recorded.                                                                           |
+| A2 Period navigation                  | Week/month controls, history navigation, period metadata, profile context, and stable share URLs.                     | Direct links, back/forward, rollover, empty periods, and inspection-preserving refresh work on desktop and mobile.                                                                                        |
+| A3 Personal best highlights           | Derived milestones, profile section, and match annotations.                                                           | Strict improvements, ties, first results, variants, corrections, and undated history behave as specified; long player histories remain practical.                                                         |
+| A4 Result sharing                     | Accessible preview, PNG download, authenticated copy-link action, and supported native sharing.                       | Export matches the preview, contains only selected result data, handles long/incomplete data, and works after live-server deletion. Update leaderboard documentation and commit the completed workstream. |
+| B1 Shared visual identity             | Original emblems, resolver, game labels, achievement tokens, and usage guidance.                                      | Every map selection resolves correctly or falls back; no misleading variant, color-only meaning, or missing-asset layout shift.                                                                           |
+| B2 Fleet and operational presentation | Apply identity/hierarchy/status rules to Fleet, server detail, Autoscaling, provisioning, and host operation history. | Existing actions and critical states remain visible and usable; no new polling or behavior changes.                                                                                                       |
+| B3 Player presentation and review     | Apply the shared visual identity to player highlights, match pages, and export layouts; refresh documentation images. | Complete the visual/accessibility review and update UI guidance, then commit the completed workstream.                                                                                                    |
+
+## Validation
+
+Use an isolated synthetic database. Extend existing fixture helpers; the current
+`seed:fixture` package script points to a missing script and must not be assumed
+to provide a working preview dataset.
+
+For achievements, cover Monday/month/year boundaries, leap years, viewer timezone
+differences, cross-boundary matches, unknown completion timestamps, and corrections
+after completion. Test upgrade from an existing database, timestamp normalization,
+and restart after the completion migration. Verify that an idle database still
+changes the current period at rollover. Test period-scoped record sources, ties, efficiency eligibility,
+GUID identity across games, T6 variants, deleted servers, profile pagination,
+and all-time response compatibility. Match/player qualification must be shared
+with existing leaderboard logic.
+
+Test personal-best comparison against full prior history, simultaneous completion
+ordering, duplicate final reports, first results, later-surpassed records, and
+grouping several improvements from one match. Benchmark a player with a long
+history and a full roster. Bound request scopes and response sizes; do not hide
+query costs by silently ignoring older matches. Keep report work off the main
+request thread. Extend the existing leaderboard benchmark for period queries
+and record before/after cold and warm timings under the same fixture.
+
+For sharing, verify the explicit export model, missing values, long/Unicode
+names, font/asset readiness, downloaded-image readability, and browser fallbacks.
+Confirm that anonymous access to match/profile pages remains unchanged and that
+sharing requires an explicit action.
+
+Review representative screens at 1440, 390, and 320 pixels, with 200% zoom,
+keyboard navigation, reduced motion, and long custom names. Include empty,
+occupied, stale, offline, starting, failed, waiting, and paused states. Check
+contrast, focus order, non-color status labels, readable export text, and absence
+of nested interactive targets. Capture before/after screenshots using identical
+fixtures; verify that map, players/round, health, and required action are easy to
+identify without opening details.
+
+During implementation, run `npm run check`, focused Vitest tests for changed
+behavior, and `npm run build` for each relevant phase. Before completing each
+workstream, run the full controller test suite and browser review. Agent tests
+are needed only if agent code or its protocol changes; neither is planned here.
+Use meaningful behavior/query tests for new rules and manual visual checks for
+decorative changes.
+
+After delivery, fold lasting behavior into the leaderboard guide and UI patterns,
+refresh the Fleet documentation image, and remove this completed plan according
+to the repository documentation convention.

docs/provisioning.md

@@ -1,89 +0,0 @@
-# Provisioning
-
-Open **Operations → Provisioning** for instance lifecycle work, blueprints, and
-persistent activity. Provisioning owns the queues, safety checks, execution, and
-recovery for all six actions. Create a server using **Create server**. For an existing
-server, choose **Manage server** or open its **Server actions** menu.
-
-| Action | Behavior |
-| --- | --- |
-| Create | Create a new instance using the latest promoted files and verify container health, RCON, and the selected map. |
-| Start | Start the existing container and verify container health and RCON. |
-| Restart | Wait until empty, restart the existing container, and verify container health and RCON. |
-| Rebuild | Wait until empty, destroy the instance, delete all server files and its registration, then create a fresh instance in the same server slot and port. Verify container health, RCON, and map. |
-| Stop | Wait until empty, stop the container, and verify that Komodo reports it stopped. Keep the instance for a later Start. |
-| Destroy | Wait until empty, remove the container and Komodo deployment, then delete server files and registration. |
-
-Rebuild requires confirmation because **no instance data is preserved**. Its fresh
-files come from the latest promoted setup when creation begins, including changes
-promoted while waiting. The independently saved key in **Keys** must exist before
-teardown. Blueprint definitions, promoted artifacts, saved keys, and historical
-activity belong to Electric Cherry, and are not instance files.
-
-Rebuild reserves the original ID and port throughout teardown and creation, even
-when a lower lobby is available. It shares the existing creation queue and startup
-limit. A failed removal never proceeds to creation. Once Komodo deletion is
-confirmed, failed local cleanup offers **Resume rebuild cleanup**; creation cannot
-proceed until the old files are fully removed. An interrupted fresh startup can
-resume health verification, without repeating teardown or creation. Inspect other
-failures before making another request.
-
-Start and Restart share the rolling restart limit with campaigns, betas, and
-maintenance. A successful API reply is followed by a recovery delay and actual
-container and game-status checks. Failed or ambiguous restart requests retain their recovery
-slot and are not automatically replayed. Stop requires an explicit stopped container state; an
-unreachable Komodo API is not evidence that a stop succeeded.
-
-Disruptive operations require a fresh zero-player result from RCON or accepted
-telemetry, unless Komodo positively reports a stopped container. Fresh occupied
-telemetry vetoes a conflicting empty RCON reading. Reports must be received within
-45 seconds; recovery verification also excludes reports from before the submitted
-operation. Telemetry can verify the game's activity and selected map after Komodo
-reports healthy. Runtime actions wait for conflicting
-campaign, beta, maintenance, or provisioning work. Duplicate pending requests for
-the same action return the existing operation; a different action returns a
-conflict. Requests and events survive application restarts.
-
-Campaigns, beta runs, and maintenance submit durable restart requests to
-Provisioning and observe the resulting operation by ID. Their pages link to that
-operation, and its history links back to the requesting work. Provisioning owns
-the empty-server check, conflict handling, restart capacity, remote command, and
-health verification. The callers retain file deployment, beta evidence, and
-Plutonium revision verification. Jobs remain visible while waiting for players,
-other work, or integration configuration.
-
-[Watchdog](watchdog.md) also submits recovery operations and links each attempt
-to its incident. A failed Watchdog restart can hand its reserved slot to a rebuild
-after five minutes; unrelated recovery slots remain held. Drift campaigns follow
-their linked rebuild without requesting another restart. Their file evidence and
-history remain available after the instance is removed.
-
-Campaign changes queued before submission can share one restart. Changes arriving
-after submission require a new operation. A failed or interrupted request keeps
-its original ID and never automatically replays. An explicit beta retry starts a
-new attempt, preserving prior history; paused or closed beta work cannot start a
-queued restart. Recovery of an already submitted restart continues even if its
-requester closes. Upgrades adopt existing restart intent for observation only.
-
-## Routes and compatibility
-
-- `/provisioning`: active operations, blueprints, and history filtered by action.
-- `/provisioning/:id`: operation stages, errors, timestamps, and activity.
-- `/provisioning/collections/:id`: collection progress.
-- `GET /api/provisioning`: lifecycle history (`deployments` response property).
-- `POST /api/provisioning`: queue `{ "server": "t5_zombie_theater_1", "action": "start" }`.
-  Supported actions are `start`, `restart`, `rebuild`, and `stop`. Returns HTTP 202
-  with `deploymentId`. Create continues through `POST /api/servers`; Destroy uses
-  `DELETE /api/servers/:name`.
-- `/api/provisioning/:id`, `/:id/resume`, `/:id/cancel`, `/preview`, and
-  `/house-party` preserve the existing operation, recovery, removal cancellation,
-  launch preview, and fleet destruction APIs.
-
-Old `/deployments` pages redirect to their `/provisioning` equivalents, preserving
-query parameters. `/api/deployments` remains compatible, including old campaign
-detail endpoints. No history or IDs are renamed or discarded.
-
-Start and Stop use Komodo's corresponding runtime commands. Rebuild uses full
-destruction and creation, not Komodo's redeploy command. See the
-[Komodo execute API](https://docs.rs/komodo_client/latest/komodo_client/api/execute/index.html)
-for the underlying container operations.

docs/research/leaderboard-query-benchmark.md

@@ -0,0 +1,46 @@
+# Leaderboard query benchmark
+
+The benchmark and measurements below describe the earlier SQL optimization,
+before the worker and revision caches were added. They are historical results,
+not measurements of the current HTTP implementation.
+
+Run the deterministic, isolated benchmark with:
+
+```sh
+node scripts/benchmark-leaderboard.mjs --scenario all --baseline-ref eac9ad4 --runs 3 --json /tmp/leaderboard-benchmark.json
+```
+
+The script creates temporary SQLite databases, loads the actual query modules,
+checks complete-response parity against the selected Git revision, and reports
+timings, source hashes and query plans. It never opens the application database.
+The small fixture has 2,000 matches and 8,000 participation rows; the large fixture
+has 50,000 matches and 200,000 participation rows. Both include completed,
+active, interrupted and nonqualifying records.
+
+On the local Intel i5-8279U / Node 25.9.0 / SQLite 3.49.2 test environment, three
+measured runs after warmup gave these median query times on the large fixture:
+
+| Request                  |   Before |  After |
+| ------------------------ | -------: | -----: |
+| Highest round, all games | 1,717 ms | 829 ms |
+| Total kills, all games   | 1,555 ms | 858 ms |
+| One game                 |   899 ms | 491 ms |
+| One map                  |   345 ms | 224 ms |
+| Player search            | 1,515 ms | 757 ms |
+| Deep page                | 1,421 ms | 752 ms |
+| Headshot percentage      | 1,444 ms | 760 ms |
+| Exact player rank        | 1,524 ms | 554 ms |
+
+The small fixture's unfiltered request fell from 41 to 30 ms. All 150 additional
+small-fixture parity cases and 22 timed small/large request cases matched the
+baseline's complete responses, including ratios and record evidence. Timings
+include SQL preparation and evidence lookup, exclude HTTP/browser rendering, and
+use a warm filesystem cache with alternating implementation order. These are
+local synthetic measurements, not production latency guarantees. SQL queries
+run synchronously within the report worker; larger histories still affect
+report latency and worker capacity.
+
+Validation for that optimization passed 1,019 unit tests, Svelte checking with no errors
+or warnings, and a production build. Isolated Firefox walkthroughs at 1440px and
+390px checked completed-only totals, record links, efficiency qualification,
+scoped ranks/history, preserved navigation, empty scopes and mobile expansion.

docs/research/t5-t6-idle-cpu-research.md

@@ -1,6 +1,14 @@
 # Reducing idle CPU use in T5 and T6 Zombies servers
 
-Research date: September 19, 2026. Scope: Plutonium T5 Zombies (`t5sp`) and T6 Zombies (`t6zm`), with particular attention to Electric Cherry's Plutainer/Docker/Komodo deployment.
+Research date: September 19, 2026. Scope: Plutonium T5 Zombies (`t5sp`) and T6 Zombies (`t6zm`), with particular attention to Electric Cherry's Plutainer/Docker host deployment.
+
+This investigation predates the implemented lobby autoscaling feature. Repository
+observations and proposed changes below describe that research baseline. For
+current behavior, see [Lobby autoscaling](../guides/autoscaling.md) and
+[Native hosts](../guides/native-hosts.md); the
+[autoscaling plan](../plans/lobby-autoscaling-plan.md) tracks the remaining pilot.
+
+## Recommendation and scope
 
 **Recommendation:** first benchmark Wine's NTSYNC support and audit the effective runtime configuration. If every lobby must remain immediately available, trial CPU limits that apply only while a server is confirmed empty. If spare lobbies can be unavailable temporarily, keep a smaller pool running and start additional servers when demand rises. Stopping unused instances offers the most predictable reduction in their CPU consumption.
 
@@ -8,14 +16,18 @@ I found no documented, supported switch that makes both T5 and T6 dedicated serv
 
 This is research and a proposed evaluation plan. No production measurements were available, and no application code, server configuration, or running servers were changed. Reported savings below are either explicitly identified operator anecdotes or illustrative calculations.
 
-**What applies to this project.** The repository launches one Plutainer container per lobby, selects `t5sp` or `t6zm`, and defaults new deployments to `ghcr.io/ayymoss/plutainer:latest`. Its creation specification does not explicitly configure CPU quotas or device passthrough. Electric Cherry refreshes server status every 30 seconds, tracks occupancy transitions, and checks container readiness before RCON. These are useful foundations for an idle policy, but they are not an implemented CPU governor. See [deployment creation](../src/lib/server/fleet/create.ts), [configuration](../src/lib/server/config.ts), [status worker](../src/lib/server/pipeline/worker.ts), and [status timing](../src/lib/server-freshness.ts).
+## Repository baseline
+
+The repository launches one Plutainer container per lobby, selects `t5sp` or `t6zm`, and defaults new deployments to `ghcr.io/ayymoss/plutainer:latest`. Its creation specification does not explicitly configure CPU quotas or device passthrough. Electric Cherry refreshes server status every 30 seconds, tracks occupancy transitions, and checks container readiness before RCON. These are useful foundations for an idle policy, but they are not an implemented CPU governor. See [deployment creation](../../src/lib/server/fleet/create.ts), [configuration](../../src/lib/server/config.ts), [status worker](../../src/lib/server/pipeline/worker.ts), and [status timing](../../src/lib/server-freshness.ts).
 
 The live host kernel, Wine version, image digest, installed scripts/plugins, CPU model, and actual idle utilization remain unknown. A repository default of `latest` does not establish which image a running server uses.
 
 **Why an empty server can still be expensive.** Zero players does not establish that the engine, map scripts, plugin timers, Wine synchronization, or container helpers have stopped working. Those are competing explanations to measure. A map can also behave differently after its final player disconnects than it does immediately after startup. The research supports investigating these layers; it does not establish which one dominates this fleet.
 
+## Options
+
 | Option                                              | Expected effect on idle CPU                               | Availability and gameplay tradeoff                                           | Evidence and priority                                                  |
-| --- | --- | --- | --- |
+| --------------------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
 | 1. Enable and verify NTSYNC                         | Potentially substantial if Wine synchronization dominates | Intended to retain normal joins and gameplay; compatibility testing required | Official Wine/kernel support; Plutonium operator anecdote. First trial |
 | 2. Audit busy-wait settings, scripts, and helpers   | Small to substantial, depending on the actual cause       | Usually compatible with keeping lobbies online                               | Documented settings plus diagnostic experiments. First trial           |
 | 3. Apply CPU quotas only when empty                 | Enforceable ceiling on each empty container's CPU time    | Excessive throttling can delay joins, RCON, and heartbeats                   | Docker-supported mechanism; automatic policy needs development         |
@@ -25,7 +37,9 @@ The live host kernel, Wine version, image digest, installed scripts/plugins, CPU
 | 7. Wake through a UDP gateway                       | Could combine stopped backends with automated wakeup      | Protocol, listing, session, and timeout problems                             | Experimental; substantially harder for T5                              |
 | 8. Change the host/runtime or develop engine idling | Potentially substantial, cause dependent                  | Migration cost or ongoing engine compatibility work                          | Conditional alternatives after profiling                               |
 
-**1. Test NTSYNC before treating Wine overhead as unavoidable.** NTSYNC moves Windows synchronization primitives into a Linux kernel driver. Wine 11.0 documents using it when available and identifies Linux 6.14 as the kernel release providing the required module. The kernel documentation explains that the driver exists because equivalent userspace synchronization has performance limitations and exposes `/dev/ntsync`. These are established mechanisms, not Plutonium benchmark results. [Wine 11.0 release notes](https://github.com/wine-mirror/wine/blob/wine-11.0/ANNOUNCE.md), [Linux NTSYNC documentation](https://docs.kernel.org/userspace-api/ntsync.html).
+### 1. Test NTSYNC before treating Wine overhead as unavoidable
+
+NTSYNC moves Windows synchronization primitives into a Linux kernel driver. Wine 11.0 documents using it when available and identifies Linux 6.14 as the kernel release providing the required module. The kernel documentation explains that the driver exists because equivalent userspace synchronization has performance limitations and exposes `/dev/ntsync`. These are established mechanisms, not Plutonium benchmark results. [Wine 11.0 release notes](https://github.com/wine-mirror/wine/blob/wine-11.0/ANNOUNCE.md), [Linux NTSYNC documentation](https://docs.kernel.org/userspace-api/ntsync.html).
 
 A particularly relevant firsthand report, posted September 9, 2026, describes running Plutonium servers on Linux with NTSYNC and seeing idle usage around 25% of one CPU core. The same operator reports limiting empty servers to 15% of a core and removing that limit when players arrive. The post does not provide a controlled before/after benchmark or establish separate T5 and T6 results, so neither figure should become our target without measurement. The forum was available through search-indexed content; direct retrieval was unreliable during this research. [Operator report on NFOservers](https://forums.nfoservers.com/viewtopic.php?p=85675).
 
@@ -38,7 +52,9 @@ For this fleet, the experiment requires all of the following:
 
 Start with one isolated T5 instance and one T6 instance, retaining the existing image and Wine prefix for rollback. A Wine upgrade may have prefix compatibility consequences; test on copies. Adding an environment variable alone does not establish working NTSYNC. If it is already active, enabling it again will not explain or fix remaining CPU use.
 
-**2. Audit the work being done while empty.** This is the fastest way to distinguish an inherent runtime cost from an avoidable loop or configuration override.
+### 2. Audit the work being done while empty
+
+This is the fastest way to distinguish an inherent runtime cost from an avoidable loop or configuration override.
 
 First inspect `com_busyWait` in the actual dedicated process. Plutonium's r5162 changelog introduced it in connection with high client frame rates; r5304 changed its default to false for T4/T5/T6. Checking for an old override that enables it is worthwhile. Its effect on dedicated-server CPU is unconfirmed: verify that the setting exists, takes effect, and changes measured behavior before calling it a fix. [Plutonium changelog](https://plutonium.pw/docs/changelog/#r5304).
 
@@ -50,7 +66,9 @@ Also identify which processes consume CPU: the game executable, `wineserver`, lo
 
 Electric Cherry's 30-second status polling is a separate source of work. Measure monitoring overhead before changing it: a long polling interval can make idle-to-active transitions harder to detect promptly. Do not infer that a high game-process CPU reading is caused by the management UI.
 
-**3. Limit CPU while empty, and restore the normal budget as players arrive.** Docker can change a running Linux container's CPU limits without restarting it. A small controller could apply an idle budget only after repeated fresh zero-player readings and restore the original active budget when activity begins. [Docker container update](https://docs.docker.com/reference/cli/docker/container/update/).
+### 3. Limit CPU while empty, and restore the normal budget as players arrive
+
+Docker can change a running Linux container's CPU limits without restarting it. A small controller could apply an idle budget only after repeated fresh zero-player readings and restore the original active budget when activity begins. [Docker container update](https://docs.docker.com/reference/cli/docker/container/update/).
 
 For example, `--cpus=0.25` means an aggregate allowance equivalent to one-quarter of one logical CPU, not 25% of the entire host. This is an illustration, not a recommended production value. A quota bounds execution time; it does not remove the work the engine wants to perform. A cap below the current demand may therefore reduce the utilization graph while worsening responsiveness. [Docker CPU constraints](https://docs.docker.com/engine/containers/resource_constraints/#cpu).
 
@@ -60,17 +78,19 @@ The policy should handle several cases explicitly:
 - Give startup, map changes, and recovery the normal CPU budget.
 - Restore that budget on a connection attempt if a reliable signal exists. Otherwise, use a conservative idle limit that still supports joining and a detection interval demonstrated to be fast enough.
 - Restore the active budget when occupancy becomes unknown. A separate watchdog or expiring idle-policy lease should restore it if the controller stops running.
-- Coordinate quota ownership with Komodo so container recreation does not silently lose the policy or reinstate an obsolete limit.
+- Coordinate quota ownership with the host agent so container recreation does not silently lose the policy or reinstate an obsolete limit.
 
 Electric Cherry's current 30-second status cycle can leave a newly joined player under the idle cap until the next successful poll. An aggressive cap should not depend on that cycle alone. Quotas can also allow brief bursts followed by throttling within their scheduling period, so evaluate join time and responsiveness, not just average CPU. The kernel exposes `nr_throttled` and `throttled_usec` in cgroup v2 `cpu.stat` to help identify this behavior. [Linux CPU controller](https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu).
 
 This is the strongest practical candidate when retaining every online lobby is a firm requirement. The ceiling is dependable; acceptable Plutonium behavior at a particular ceiling must be demonstrated.
 
-**4. Keep a smaller pool of spare lobbies running.** If many maps have several empty copies, running all copies continuously multiplies the baseline cost. Keep occupied lobbies plus a chosen number of ready empty lobbies per exact map/ruleset, and stop surplus instances after an idle grace period. Start another spare when demand consumes the existing one; scheduled prewarming can cover known busy periods.
+### 4. Keep a smaller pool of spare lobbies running
+
+If many maps have several empty copies, running all copies continuously multiplies the baseline cost. Keep occupied lobbies plus a chosen number of ready empty lobbies per exact map/ruleset, and stop surplus instances after an idle grace period. Start another spare when demand consumes the existing one; scheduled prewarming can cover known busy periods.
 
 This is a proposed capacity policy. Docker stop ends container processes; the resulting elimination of their execution is predictable. Its shutdown behavior depends on the configured stop signal. The upstream Plutainer Dockerfile retrieved during this research specifies `STOPSIGNAL SIGKILL`, so do not assume a graceful shutdown: verify the deployed image and check that logs, stats, and other persistent state finish writing before stopping. [Docker stop behavior](https://docs.docker.com/reference/cli/docker/container/stop/), [Plutainer Dockerfile](https://github.com/Ayymoss/Plutainer/blob/main/Dockerfile).
 
-Use stop/start semantics that retain the deployment, files, port reservation, and server key. Electric Cherry's destruction workflow removes infrastructure and files and is not a standby mechanism. Collection blueprints currently create shortfalls without removing excess servers, so they do not implement this policy either. [Deployment blueprints](deployment-blueprints.md).
+Use stop/start semantics that retain the deployment, files, port reservation, and server key. Electric Cherry's destruction workflow removes infrastructure and files and is not a standby mechanism. Collection blueprints currently create shortfalls without removing excess servers, so they do not implement this policy either. [Collections](../guides/collections.md).
 
 Stopping and pausing need a final occupancy check, protection against a concurrent join, and coordination with deployments, recovery, and monitoring. A zero-player RCON reply alone cannot make a later stop atomic with joining; use a supported admission/drain mechanism if available, or explicitly accept and measure that race in the design. Unknown occupancy must prevent automatic stop or pause.
 
@@ -78,19 +98,25 @@ Preserve exact T6 selections: Town, Farm, and other variants cannot be pooled me
 
 An illustrative calculation: 20 empty instances averaging 0.40 CPU each consume eight logical-CPU equivalents. Keeping five running at the same rate uses two, a 75% reduction in that empty-instance component. These are invented inputs to demonstrate scaling; occupied games, startup bursts, and management services remain additional costs.
 
-**5. Start infrequent maps on request, or prewarm them on a schedule.** A map reservation page or operator action can start a stopped server, wait for readiness, and then tell players it is available. A schedule can start extra lobbies before a recurring play session and stop them later only if they are confirmed empty. This removes the need to infer intent from game packets.
+### 5. Start infrequent maps on request, or prewarm them on a schedule
+
+A map reservation page or operator action can start a stopped server, wait for readiness, and then tell players it is available. A schedule can start extra lobbies before a recurring play session and stop them later only if they are confirmed empty. This removes the need to infer intent from game packets.
 
 For public discovery, keep popular maps online and make rarely used maps available through this explicit flow. The tradeoff is waiting for startup and reduced browser presence while stopped. Pre-pull images and stage updates outside the wake path; measure startup with realistic caches and map assets.
 
 T5 needs special treatment. Its official setup documentation says joins use a backend-issued session ID rather than direct IP connections. T6's setup guide documents IP/port connections. Therefore, a T5 wake flow should verify renewed backend registration and browser joinability before telling users it is ready; an RCON response alone does not prove that. The T5 page also contains generic IP-connect advice elsewhere, so use its explicit T5 session caveat and validate actual behavior. [T5 connection FAQ](https://plutonium.pw/docs/server/t5/setting-up-a-server/#why-cant-i-connect-to-my-server-via-the-console), [T6 setup guide](https://plutonium.pw/docs/server/t6/setting-up-a-server/).
 
-**6. Pause containers as an experiment for faster wakeup.** Docker pause freezes container processes on Linux. This preserves process memory while suspending execution, making it a candidate when cold startup is expensive and RAM is plentiful. [Docker pause](https://docs.docker.com/reference/cli/docker/container/pause/).
+### 6. Pause containers as an experiment for faster wakeup
+
+Docker pause freezes container processes on Linux. This preserves process memory while suspending execution, making it a candidate when cold startup is expensive and RAM is plentiful. [Docker pause](https://docs.docker.com/reference/cli/docker/container/pause/).
 
 The consequence is equally important: the frozen game cannot execute RCON replies, join handling, or backend heartbeats. An external controller must resume it. Treat possible listing expiry, stale sessions, timer jumps, and failed first joins as acceptance tests; successful unpause alone is insufficient. Coordinate monitoring so deliberate suspension is not treated as a fault, and test after both short and overnight pauses.
 
 I would test stop/start first because it gives a clean startup path. Pause becomes attractive only if measured resume reliability and latency justify it. It retains memory, and it does not make a server remain genuinely responsive while asleep.
 
-**7. Build a UDP wake gateway only if seamless demand detection is valuable enough.** The concept is a lightweight listener that owns a stable public endpoint, distinguishes a join attempt from a status query, wakes the backend, and forwards game traffic when ready. This is a design proposal, not a verified T5/T6 product or a configuration toggle.
+### 7. Build a UDP wake gateway only if seamless demand detection is valuable enough
+
+The concept is a lightweight listener that owns a stable public endpoint, distinguishes a join attempt from a status query, wakes the backend, and forwards game traffic when ready. This is a design proposal, not a verified T5/T6 product or a configuration toggle.
 
 There is an architectural precedent: Game Server Snooze implements packet-triggered startup, bounded buffering, and idle shutdown for Palworld under Pterodactyl. Its documented game support is Palworld, so it is a reference design, not something established to work with this fleet. [Game Server Snooze upstream](https://github.com/dawidkulpa/game-server-snooze).
 
@@ -103,7 +129,9 @@ Its difficult parts are protocol-specific:
 
 T6 is the more plausible first prototype because direct IP joining is documented, but transparent success still needs testing. For T5, a request-to-start flow is the more credible initial design. Do not assume that answering a generic status query reproduces Plutonium's registration or authentication behavior.
 
-**8. Conditional alternatives: host changes and engine support.** These merit investigation when the measurements point to them.
+### 8. Conditional alternatives: host changes and engine support
+
+These merit investigation when the measurements point to them.
 
 If the Docker host itself runs inside a VM, audit virtual CPU exposure and host contention. One T4/T5 operator reported resolving excessive CPU consumption by changing a Proxmox Windows VM from `x86-64-v3` to `host`. Another operator in the same thread reported normal behavior with `x86-64-v3`, so this is a conditional clue, not a universal fix or proof of a Plutonium regression. [Firsthand performance discussion](https://forum.plutonium.pw/topic/45013/server-performance-issues/11).
 
@@ -115,16 +143,20 @@ The most ambitious route is an upstream engine idle mode or supported plugin-ass
 
 This route has the best conceptual fit for an always-listed, low-CPU server, but the highest development and compatibility uncertainty. A useful upstream request would include clean-server profiles, per-thread CPU data, exact versions, and measured differences between zero players, connecting players, and active play.
 
-**Settings and shortcuts I would not prioritize.** Reducing `sv_fps` fleet-wide is not an established solution: Plutonium's changelog records T5 SP `sv_fps` becoming cheat-protected and a historical T6 Zombies dedicated tick-rate increase from 10 to 20 Hz. Neither entry establishes a safe idle-only override. Client FPS advice for `com_maxfps` also does not establish a dedicated-server fix. I found no documented common T5/T6 hibernation setting. [Plutonium changelog](https://plutonium.pw/docs/changelog/).
+## Lower-priority settings and shortcuts
+
+Reducing `sv_fps` fleet-wide is not an established solution: Plutonium's changelog records T5 SP `sv_fps` becoming cheat-protected and a historical T6 Zombies dedicated tick-rate increase from 10 to 20 Hz. Neither entry establishes a safe idle-only override. Client FPS advice for `com_maxfps` also does not establish a dedicated-server fix. I found no documented common T5/T6 hibernation setting. [Plutonium changelog](https://plutonium.pw/docs/changelog/).
 
 CPU shares and affinity can help manage contention, but they should not be mistaken for eliminating idle work. Docker documents that shares act when CPU is constrained and that a CPU set restricts where execution occurs. For actual utilization ceilings, use quotas; for elimination of unnecessary instances' work, stop them. [Docker resource controls](https://docs.docker.com/engine/containers/resource_constraints/#cpu).
 
-**How to choose with a small, repeatable trial.** First record the actual host/VM details, kernel, image digest, Wine version, Plutonium revision, scripts/plugins, map/ruleset, and CPU limits. Keep those fixed while changing one variable. Pinning an image alone is insufficient if the wrapper updates game binaries at startup; record the resulting binary versions too.
+## Run a repeatable trial
+
+First record the actual host/VM details, kernel, image digest, Wine version, Plutonium revision, scripts/plugins, map/ruleset, and CPU limits. Keep those fixed while changing one variable. Pinning an image alone is insufficient if the wrapper updates game binaries at startup; record the resulting binary versions too.
 
 Use one representative instance from each game, then expand successful changes to other maps. Suggested measurement windows are 10–15 minutes after startup settles, repeated after players join and all leave. These are proposed test windows, not product requirements.
 
 | Test                                                       | What it resolves                                          |
-| --- | --- |
+| ---------------------------------------------------------- | --------------------------------------------------------- |
 | Normal deployment, empty immediately after startup         | Initial baseline                                          |
 | Same instance after the last player disconnects            | Whether work accumulates or persists after play           |
 | Minimal scripts/plugins on the same map                    | Contribution from custom behavior                         |

docs/ui-refresh.md

@@ -1,30 +0,0 @@
-# UI refresh
-
-The interface keeps the existing dark palette and blue accent, with shared controls and task-focused pages.
-
-The follow-up [UI/UX review implementation](ui-ux-implementation.md) records the approved operational, recovery, accessibility, and mobile improvements, with recommendation coverage and validation.
-
-1. **Shared design system:** common headers, buttons, fields, dialogs, spacing, radii, and focus styles. Removed unused styles from retired layouts.
-2. **Libraries:** Plugins opens on the searchable library. Add plugin opens Upload / GitHub tabs. Scripts uses compact status filters; Configs uses a compact summary and Library / Inspect tabs.
-3. **Beta setup:** `/betas/new` guides Files → Servers → Test plan → Review. Exact versions, selected servers, and the plan survive Back / Continue. Existing beta selection links still redirect correctly.
-4. **Fleet density:** compact totals and attention summary, with grouping, sorting, and display options behind View options.
-5. **Server identity:** friendly map names and visible lobby numbers across cards, lists, beta setup, and operations. T6 variants retain their selected map name. Server details provides a copyable raw ID.
-6. **Deployment organization:** Active / Blueprints / History views. Failures and warnings link to filtered history. Destroy fleet lives in Fleet actions and retains its explicit preview.
-7. **Navigation:** Fleet / Library / Operations / Betas / Settings, with section navigation and a mobile menu. Existing route URLs remain usable.
-8. **Status and progress:** shared status labels and tones distinguish ordinary waiting, work in progress, attention, passing tests, and promotion. Deployment, campaign, and beta details share operation history.
-9. **Server detail hub:** live status, current operations, scoreboard, logs, effective configuration, inventory, diagnostics, and scoped campaign history. The scoreboard shows game-reported names, copyable GUIDs, score, kills, headshots, downs, and revives, with sortable stats, map, round, and update time. Final scores remain visible until the next game starts. Missing or stale player counts are unknown. Inventory identifies scripts, plugins, configs, and other files.
-10. **Dialogs and feedback:** in-app confirmations for deletion, rollback, reconciliation, and unsaved edits. File comparisons use a native modal dialog with Escape and focus restoration. Refresh errors retain loaded data and provide retry controls.
-11. **Keyboard and forms:** labeled fields, roving keyboard tabs, visible focus, a skip link, and separate navigation/action targets. Removed nested buttons and buttons inside campaign links.
-12. **Readability:** brighter secondary metadata, readable small labels, contrasting primary-button text, and full-opacity offline cards.
-13. **Page states:** compact loading and empty states, actionable failures, and retained snapshots during refresh errors.
-14. **Mobile:** wrapping identities, scrollable tab/filter rows, compact summaries, responsive forms, and sticky form actions.
-15. **Loading and rendering:** shared Fleet data, detail tabs loaded on demand, no overlapping scheduled page requests, and polling paused in hidden tabs. Watches can keep Fleet monitoring active. Long libraries, histories, and evidence lists progressively render more items. Server history is filtered before the API limit.
-
-## Maintaining the patterns
-
-- Use `SectionHeader` for page titles and primary actions. The containing page owns spacing between sections.
-- Use `Field` with `Input`, `Select`, or `Textarea` for associated labels, help, and errors. Give controls distinct `ariaLabel` values when several share one field.
-- Use `Dialog` for new modal workflows and `confirmAction` for ordinary confirmations. Browser unload protection remains native.
-- Use `StatusBadge` for workflow states and `ServerIdentity` for server names. Preserve raw IDs where operators need to copy or troubleshoot them.
-- Pair `Tabs` with a labeled panel. Use `ListFooter` for long lists and `pollVisible` for scheduled page refreshes.
-- Read live server data from the Fleet store instead of starting another `/api/servers` polling loop.

docs/ui-ux-implementation.md

@@ -1,57 +0,0 @@
-# UI/UX review implementation
-
-Implemented September 19, 2026, following [the approved review](ui-ux-review.md).
-
-The changes prioritize clear operational outcomes, reviewed action scope, recoverable input, and usable small-screen layouts. Existing server, file, beta, and launch semantics remain visible in the interface rather than being implied by a successful request.
-
-## Recommendation coverage
-
-| Review | Implemented behavior |
-| --- | --- |
-| 1. Stale Fleet data | Stale cards explicitly show **Players unknown**, retain labeled last-known counts and rosters, and stop advancing live durations. Fleet shows snapshot age and refresh/reconnection status. |
-| 2. Promotion impact | A promotion review lists exact versions, tested and additional servers, eligibility, staging, and restart policy. The backend checks a review token before writes; changed targets or composed config require a new review. Completion links to the resulting campaign. |
-| 3. Reconciliation scope | Selected-server and entire-fleet reviews are separate actions. The plan includes scan time, file operations and restart policy. Execution checks target identities and before/after content fingerprints; changed plans return for review. |
-| 4. Beta reading order | Outcome, blockers, deadline, actions, and server results precede evidence and activity. Shared activity history is collapsed, searchable, ordered, and progressively displayed. |
-| 5. Beta recovery | Deadlines and pause/retry consequences are explicit. A fresh attempt confirms evidence reset and restarts; expired tests explain rollback and plan reuse. Sign-off availability follows test/run readiness. |
-| 6. Launch input retention | Refreshing previews retains entered keys only for unchanged exact server IDs. Changed assignments are explained. Keys remain in memory; managing keys in another tab preserves the launch context. |
-| 7. Removal cancellation | Waiting removals can be withdrawn from operation lists/details. A database transition competes atomically with the worker claim before external deletion. Once removal or confirmed-deletion cleanup begins, cancellation is unavailable. |
-| 8. Partial outcomes | Reconciliation reports updated and skipped servers separately, with reasons and follow-up links. All-skipped work is an error outcome. Rollback feedback stays beside its initiating operation. |
-| 9. Scope clarity | Scope editing previews added/removed eligible servers. Script/plugin unscoping explains retained live copies and tracking detachment; configs explain composition impact. Completion links to reconciliation. Eligibility, promoted versions, and installed inventory have separate labels. |
-| 10. Accessible controls | Busy buttons retain their accessible names. Field labels focus the associated native control. Consequential errors use alert semantics; completion uses status semantics. Campaign server states have accessible descriptions. |
-| 11. Server investigation | Offline/stale summaries offer Diagnostics and Logs. URL parameters retain the selected tab, RCON diagnosis precedes telemetry history, and removal is under Server actions. |
-| 12. Beta drafts | Drafts retain exact versions, label, test plan, selected servers, and wizard step. Resume/discard is explicit, including returns through artifact links. Eligibility is rechecked and compatible selections retained. Unavailable browser storage triggers leave protection. |
-| 13. Local validation | Blueprints display total capacity and duplicate-map errors. Beta requirements show field-specific errors and units, expand/focus invalid checks, and validate quorum against the selected cohort. |
-| 14. Navigation context | Fleet views and library/operation filters are represented in URLs. Library loaded counts and appropriate selections survive detail navigation and Back. Server and log tabs have direct links. Explicit Fleet URL choices override saved personal defaults. |
-| 15. System setup | Settings opens a read-only System status page with configured/unconfigured/unreachable/unverified states, prerequisite instructions, saved-key counts, and first-server guidance. Empty/unconfigured Fleet links directly to setup. |
-| 16. Stable config preview | The last successful inspection remains visible during edits, with an updating state. Expanded keys survive refresh. Draft comparisons and ignored-content warnings remain near the editing task. |
-| 17. Config composition | Inspection numbers composition layers, identifies winning sources and superseded assignments, and compares draft values with promoted values. Structured editing supports adding a setting without raw syntax. |
-| 18. Beta target selection | Files show scope and eligible counts. Conflicts identify incompatible file scopes. Server picking supports search, game/map grouping, health/player context, selected counts and select-visible controls. |
-| 19. Evidence interpretation | Requirements explain the next qualifying condition using current observations. Relevant-evidence links apply server/event filters; readable summaries precede raw payloads and telemetry diagnosis is directly linked. |
-| 20. Artifact next action | The latest exact version and its test/active-beta action lead the detail page. Technical hashes/sizes are disclosures, version cards remain readable on phones, unscoped game filtering is consistent, and hidden selections are counted. |
-| 21. Import continuity | GitHub import shares pending state with its parent dialog, prevents dismissal/tab switches during import, and keeps a completion panel with version/beta links. Script batches show aggregate progress and sticky actions. |
-| 22. Key management | Game/map/lobby selectors generate the exact ID, with an expert raw-ID option. Rows show saved time, presence-versus-validation wording, deployment implications, and links to existing servers. Launch forms accept the exact missing/replacement key inline. |
-| 23. Blueprint coverage | Existing/queued inventory is distinguished from healthy playable servers, with linked coverage details. Collections group assignments and show a sticky key count and jump-to-missing-key action. Creation remains limited to the target shortfall. |
-| 24. Mobile overhead | Fleet avoids redundant headings/zero campaign summaries, empty libraries emphasize their next action, and Leaderboard totals collapse on phones with a compact selected-stat table. Long forms retain reachable actions and focus spacing. |
-| 25. Consistent hierarchy | Editable controls use stronger dedicated boundaries. Routes have descriptive titles, operational language is clearer, keyboard shortcuts have a help panel, and selected overflow tabs scroll into view without moving the page. |
-| 26. Investigation/handoffs | Logs support literal search/highlighting, copy, tail download, deep links, explicit follow/refresh state and jump-to-latest. A bounded recent-notifications panel retains results in the current browser tab. Beta rows include labels, age and blocker context. |
-
-The review explicitly made overlay-priority reordering and Leaderboard player history/recent-period views conditional on further design or operator demand. Those product extensions remain future discovery work; the composition explanations and mobile Leaderboard changes are implemented.
-
-## API and operational boundaries
-
-- Promotion uses `GET /api/betas/:id/promote` to review impact and `POST` with the returned token as `previewToken` to apply it. A changed plan returns HTTP 409 and needs renewed review.
-- Reconciliation uses `POST /api/drift` with `dryRun: true` and optional `servers`, then executes with the returned `expectedFingerprint`. Omitted servers explicitly mean the entire fleet. A stale review returns HTTP 409; changes detected during application are reported as skipped files.
-- `POST /api/deployments/:id/cancel` only withdraws a waiting removal. It does not restore a server after deletion has started.
-- System status exposes no credential values. Its optional connection check verifies Komodo network reachability, not credential validity. Remote game-file volumes remain explicitly unverified.
-- Drafts contain nonsecret beta setup. Launch keys are not stored in browser drafts. Recent notifications use tab-scoped session storage, capped at 50 entries, and do not imply background push delivery.
-- Log search/export covers the loaded tail, not an entire remote file or a full diagnostic archive.
-
-## Validation
-
-- Full regression suite: **473 tests passed across 44 files**. Added coverage includes changed promotion scope/config, changed reconciliation bytes/targets, cancellation racing a worker, draft parsing/compatibility, scope previews, config comparison and system-status semantics.
-- `npm run check`: **0 errors, 0 warnings**.
-- `npm run build`: **passed**. The existing SvelteKit CSRF configuration deprecation remains a tooling notice.
-- Local browser walkthroughs use an isolated database and synthetic populated API fixtures, with production integration credentials disabled. Fleet, server diagnostics, logs, Settings, Leaderboard, libraries, beta setup/progress, launches, keys, and reconciliation are exercised at desktop and phone widths.
-- Browser checks include the loading button accessibility tree, clickable native labels, URL/Back restoration, retained previews/drafts, selected-tab visibility, pending import ownership, and horizontal overflow. These are targeted checks, not a claim of complete accessibility conformance.
-
-Live Komodo provisioning, real game-server restarts and removals were not executed during verification. Their boundary behavior is covered by regression tests and simulated UI flows.

docs/ui-ux-review.md

@@ -1,359 +0,0 @@
-# Electric Cherry UI/UX review
-
-Reviewed September 19, 2026.
-
-Implementation follow-up: [completed changes and validation](ui-ux-implementation.md).
-
-Electric Cherry has a coherent visual foundation and unusually capable operational workflows. The highest-value next investment is making operational decisions easier: what is known, what is waiting, what an action will affect, and how to recover. Preserve the existing dark palette, shared controls, friendly server identities, and guided setup while improving those decisions.
-
-## Scope and evidence
-
-This review combines source inspection with a local browser walkthrough. It covers Fleet, server details and diagnostics, Scripts, Configs, Plugins, beta creation and progress, server deployments, blueprints, campaigns, keys, and Leaderboard. Existing design notes and workflow documentation informed the review.
-
-Browser exploration used an isolated database with Komodo/RCON credentials disabled. Nine top-level pages were examined at 1440×1000 and 390×844. Populated Fleet, server details, and beta scenarios used synthetic API responses. Beta setup, blueprint validation, and the browser accessibility tree were also exercised. Navigation was checked around its responsive breakpoint. Production integrations, actual restarts, and destructive actions were not executed.
-
-**Evidence labels:** **Rendered** means reproduced in the local browser, sometimes with an explicitly synthetic populated state. **Source** means verified in UI/API implementation. **Opportunity** means a proposed capability or design direction, rather than a defect. Recommendations are expert judgments; they are not findings from end-user interviews or a complete accessibility conformance audit.
-
-No application code was changed for this review. The accompanying screenshots contain synthetic review data.
-
-## What to preserve
-
-- The shared tokens, restrained blue accent, readable text, friendly map/lobby identities, and separate navigation/action targets.
-- The existing Fleet search, status filters, pins, card/table views, grouping, live freshness handling in aggregate counts, and lobby watches.
-- The four-step beta setup, exact version selection, explicit server opt-in, readable test-plan summaries, and review step.
-- Script batch intake, duplicate recognition, partial-success retry, and direct transfer into beta setup.
-- Config draft recovery, unsaved-edit protection, effective-setting inspection, and override provenance.
-- Deployment previews, shared-queue visibility, and separate recovery actions for health verification and filesystem cleanup.
-- Native modal dialogs, keyboard-operated tabs, visible focus, reduced-motion support, retained snapshots after refresh errors, and progressive list rendering.
-
-Many generic dashboard recommendations are already implemented. The findings below build on them.
-
-## Recommended order
-
-P1 means address first because the issue affects trust, consequential actions, lost work, or basic access. P2 means a meaningful workflow improvement. P3 means refinement or a capability to validate with operators. Relative effort is directional: S is a localized UI change; M crosses several components or state flows; L requires a new backend operation or product capability.
-
-| First work | Recommendation | Priority | Relative effort |
-| --- | --- | --- | --- |
-| 1 | Label stale player data consistently | P1 | S |
-| 2 | Preserve loading-button accessible names | P1 | S |
-| 3 | Put beta results and decisions above the event log | P1 | S–M |
-| 4 | Preview promotion targets and link the resulting campaign | P1 | M |
-| 5 | Make reconciliation scope and partial results explicit | P1 | M |
-| 6 | Preserve keys when launch previews refresh | P1 | M |
-| 7 | Explain blueprint and test-plan validation locally | P1/P2 | S–M |
-| 8 | Surface beta deadlines, reset consequences, and valid recovery actions | P1 | M |
-
-Cancellation of queued destruction is also important, but needs an explicit backend boundary and should be designed as a separate capability.
-
-## Recommendations
-
-### 1. Make stale Fleet cards unmistakable — P1 · Rendered + Source
-
-Fleet correctly excludes stale servers from live player totals, but passes their old player counts into cards. A stale card can still display a large blue player count, roster, and advancing match duration; its main distinction is an amber status dot and an older “Last seen” timestamp. In the synthetic review, the header reported **3 players online**, while an additional stale card prominently displayed **4**.
-
-Show **Players unknown** or an em dash as the primary value. Retain historical information under “Last known: 4 players, 3 minutes ago,” with an explicit **Stale** label. Apply the same current-versus-last-known distinction to roster and timing. Add the age of the last successful Fleet refresh and a reconnecting state when appropriate.
-
-Acceptance: when a fresh server ages into stale status, both card and table stop presenting its previous player count as current. Preserve historical context without requiring interpretation of color.
-
-Sources: [Fleet page](../src/routes/+page.svelte), lines 56, 62, 208; [ServerCard](../src/lib/ui/components/ServerCard.svelte), lines 72, 114, 150. [Screenshot](ux-review-assets/fleet-stale-count.png).
-
-### 2. Review the full impact of promotion — P1 · Source
-
-“Promote” immediately sends a request. The backend expands the tested files to all currently matching scope targets and queues restarts for additional servers. The UI does not show that expansion at the moment of approval. Afterward, it ignores the returned campaign ID.
-
-Add a concise promotion review with exact versions, tested servers, additional eligible servers, and restart behavior. Use a final label such as **Promote to 12 servers**. Explain eligibility, file staging, and activation separately. After submission, show **Promotion created — 9 servers awaiting activation** with **View campaign**. Keep “Promoted version” distinct from “Running everywhere.”
-
-Acceptance: an operator who tested one lobby can identify the entire affected set and follow remaining restarts without searching another section.
-
-Sources: [Beta detail](../src/routes/betas/[id]/+page.svelte), lines 72–79; [promotion implementation](../src/lib/server/fleet/beta.ts), lines 778, 1017, 1060.
-
-### 3. Make drift reconciliation match the reviewed scope — P1 · Source
-
-The drift list can be filtered by server, but Reconcile posts an empty body and affects the full fleet. The confirmation includes counts but omits automatic restarts. Execution rescans, so the action can also differ from the displayed scan.
-
-Separate **Reconcile selected servers** from **Reconcile entire fleet**. Show a file-level plan, scan time, affected servers, and activation policy. If the reviewed plan changes materially, show the updated plan before execution. The API already accepts `servers` and `dryRun`, providing a starting point.
-
-Acceptance: filtering or selecting one server never leaves the action scope ambiguous. An operator can explain which files will be restored or removed and when servers will restart.
-
-Sources: [Campaigns page](../src/routes/campaigns/+page.svelte), lines 118, 124, 203, 268; [drift API](../src/routes/api/drift/+server.ts), line 28; [drift implementation](../src/lib/server/fleet/drift.ts), line 265.
-
-### 4. Put beta outcome and next action above activity — P1 · Rendered + Source
-
-Beta detail renders up to 25 expanded activity events before files, requirements, per-server progress, and actions. In a synthetic three-server beta with three requirements per server and 25 events:
-
-| Content starts | Desktop 1440×1000 | Mobile 390×844 |
-| --- | ---: | ---: |
-| Activity | 436 px | 720 px |
-| Test plan/results | 2,623 px | 3,590 px |
-| Pending sign-off | 3,528 px | 4,846 px |
-| Promote / Rollback | 4,011 px | 5,462 px |
-
-Place a compact outcome summary first: passed servers, outstanding conditions, deadline, and primary action. Follow with a failed/waiting-first server table. Put activity under a disclosure or separate tab. Keep the useful operation-flow steps, but reduce the oversized started/finished block.
-
-Acceptance: without scrolling through historical events, an operator can answer “What is blocking this beta, and what can I do?”
-
-Sources: [Beta detail](../src/routes/betas/[id]/+page.svelte), line 137; [OperationHistory](../src/lib/ui/components/OperationHistory.svelte), lines 21, 46. [Screenshot](ux-review-assets/beta-activity-first.png).
-
-### 5. Make beta time and recovery rules visible — P1 · Source
-
-Progress describes a testing window but does not show the actual expiry or time remaining. Waiting for empty servers consumes the window; pausing and retrying retain the original deadline. Starting a fresh attempt resets evidence and causes new restarts. Sign-off controls can appear in states where the API rejects them.
-
-Show **Ends tomorrow at 14:30 · 6h remaining**, plus **Pausing does not extend this deadline**. Explain evidence reset and restart consequences before a fresh attempt. Offer only valid actions, with a reason for unavailable sign-off. For deadline expiry, explain the required rollback/new-beta sequence and offer a way to reuse the plan.
-
-Acceptance: paused, failed, inconclusive, and passed tests each have an understandable next step; a retry never silently implies extra testing time.
-
-Sources: [BetaTestProgress](../src/lib/ui/components/BetaTestProgress.svelte), lines 29, 35, 53; [test action API](../src/routes/api/betas/[id]/test/+server.ts), lines 21, 48–53; [test-plan documentation](beta-test-plans.md).
-
-### 6. Preserve launch inputs through preview refresh — P1 · Source
-
-Refreshing a single-server preview clears its entered key. Refreshing a collection clears all entered keys. Preview expiry and capacity changes are normal recovery paths, so users can lose substantial input while doing what the UI requires.
-
-Keep entered keys in memory, indexed by exact server ID. Reuse a key only when that ID survives the refreshed allocation; flag changed assignments explicitly. Preserve nonsecret launch choices across a visit to Manage keys. Do not persist raw keys into local browser drafts.
-
-Acceptance: after entering five keys, an unchanged preview refresh retains all five. A new lobby assignment never inherits another server’s key.
-
-Sources: [DeployServerDialog](../src/lib/ui/components/DeployServerDialog.svelte), lines 41–45, 66, 123; [DeployCollectionDialog](../src/lib/ui/components/DeployCollectionDialog.svelte), line 27.
-
-### 7. Give queued destructive work a withdrawal window — P1 · Opportunity
-
-Server removal can wait for occupied or unreachable lobbies. Current detail pages provide resume actions but no cancellation before destruction starts. A delayed command can therefore outlive the operator’s original intention.
-
-Add **Cancel queued removal** while the operation is still waiting, with a precise backend transition after which cancellation is unavailable. For fleet removal, show per-server waiting/removing/completed states and cancellation eligibility. Clearly distinguish cancelling a request from restoring a server already removed.
-
-Acceptance: an operator can withdraw a waiting removal, and a race with worker execution cannot leave the UI claiming cancellation after removal has begun.
-
-Sources: [DestroyServerModal](../src/lib/ui/components/DestroyServerModal.svelte), lines 36–40; [HousePartyDialog](../src/lib/ui/components/HousePartyDialog.svelte), line 51; [deployment detail](../src/routes/provisioning/[id]/+page.svelte), lines 110–123. Existing consequence confirmations should remain.
-
-### 8. Report partial success as a distinct outcome — P1 · Source
-
-Reconciliation returns per-server skipped reasons, but the UI reads aggregate changed-file counts. Errors and success messages also share a blue-toned flash container. Rollback feedback can appear in the drift card far from the row that initiated it.
-
-Use structured completion feedback: **7 servers updated, 2 skipped**, with reasons, relevant retry actions, and a campaign link. Place feedback beside the affected task. Keep success, partial success, and failure visually and programmatically distinct.
-
-Acceptance: a request that skipped every target cannot look like an uneventful successful run; the operator can find the unresolved servers immediately.
-
-Sources: [Campaigns page](../src/routes/campaigns/+page.svelte), lines 128–138, 181–184, 299, 508; [drift implementation](../src/lib/server/fleet/drift.ts), lines 349, 371, 383–410.
-
-### 9. Explain scope changes and distinguish eligibility from installation — P1 · Source
-
-The artifact scope dialog offers selectors and Save. Unscoping scripts/plugins detaches inventory tracking while leaving live copies; unscoping configs changes the intended composed configuration. Both can create reconciliation work. The UI discards the returned detach result for scripts/plugins. The detail label “Applies to” also represents matching scope, not verified installed versions.
-
-Preview added/removed eligible servers. Explain retained live files when unscoping, show the affected count afterward, and link to the relevant reconciliation view. Determine config impact from composition rather than the script/plugin detach count. Use separate labels for **Eligible servers**, **Promoted version**, and **Installed version**.
-
-Acceptance: changing eligibility never suggests that running servers have already changed. An operator knows when follow-up reconciliation is required.
-
-Sources: [ArtifactDetail](../src/lib/ui/components/ArtifactDetail.svelte), lines 183, 298, 380; [scope API](../src/routes/api/scripts/[id]/scope/+server.ts), lines 12, 30; [unscoping behavior](../src/lib/server/fleet/beta.ts), lines 1081–1108.
-
-### 10. Keep controls named and feedback accessible during actions — P1 · Rendered + Source
-
-The shared Button hides its label with `visibility: hidden` while its spinner is hidden from assistive technology. Browser accessibility inspection confirmed that **Refresh** becomes an unnamed button while busy. It retains `disabled` and `busy`, but loses its name.
-
-Preserve the accessible label during loading and announce meaningful progress separately. Audit action failures for alert semantics, and success messages for status semantics. Replace per-server status dots whose explanation exists only in hover text with visible or accessible text. Ensure Field labels focus their associated controls; current label context provides accessible naming, but many visible labels are divs.
-
-Acceptance: the same action has a stable accessible name before, during, and after execution; keyboard and touch users can identify which server failed without hovering.
-
-Sources: [Button](../src/lib/ui/components/primitives/Button.svelte), lines 63–71, 155; [Field](../src/lib/ui/components/primitives/Field.svelte), line 25; [campaign mini-pipelines](../src/routes/campaigns/+page.svelte), line 380. These recommendations follow W3C guidance on [Name, Role, Value](https://www.w3.org/WAI/WCAG22/Understanding/name-role-value.html) and [Status Messages](https://www.w3.org/WAI/WCAG22/Understanding/status-messages.html).
-
-### 11. Make server detail lead toward the likely next task — P2 · Rendered + Source
-
-An offline server still opens on Scoreboard. In the reviewed state this shows “Waiting for game data,” while the only header action is Destroy. The diagnostic view places telemetry request history before the RCON diagnostic, even when connectivity is the immediate issue.
-
-Keep a stable tab choice, but add an issue summary above it: **RCON has not answered for 1 hour — Diagnose connection · Open logs**. Put removal under a secondary Server actions menu. Allow direct URLs to Logs, Config, and Diagnostics so operation failures can link to the relevant place. Prioritize the diagnostic relevant to the reported failure.
-
-Acceptance: from an offline Fleet card, the operator can reach the relevant diagnostic in one clear action. Do not automatically switch tabs while someone is reading.
-
-Sources: [server detail](../src/routes/servers/[name]/+page.svelte), lines 103, 226, 331–337; [PlayerScoreboard](../src/lib/ui/components/PlayerScoreboard.svelte), line 37. [Screenshot](ux-review-assets/offline-server.png).
-
-### 12. Preserve beta drafts and compatible server selections — P2 · Source
-
-The wizard preserves Back/Continue state, but navigation or reload loses the draft. Changing or removing a file clears every selected server without explanation.
-
-Extend the Config editor’s existing draft pattern to beta setup: resume/discard an unfinished draft, retain exact version IDs, and revalidate availability when reopening. Preserve selected servers that still satisfy the new scope intersection; explain any removals. Warn before leaving a dirty draft when recovery is unavailable.
-
-Acceptance: visiting a file’s detail to check a version does not require rebuilding the test plan and server selection.
-
-Sources: [beta setup](../src/routes/betas/new/+page.svelte), lines 34, 114, 130, 139; [existing ConfigEditor pattern](../src/lib/ui/components/ConfigEditor.svelte), lines 27–79.
-
-### 13. Explain why a form cannot proceed — P1/P2 · Rendered + Source
-
-With six distinct blueprint maps set to 20 lobbies each, both Save actions are disabled with no explanation of the 100-server limit. This was reproduced in the browser. Duplicate maps can also pass client validation and fail on submission. Beta-plan errors can refer to internal names such as `timeoutHours` or an unspecified requirement target.
-
-Show **120 / 100 target lobbies — reduce the total by 20**. Prevent or merge duplicate maps. Associate errors with the relevant controls, expand a collapsed invalid requirement, and move focus to it. Use target units such as rounds, matches, minutes, or players. Validate quorum against selected servers before final submission.
-
-Acceptance: every disabled forward/save action has a discoverable reason, and a rejected submission points directly to the field needing correction.
-
-Sources: [BlueprintBuilder](../src/lib/ui/components/BlueprintBuilder.svelte), lines 18, 81–90; [blueprint validation](../src/lib/server/fleet/launch-recipes.ts), lines 57–63; [BetaPlanBuilder](../src/lib/ui/components/BetaPlanBuilder.svelte), line 123; [beta setup validation](../src/routes/betas/new/+page.svelte), line 68.
-
-### 14. Preserve task context in URLs and navigation — P2 · Source
-
-Library filters, sort, loaded count, and selection mostly live in component state. Server detail tabs also default locally. Moving among details and returning can force the operator to rebuild context; another person cannot open the same filtered working view.
-
-Put filter/sort/view/tab state in URL parameters. Preserve scroll and appropriate selection state across detail navigation. Keep Fleet display preferences in local storage, but distinguish personal defaults from an explicitly shared view. Reuse the Leaderboard’s existing URL-filter pattern.
-
-Acceptance: browser Back restores the operator’s working set, and a copied Diagnostics or filtered library URL opens the intended view.
-
-Sources: [Scripts](../src/routes/scripts/+page.svelte), lines 13–23; [Plugins](../src/routes/plugins/+page.svelte), lines 44–49; [ConfigLibrary](../src/lib/ui/components/ConfigLibrary.svelte), lines 24–29; [Leaderboard](../src/routes/leaderboard/+page.svelte).
-
-### 15. Add a setup and integration-health home — P2 · Opportunity
-
-Settings currently leads only to server keys. Integration information is available deep in server diagnostics; the empty Fleet encourages creation even when external services are not configured. The first-run user needs a short path from prerequisites to a functioning lobby.
-
-Add a read-only System status page and a contextual first-run checklist: Komodo configured/reachable, RCON configured, game files available where verifiable, keys available, and first server ready. Link directly to relevant setup instructions. Distinguish **not configured**, **unreachable**, and **server offline**. Avoid repeatedly displaying the full checklist after setup succeeds.
-
-Acceptance: someone with an empty installation can identify the next missing prerequisite without opening an unrelated server or reading the full README.
-
-Sources: [navigation](../src/routes/+layout.svelte), lines 80–96; [Fleet empty state](../src/routes/+page.svelte), line 177; [integration store](../src/lib/ui/stores/fleet.ts), line 13; [README setup](../README.md).
-
-### 16. Keep config previews stable while typing — P2 · Source
-
-ConfigInspector clears its current inspection immediately on an edit, then waits for its debounce and request. The effective settings disappear while typing, which can disrupt comparison and expanded setting details.
-
-Retain the last result with **Updating preview…**, replace it when the current request completes, and preserve expanded keys and scroll position. Identify values changed by the draft. Show ignored-content warnings near Save when they affect the submitted config.
-
-Acceptance: typing several characters does not repeatedly collapse the working preview or shift the editor layout.
-
-Sources: [ConfigInspector](../src/lib/ui/components/ConfigInspector.svelte), lines 35–54, 75–79; [ConfigTextEditor](../src/lib/ui/components/ConfigTextEditor.svelte).
-
-### 17. Make config composition easier to reason about — P2 · Opportunity
-
-The inspector already provides useful provenance, but overlay precedence depends on creation order. Understanding why a setting wins can require understanding historical ordering.
-
-Extend the existing final-value and winning-source summaries with a numbered composition sequence and per-layer overridden-setting counts. Consider explicit overlay priorities only with a before/after effective-value preview. Add an “Add setting” key/value row to the structured editor, while preserving raw text mode.
-
-Acceptance: an operator can explain a setting’s final value and add a basic assignment without reverse-engineering creation history or switching to raw syntax unnecessarily.
-
-Sources: [ConfigInspector](../src/lib/ui/components/ConfigInspector.svelte), lines 79–97; [ConfigTextEditor](../src/lib/ui/components/ConfigTextEditor.svelte), lines 19–35.
-
-### 18. Make beta compatibility and target selection visible — P2 · Source
-
-File picking exposes little scope context, while target eligibility is the intersection of all file scopes. An empty intersection produces a generic instruction to choose different scripts. Candidate servers then appear as a flat checkbox collection.
-
-Show scope and eligible count beside each file/version. Explain concrete conflicts, such as **This Origins script and Town config have no shared servers**. Add target search, map/game grouping, selected count, and current player/health context. Include select-visible for larger fleets while preserving explicit opt-in.
-
-Acceptance: users can resolve an incompatible bundle without trial and error, and can select a sensible test cohort without scanning every raw entry.
-
-Sources: [beta setup](../src/routes/betas/new/+page.svelte), lines 159, 237, 276–278.
-
-### 19. Explain what stalled requirements are waiting for — P2 · Source
-
-Counters such as `current / target` do not sufficiently distinguish missing telemetry, too few players, a pending game end, or the next report needed to credit gameplay time. Raw evidence JSON leaves that interpretation to the operator.
-
-Add a next-condition explanation per requirement: **Waiting for a completed match with at least 2 players; last report 4m ago**. Link to the relevant telemetry diagnostic. Summarize evidence in ordinary language, then offer raw JSON. Add server/event filters and “Show evidence for this check.”
-
-Acceptance: a requirement stuck at zero has a visible explanation and a direct investigation path.
-
-Sources: [BetaTestProgress](../src/lib/ui/components/BetaTestProgress.svelte), lines 46, 67; [evaluation](../src/lib/server/fleet/beta-evaluation.ts), line 79; [telemetry contract](beta-test-plans.md).
-
-### 20. Put the next release action at the top of artifact details — P2 · Source
-
-The header emphasizes Edit scope and Upload new version. Testing is lower in version history, and the “open a beta” link goes to the generic beta list. Version rows emphasize checksum, size, and timestamps; their grid also has fewer explicit columns than some rows have children.
-
-Add a latest-version summary even before first promotion, with **Test latest version**, **View active beta**, or **Review passed test** as appropriate. Put version, state, change notes, and action first. Move hash and byte size into technical details. Use deliberately labeled mobile version cards.
-
-Also standardize game-filter behavior for unscoped files across Scripts and Plugins, and show **3 selected · 1 hidden by filters** when selections are concealed.
-
-Acceptance: after uploading a first version, the next useful action is immediately visible and carries that exact version into setup.
-
-Sources: [ArtifactDetail](../src/lib/ui/components/ArtifactDetail.svelte), lines 269–278, 312–327, 347–359, 500; [Scripts filters](../src/routes/scripts/+page.svelte), lines 37, 135; [Plugins filters](../src/routes/plugins/+page.svelte), line 51.
-
-### 21. Own import progress at the dialog level — P2 · Source
-
-GitHub import tracks busy state inside its child component, while the parent dialog and source tabs only observe file-upload state. The import can continue after dismissal or a tab change, then complete with a navigation or modal-close callback. Import-only success content is immediately hidden when the parent closes.
-
-Share the pending operation state with the parent. Either keep the import visibly in progress, or allow dismissal with an explicit persistent completion notification. Avoid surprise navigation after the user has moved on. Show **Plugin imported — View version · Start beta** after success.
-
-For long script batches, add aggregate inspection/upload progress and a sticky action footer while preserving current partial-success retry.
-
-Acceptance: closing or switching import UI never makes the request’s outcome disappear or causes unexplained later navigation.
-
-Sources: [Plugins dialog](../src/routes/plugins/+page.svelte), lines 217–220; [PluginGithubImport](../src/lib/ui/components/PluginGithubImport.svelte), lines 18, 60–75, 141; [ScriptUploadDrawer](../src/lib/ui/components/ScriptUploadDrawer.svelte), lines 59, 101, 185.
-
-### 22. Complete key management within the launch journey — P2 · Source + Opportunity
-
-The standalone key form asks users to type a machine ID despite existing game/map catalogs. Key rows omit their available update timestamp. “Deployed” indicates that the server exists, not that it uses the latest saved key.
-
-Generate the exact ID from game/map/lobby selectors, retain a raw-ID option for experts, and link deployed rows to their server. Show **Saved on… · Used on next deployment**. Let the launch flow add or replace its exact missing key without losing launch context. Distinguish a stored key’s presence from credential validation.
-
-Acceptance: users can add the right key without manually constructing an identifier and can understand whether a running server is affected by an update.
-
-Sources: [server keys](../src/routes/server-keys/+page.svelte), lines 14, 81–105; [launch dialog](../src/lib/ui/components/DeployServerDialog.svelte), line 123.
-
-### 23. Separate blueprint coverage from playable capacity — P2 · Source
-
-A blueprint can be satisfied by existing stopped servers. Its “Existing / queued” identities are exposed through hover text, without an obvious path to inspect them. Meeting an inventory target does not establish that those lobbies are playable.
-
-Show **3 existing: 1 healthy, 1 stopped, 1 queued** with an expandable linked list. When no creation is needed, offer **Review existing servers**. Retain the current create-only-shortfall semantics, and say explicitly that excess servers remain and existing servers are not updated.
-
-For collections needing many keys, add a sticky summary, **12 of 20 keys ready**, grouping, and jump-to-next-missing-key.
-
-Acceptance: “Target satisfied” cannot be mistaken for “All lobbies ready to play.”
-
-Sources: [DeployCollectionDialog](../src/lib/ui/components/DeployCollectionDialog.svelte), lines 57–59, 72–84.
-
-### 24. Reduce mobile overhead before useful content — P2 · Rendered + Opportunity
-
-The inspected mobile pages generally fit the viewport, but some put substantial control and summary space ahead of the task. On the reviewed 390px Fleet, the first server card began around 614px down. The empty Scripts screen still shows zero-count filters, search, sorting, and select-visible controls. Leaderboard rankings begin below its filters and five summary metrics.
-
-On phones, prioritize one compact health summary, search, and the most useful status filter; keep display options in a small secondary control. Simplify true empty states to the relevant next action. Let Leaderboard summary metrics collapse and offer a compact player/selected-stat presentation while retaining the full scrollable table. Add sticky summaries/actions to long collection and batch forms, checking that they do not cover focused fields.
-
-Acceptance: a phone user reaches a real server, record, or first-run action quickly, without losing access to advanced controls. Validate sticky behavior with keyboard navigation and zoom, following [W3C focus visibility guidance](https://www.w3.org/WAI/WCAG22/Understanding/focus-not-obscured-minimum).
-
-Sources: [Fleet layout](../src/routes/+page.svelte), lines 180–202, 334; [Scripts](../src/routes/scripts/+page.svelte); [Leaderboard layout](../src/routes/leaderboard/+page.svelte), lines 126–187.
-
-### 25. Sharpen visual hierarchy and vocabulary consistently — P2/P3 · Rendered + Source
-
-The visual language is coherent, but small controls, nested cards, repeated section headings, and technical metadata can compete with primary decisions. Input borders use the same very faint token family as passive surfaces. Some routes also retain the generic browser title “Electric Cherry,” making multiple open tabs hard to distinguish.
-
-Use a dedicated editable-control boundary token, reserve strong surfaces for actionable sections, and reduce repeated title/subtitle layers in builders. Keep body text and important form instructions comfortably readable; use compact metadata selectively. Add descriptive route titles, especially server name and lobby number.
-
-Use “server” consistently for the managed unit; introduce “lobby” as its player-facing meaning. Prefer **Preview missing servers** over “Conform fleet.” Use “Files” consistently for scripts/configs/plugins where a collective noun is needed. Keep raw IDs, RCON, Komodo, and hashes accessible in technical details. Expose the existing keyboard shortcuts through a small help panel.
-
-Acceptance: primary tasks are identifiable by placement and wording, and multiple browser tabs are distinguishable. Contrast should be measured before claiming a conformance result.
-
-Sources: [tokens](../src/lib/ui/styles/tokens.css), lines 57–71, 86–87; [BetaPlanBuilder](../src/lib/ui/components/BetaPlanBuilder.svelte); [navigation and shortcuts](../src/routes/+layout.svelte), lines 37–65; [deployment blueprint actions](../src/routes/provisioning/+page.svelte), lines 125–132.
-
-### 26. Make investigation and handoffs easier over time — P3 · Opportunity
-
-Logs provide refresh, auto-follow behavior, and a useful tail, but no in-panel search or downloadable investigation bundle. Lobby watches work across navigation, while completed notifications are transient application state and are not a reviewable activity inbox. Beta list rows could also provide more timing and blocker context.
-
-Add log search/highlighting, copy/download, and a deep link preserving the selected server/log. Make the existing auto-refresh/follow state explicit and offer **Jump to latest**. Consider a small recent-notifications panel for completed watches and actions, without implying background push support. Add beta age, an operator label, and a one-line blocker to list rows. For Leaderboard, investigate player detail/history and recent-period views only after confirming operator demand; existing all-time statistics and counting explanations are a solid base.
-
-Acceptance: operators can return to a completed notification or share a precise diagnostic view without reconstructing the investigation.
-
-Sources: [LogPanel](../src/lib/ui/components/LogPanel.svelte); [notification store](../src/lib/ui/toast.ts); [watch handling](../src/lib/ui/stores/fleet.ts); [beta list](../src/routes/betas/+page.svelte); [Leaderboard](../src/routes/leaderboard/+page.svelte).
-
-## Suggested layout for operational detail pages
-
-Use the same reading order across beta, campaign, and server-deployment details:
-
-1. **Outcome and next action:** current state, elapsed/remaining time, blocker, primary recovery or completion action.
-2. **Affected servers:** progress counts and a failed/waiting-first list with links to relevant diagnostics.
-3. **Change summary:** exact files/versions or deployment recipe, including what is staged versus active.
-4. **Evidence and activity:** searchable or collapsible history, with the latest meaningful event summarized near the top.
-
-This gives the existing shared components a consistent purpose without forcing every operation into identical controls.
-
-## Practical delivery sequence
-
-**First pass — clarity and trust:** address stale card values, unnamed loading buttons, silent blueprint limits, beta results placement, and misleading/remote action feedback. These are bounded changes with clear reproduction cases.
-
-**Second pass — complete action flows:** add promotion and reconciliation reviews, explicit beta timing/retry semantics, retained launch keys, beta drafts, import completion state, and links from outcomes to their campaign or recovery destination.
-
-**Third pass — scale and polish:** improve target pickers, library context restoration, mobile density, config composition, setup status, and evidence navigation. Design queued-operation cancellation separately with its backend guarantees.
-
-## Validation plan
-
-Use a fixture with online, occupied, empty, starting, stale, and unreachable servers; a library large enough to require filtering; an expired preview; an active beta with a long history; and partial deployment failures. Ask representative operators to perform these tasks without narration or hints:
-
-| Task | Observable success |
-| --- | --- |
-| Identify a safe empty lobby | User distinguishes confirmed empty from stale/unknown counts. |
-| Promote a beta tested on one server | User correctly predicts the entire affected set and restart behavior. |
-| Reconcile one selected server | User can name the scope and identify skipped or failed work afterward. |
-| Recover a stalled beta | User finds the blocker, deadline, and valid next action without reading the event log. |
-| Refresh a five-key launch preview | Unchanged server IDs retain their entered keys; changed IDs are clearly called out. |
-| Fix an invalid blueprint | User finds the total/duplicate issue without guessing why Save is disabled. |
-| Investigate an offline server on a phone | User reaches the relevant diagnostic promptly and keeps server context. |
-| Complete a flow using keyboard/assistive technology | Actions keep names while loading; focus remains visible; errors and completion are announced. |
-
-Measure completion, errors, unnecessary backtracking, lost input, and time to identify the next action. Establish a baseline before setting numerical improvement targets. Source review and synthetic browser scenarios establish the issues above, but observed operator behavior should guide the final interaction details.

docs/watchdog.md

@@ -1,80 +0,0 @@
-# Watchdog
-
-**Operations → Watchdog** detects unhealthy servers and drift, records each
-incident, and links its recovery attempts to Provisioning. File drift also opens
-a Campaign with the observed file differences. Automation is enabled by default;
-**Pause automation** persists across application restarts. Scanning and verification
-continue while paused, but new actions and queued Watchdog actions are held.
-Already submitted actions continue to completion.
-
-The worker scans once per minute while Electric Cherry is running. **Scan now**
-requests a scan immediately. Repeated button presses do not count as independent
-health failures. A health incident must have three observations spanning at least
-two minutes before an automatic restart is queued. Starting or restarting
-containers get five minutes before recovery is requested. Unknown Komodo responses
-are recorded without interpreting them as a stopped container.
-
-## Recovery
-
-A health incident gets one restart, then one rebuild if the restart has not
-recovered within five minutes. Both are normal Provisioning jobs with durable
-history and shared capacity limits. Escalation transfers only that incident's
-restart recovery slot to the rebuild. An unsuccessful rebuild requires operator
-attention; Watchdog does not start an endless restart/rebuild loop. Verification
-of an already submitted rebuild continues, including after the incident is marked
-as needing attention.
-
-An intentional **Stop** is retained as the server's desired state. Watchdog leaves
-it stopped until an explicit Start, Restart, or Rebuild request changes that intent.
-Active creations, removals, betas, and campaigns defer new Watchdog actions.
-
-Provisioning checks current container health before using game status. A disruptive
-action requires a confirmed stopped container or a fresh zero-player reading from
-RCON or accepted telemetry. A positive player count from either source prevents
-an interruption. Telemetry evidence must have been received within 45 seconds;
-future, malformed, stale, and pre-restart reports cannot verify a new process.
-A game-end event retains its reported roster; it does not imply zero players.
-Fresh telemetry can also verify game activity and the selected engine map after
-Komodo reports the container healthy. A healthy container with neither current
-RCON status nor current telemetry remains queued for player-count verification.
-
-## Drift
-
-Drift includes missing promoted files, changed bytes, and extra files in managed
-script, config, and plugin destinations. Config comparison uses the composed
-baseline and overlays with the server's selected map. Plutainer's unmanaged default
-config remains exempt until a baseline is promoted. Active betas are deferred;
-missing server mounts are reported as scan errors instead of being treated as
-empty directories.
-
-Watchdog rebuilds a drifting server in place using the same identity, map, port,
-and saved server key. **Rebuild deletes all old instance files.** The new instance
-uses the latest promoted setup when provisioning starts. A missing saved key
-leaves the incident needing attention without deleting the old server.
-
-The linked Campaign captures before/after evidence at detection time. Its activity
-identifies this as the detected plan, and it follows the rebuild's outcome without
-queuing an additional restart. Promotions can change while the rebuild is queued;
-the Provisioning launch recipe records the files actually selected for that launch.
-Campaign evidence and Watchdog history survive subsequent rebuilds and removals.
-
-## Review and retry
-
-The list can be filtered by server, incident type, and status. Each detail page
-shows the triggering evidence, timestamped events, and links to every attempt.
-**Dismiss** suppresses further automatic attempts until the condition clears.
-**Retry** explicitly starts a new bounded attempt after inspecting a dismissed or
-failed incident. Active Provisioning operations must finish or be cancelled first.
-A verified recovery resolves the incident automatically.
-
-## API
-
-- `GET /api/watchdog`: automation setting, last scan, scan errors, and latest incidents.
-- `POST /api/watchdog` with `{ "action": "scan" }`: run a scan now.
-- `POST /api/watchdog` with `{ "enabled": false }` (or `true`): pause/resume automation.
-- `GET /api/watchdog/:id`: incident, event history, and linked actions.
-- `POST /api/watchdog/:id` with `{ "action": "retry" }` or `{ "action": "dismiss" }`.
-
-The legacy `/api/drift` GET and dry-run POST still return previews. Mutating POST
-requests return HTTP 410 and direct clients to Watchdog; file repair is now a
-provisioning rebuild instead of an immediate rewrite.

entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 set -eu
 
+if [ -n "${SQLITE_TMPDIR:-}" ]; then
+    mkdir -p "$SQLITE_TMPDIR"
+fi
+
 echo "[entrypoint] starting node server on ${HOST:-0.0.0.0}:${PORT:-3000}"
 exec node build/index.js

eslint.config.js

@@ -0,0 +1,64 @@
+import js from '@eslint/js';
+import { defineConfig, globalIgnores } from 'eslint/config';
+import prettier from 'eslint-config-prettier';
+import svelte from 'eslint-plugin-svelte';
+import globals from 'globals';
+import ts from 'typescript-eslint';
+import svelteConfig from './svelte.config.js';
+
+export default defineConfig(
+    globalIgnores([
+        '**/node_modules/',
+        '.svelte-kit/',
+        'build/',
+        'data/',
+        'test-fixtures/',
+        'static/agent/',
+        '.claude/',
+        '.gstack/'
+    ]),
+    js.configs.recommended,
+    ts.configs.recommended,
+    svelte.configs.recommended,
+    prettier,
+    svelte.configs.prettier,
+    {
+        // The controller is deployed at the origin root, including imperative navigation in TS.
+        rules: { 'svelte/no-navigation-without-resolve': 'off' }
+    },
+    {
+        languageOptions: {
+            globals: { ...globals.browser, ...globals.node }
+        },
+        rules: {
+            '@typescript-eslint/no-unused-vars': [
+                'error',
+                {
+                    argsIgnorePattern: '^_',
+                    varsIgnorePattern: '^_',
+                    caughtErrorsIgnorePattern: '^_',
+                    ignoreRestSiblings: true
+                }
+            ]
+        }
+    },
+    {
+        files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js'],
+        languageOptions: {
+            parserOptions: {
+                parser: ts.parser,
+                svelteConfig
+            }
+        },
+        rules: {
+            // Existing lists include intentionally positional rows and repeated primitive values.
+            'svelte/require-each-key': 'off',
+            // Local collection helpers do not need reactive replacements.
+            'svelte/prefer-svelte-reactivity': 'off',
+            // Preserve the existing state-reset behavior during tooling adoption.
+            'svelte/prefer-writable-derived': 'off',
+            // Explicit string interpolation preserves whitespace around inline controls and logs.
+            'svelte/no-useless-mustaches': 'off'
+        }
+    }
+);

komodo/stack.toml

@@ -1,43 +0,0 @@
-# Komodo stack definition. Deploys electric-cherry as a managed container on
-# the same host running the T5/T6 Plutonium servers. Mounts /root/zombies so the
-# app can push scripts to the same paths push_scripts.sh used, and dedicates
-# a small subdir for its SQLite state.
-
-[[stack]]
-name = "electric-cherry"
-
-[stack.services.web]
-image = "electric-cherry:latest"
-restart = "unless-stopped"
-ports = ["3000:3000"]
-
-volumes = [
-  "/root/zombies:/root/zombies:rw",
-  "/root/zombies/.electric-cherry:/data:rw"
-]
-
-# Lets RCON_HOST=host.docker.internal reach game servers running on the
-# same Komodo host without knowing each container's internal IP.
-extra_hosts = ["host.docker.internal:host-gateway"]
-
-[stack.services.web.env]
-SERVERS_ROOT     = "/root/zombies/t5"
-T6_SERVERS_ROOT  = "/root/zombies/t6"
-PLUTAINER_T6_GAMEFILES_VOLUME = "t6_game_files"
-REL_DEST         = "runtime/plutonium/storage/t5/raw/scripts/sp"
-BETA_ARCHIVE_DIR = "/root/zombies/beta-archive"
-DB_PATH          = "/data/cherry.db"
-DEPLOY_KEEP_DAYS = "7"
-DEPLOYMENT_RESTART_CONCURRENCY = "1"
-
-# secrets: bind via Komodo's env-file or secret store
-KOMODO_ADDRESS   = "https://komodo.example.internal"
-KOMODO_API_KEY   = ""
-KOMODO_API_SECRET = ""
-# Host new deployments land on — must match the name shown in Komodo's UI.
-KOMODO_TARGET_SERVER = ""
-
-# RCON — shared password across every Plutonium server in the fleet.
-RCON_PASSWORD    = ""
-RCON_HOST        = "host.docker.internal"
-RCON_TIMEOUT_MS  = "3000"

package.json

@@ -4,14 +4,30 @@
     "description": "T5/T6 Plutonium Zombies fleet manager — replaces push_scripts.sh with a live web UI",
     "private": true,
     "type": "module",
+    "engines": {
+        "node": "^22.13.0 || >=24.0.0"
+    },
     "scripts": {
         "dev": "vite dev",
-    "build": "vite build",
+        "build": "vite build && npm run build:reports -- --outDir build/reports && npm run build:launch -- --outDir build/launch",
         "preview": "vite preview",
-    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+        "check": "npm run check:styles && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
         "test": "vitest run",
         "test:watch": "vitest",
-    "seed:fixture": "tsx scripts/seed-fixture.ts"
+        "benchmark:leaderboard": "node scripts/benchmark-leaderboard.mjs",
+        "diagnostics:database": "node scripts/database-diagnostics.mjs",
+        "seed:fixture": "node --import tsx scripts/seed-fixture.ts",
+        "agent:package": "python3 agent/package_release.py",
+        "prebuild": "npm run agent:package",
+        "test:agent": "python3 -m unittest discover -s agent/tests -v",
+        "build:reports": "vite build --config vite.reports.config.ts",
+        "build:launch": "vite build --config vite.launch.config.ts",
+        "predev": "npm run agent:package && npm run build:reports && npm run build:launch",
+        "pretest": "npm run build:reports && npm run build:launch",
+        "pretest:watch": "npm run build:reports && npm run build:launch",
+        "check:styles": "node scripts/check-ui-styles.mjs",
+        "lint": "prettier --check . && eslint .",
+        "format": "prettier --write ."
     },
     "dependencies": {
         "@fontsource/alfa-slab-one": "^5.3.0",
@@ -20,16 +36,31 @@
         "better-sqlite3": "^11.3.0"
     },
     "devDependencies": {
+        "@eslint/js": "^10.0.1",
         "@sveltejs/adapter-node": "^5.2.9",
-    "@sveltejs/kit": "^2.8.0",
+        "@sveltejs/kit": "^2.70.3",
-    "@sveltejs/vite-plugin-svelte": "^4.0.0",
+        "@sveltejs/vite-plugin-svelte": "^6.2.4",
         "@types/better-sqlite3": "^7.6.11",
         "@types/node": "^22.7.0",
+        "eslint": "^10.12.0",
+        "eslint-config-prettier": "^10.1.8",
+        "eslint-plugin-svelte": "^3.23.1",
+        "globals": "^17.13.0",
+        "happy-dom": "^21.0.1",
+        "prettier": "^3.9.9",
+        "prettier-plugin-svelte": "^3.5.2",
         "svelte": "^5.1.0",
         "svelte-check": "^4.0.0",
         "tsx": "^4.19.0",
         "typescript": "^5.6.0",
-    "vite": "^5.4.0",
+        "typescript-eslint": "^8.71.1",
-    "vitest": "^2.1.0"
+        "vite": "^7.3.7",
+        "vitest": "^4.1.11"
+    },
+    "overrides": {
+        "@sveltejs/kit": {
+            "cookie": "^0.7.2"
+        },
+        "devalue": "^5.9.4"
     }
 }

scripts/benchmark-leaderboard.mjs

@@ -0,0 +1,411 @@
+#!/usr/bin/env node
+/**
+ * Benchmark actual leaderboard modules against deterministic, isolated SQLite histories.
+ * node scripts/benchmark-leaderboard.mjs --scenario all --baseline-ref <commit> --runs 3
+ * Uses a TypeScript module loader to replace only db(); it never opens the application DB.
+ */
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { arch, cpus, platform, tmpdir } from 'node:os';
+import { dirname, join, relative, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { fileURLToPath } from 'node:url';
+
+const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const require = createRequire(join(repo, 'package.json'));
+const Database = require('better-sqlite3');
+const ts = require('typescript');
+const arguments_ = process.argv.slice(2);
+const options = { scenario: 'small', runs: 3 };
+const allowed = new Set(['--scenario', '--runs', '--baseline-ref', '--json']);
+for (let index = 0; index < arguments_.length; index += 2) {
+    const key = arguments_[index];
+    const value = arguments_[index + 1];
+    if (!allowed.has(key) || !value)
+        throw new Error(
+            'Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH'
+        );
+    options[key.slice(2)] = value;
+}
+options.runs = Number(options.runs);
+if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20)
+    throw new Error('--runs must be an integer from 1 to 20');
+if (!['small', 'large', 'all'].includes(options.scenario))
+    throw new Error('--scenario must be small, large, or all');
+const baseline = options['baseline-ref']
+    ? execFileSync(
+          'git',
+          ['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`],
+          { cwd: repo, encoding: 'utf8' }
+      ).trim()
+    : null;
+const definitions = [
+    { name: 'small', matches: 2000, players: 500 },
+    { name: 'large', matches: 50000, players: 10000 }
+].filter((item) => options.scenario === 'all' || item.name === options.scenario);
+const temporary = mkdtempSync(join(tmpdir(), 'cherry-leaderboard-benchmark-'));
+const source = (file, ref) =>
+    ref
+        ? execFileSync('git', ['show', `${ref}:${file}`], {
+              cwd: repo,
+              encoding: 'utf8',
+              maxBuffer: 10 * 1024 * 1024
+          })
+        : readFileSync(join(repo, file), 'utf8');
+const percentile = (values, fraction) =>
+    [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1];
+
+function seed(handle, definition) {
+    handle.exec(source('src/lib/server/db/schema.sql'));
+    handle.exec(
+        "CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)"
+    );
+    const match = handle.prepare(`INSERT INTO leaderboard_matches
+        (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at,
+            status, start_observed, start_players, highest_round, last_event, last_fingerprint)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 4, ?, ?, ?)`);
+    const identity =
+        handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
+        VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
+            name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
+            last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
+    const participation = handle.prepare(`INSERT INTO leaderboard_match_players
+        (match_id, guid, highest_round, rounds_survived, kills, headshots, downs, revives, best_score,
+            last_round, present, has_stats, completed, first_seen_at, last_seen_at,
+            first_observed_round, present_at_start)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    let state = 20260918;
+    const random = () => {
+        state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
+        return state / 4294967296;
+    };
+    handle.transaction(() => {
+        for (let id = 1; id <= definition.matches; id++) {
+            const mapIndex = Math.floor(random() * 12);
+            const game = mapIndex < 6 ? 't5' : 't6';
+            const map = `map_${mapIndex}`;
+            const status = id % 20 === 0 ? 'active' : id % 20 === 1 ? 'interrupted' : 'completed';
+            const started = new Date(
+                Date.UTC(2023, 0, 1) + Math.floor((id / definition.matches) * 3 * 365 * 86400000)
+            ).toISOString();
+            const ended = new Date(
+                Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000
+            ).toISOString();
+            const round = id % 101 === 0 ? 1 : 2 + Math.floor(random() * 60);
+            match.run(
+                id,
+                `server_${id % 24}`,
+                game,
+                map,
+                `${game}:${map}`,
+                `Map ${mapIndex}`,
+                started,
+                status === 'completed' ? ended : null,
+                ended,
+                status,
+                round,
+                status === 'completed' ? 'game_end' : 'heartbeat',
+                `event_${id}`
+            );
+            const roster = new Set();
+            while (roster.size < 4) roster.add(Math.floor(random() ** 2 * definition.players));
+            // Reserve one completed participation for an explicitly ineligible player.
+            if (id === definition.matches - 2) roster.delete([...roster][3]);
+            for (const player of roster) {
+                const guid = String(10000000000000000n + BigInt(player));
+                const name = `${game === 't6' ? 'Alias' : 'Player'} ${String(player).padStart(5, '0')}`;
+                identity.run(game, guid, name, started, ended);
+                const kills = Math.floor(round * 18 + random() * 800);
+                const presentAtStart = random() > 0.1;
+                const firstRound = presentAtStart ? 0 : Math.max(1, Math.floor(random() * round));
+                participation.run(
+                    id,
+                    guid,
+                    round,
+                    Math.max(0, round - Math.max(1, firstRound)),
+                    kills,
+                    Math.floor(kills * random()),
+                    Math.floor(random() * 8),
+                    Math.floor(random() * 12),
+                    Math.floor(random() * 100000),
+                    Math.max(0, round - 1),
+                    Number(status === 'active'),
+                    Number(random() > 0.01),
+                    Number(status === 'completed' && random() > 0.15),
+                    started,
+                    ended,
+                    firstRound,
+                    Number(presentAtStart)
+                );
+            }
+            if (id === definition.matches - 2) {
+                identity.run(game, 'benchmark-low-sample', 'Low sample', started, ended);
+                participation.run(
+                    id,
+                    'benchmark-low-sample',
+                    2,
+                    1,
+                    20,
+                    5,
+                    0,
+                    0,
+                    1000,
+                    1,
+                    0,
+                    1,
+                    1,
+                    started,
+                    ended,
+                    0,
+                    1
+                );
+            }
+        }
+    })();
+    handle.pragma('wal_checkpoint(TRUNCATE)');
+}
+
+function loadImplementation(handle, ref) {
+    const modules = new Map();
+    const hashes = {};
+    let recorded = null;
+    const database = {
+        prepare(sql) {
+            const statement = handle.prepare(sql);
+            return Object.fromEntries(
+                ['all', 'get'].map((method) => [
+                    method,
+                    (...parameters) => {
+                        const start = performance.now();
+                        const result = statement[method](...parameters);
+                        if (recorded)
+                            recorded.push({
+                                sql,
+                                parameters,
+                                milliseconds: performance.now() - start
+                            });
+                        return result;
+                    }
+                ])
+            );
+        }
+    };
+    function load(file) {
+        if (file === 'src/lib/server/db/index.ts') return { db: () => database };
+        if (modules.has(file)) return modules.get(file);
+        const text = source(file, ref);
+        hashes[file] = createHash('sha256').update(text).digest('hex');
+        const compiled = ts.transpileModule(text, {
+            compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }
+        }).outputText;
+        const exports = {};
+        modules.set(file, exports);
+        const localRequire = (specifier) => {
+            if (!specifier.startsWith('.')) return require(specifier);
+            const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(
+                /\.js$/,
+                '.ts'
+            );
+            if (!target.startsWith('src/lib/'))
+                throw new Error(`Unexpected benchmark dependency: ${target}`);
+            return load(target);
+        };
+        new Function('require', 'exports', compiled)(localRequire, exports);
+        return exports;
+    }
+    const api = load('src/lib/server/fleet/leaderboard.ts');
+    return {
+        name: ref ? 'baseline' : 'current',
+        ref,
+        hashes,
+        metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map((metric) => metric.key),
+        run(test) {
+            return test.guid
+                ? api.getPlayerLeaderboardEntry(test.guid, test.options)
+                : api.getLeaderboard(test.options);
+        },
+        measure(test) {
+            recorded = [];
+            const start = performance.now();
+            const result = this.run(test);
+            const milliseconds = performance.now() - start;
+            const statements = recorded;
+            recorded = null;
+            return { result, milliseconds, statements };
+        }
+    };
+}
+
+const report = {
+    environment: {
+        node: process.version,
+        sqlite: null,
+        platform: platform(),
+        arch: arch(),
+        cpu: cpus()[0]?.model
+    },
+    baselineRef: baseline,
+    runs: options.runs,
+    seed: 20260918,
+    notes: 'Synthetic file-backed SQLite, one warmup per case, rotating execution order, warm OS cache. Includes SQL preparation and source-record lookup; excludes HTTP and browser rendering. Shared local CPU; timings are not production latency. Schema and indexes are current for both implementations.',
+    scenarios: []
+};
+try {
+    for (const definition of definitions) {
+        const handle = new Database(join(temporary, `${definition.name}.sqlite`));
+        try {
+            report.environment.sqlite = handle
+                .prepare('SELECT sqlite_version() AS version')
+                .get().version;
+            seed(handle, definition);
+            const implementations = [
+                ...(baseline ? [loadImplementation(handle, baseline)] : []),
+                loadImplementation(handle)
+            ];
+            const cases = [
+                { name: 'unfiltered', options: {} },
+                { name: 'kills', options: { metric: 'kills' } },
+                { name: 'game', options: { game: 't5', metric: 'kills' } },
+                { name: 'map', options: { game: 't6', map: 't6:map_8', metric: 'kills' } },
+                { name: 'search', options: { q: '00042', metric: 'kills' } },
+                {
+                    name: 'deep-page',
+                    options: { metric: 'kills', offset: Math.floor(definition.players / 2) }
+                },
+                { name: 'efficiency', options: { metric: 'headshotRate' } },
+                { name: 'exact-player', guid: '10000000000000042', options: { metric: 'kills' } },
+                { name: 'empty-search', options: { q: 'No such player exists' } },
+                { name: 'past-end', options: { offset: 999999 } },
+                { name: 'empty-map', options: { map: 't5:missing' } }
+            ];
+            const counts = handle
+                .prepare(
+                    `SELECT COUNT(*) AS participationRows,
+                COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`
+                )
+                .get();
+            const scenario = {
+                ...definition,
+                ...counts,
+                maps: 12,
+                servers: 24,
+                completedShare: 0.9,
+                results: []
+            };
+            report.scenarios.push(scenario);
+            // Broad parity is separate from timings so every ratio and edge case is
+            // checked without multiplying the larger fixture's timed request matrix.
+            if (baseline && definition.name === 'small') {
+                const scopes = [
+                    {},
+                    { game: 't5' },
+                    { game: 't6', map: 't6:map_8' },
+                    { q: '00042' },
+                    { q: 'No such player exists' },
+                    { offset: 999999 },
+                    { map: 't5:missing' }
+                ];
+                let checked = 0;
+                for (const metric of implementations[1].metrics) {
+                    for (const scope of scopes) {
+                        const test = { options: { ...scope, metric } };
+                        assert.deepStrictEqual(
+                            implementations[1].run(test),
+                            implementations[0].run(test),
+                            `${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`
+                        );
+                        checked++;
+                    }
+                    for (const guid of [
+                        '10000000000000042',
+                        'benchmark-low-sample',
+                        'benchmark-missing'
+                    ]) {
+                        const test = { guid, options: { metric } };
+                        assert.deepStrictEqual(
+                            implementations[1].run(test),
+                            implementations[0].run(test),
+                            `${definition.name}/${metric}/${guid}: exact-player parity`
+                        );
+                        checked++;
+                    }
+                }
+                scenario.parityCases = checked;
+                console.log(
+                    JSON.stringify({
+                        scenario: definition.name,
+                        parityCases: checked,
+                        parity: true
+                    })
+                );
+            }
+            for (const test of cases) {
+                const expected = implementations[0].run(test);
+                for (const implementation of implementations.slice(1)) {
+                    assert.deepStrictEqual(
+                        implementation.run(test),
+                        expected,
+                        `${definition.name}/${test.name}: baseline parity`
+                    );
+                }
+                const samples = Object.fromEntries(implementations.map((item) => [item.name, []]));
+                const latestStatements = {};
+                for (let index = 0; index < options.runs; index++) {
+                    const rotation = index % implementations.length;
+                    for (const implementation of [
+                        ...implementations.slice(rotation),
+                        ...implementations.slice(0, rotation)
+                    ]) {
+                        const measured = implementation.measure(test);
+                        samples[implementation.name].push(measured.milliseconds);
+                        latestStatements[implementation.name] = measured.statements;
+                    }
+                }
+                const result = {
+                    ...test,
+                    parity: baseline ? true : null,
+                    implementations: implementations.map((implementation) => ({
+                        name: implementation.name,
+                        sourceHashes: implementation.hashes,
+                        medianMs: percentile(samples[implementation.name], 0.5),
+                        p95Ms: percentile(samples[implementation.name], 0.95),
+                        samplesMs: samples[implementation.name],
+                        statements: latestStatements[implementation.name].map((statement) => ({
+                            ...statement,
+                            plan: handle
+                                .prepare(`EXPLAIN QUERY PLAN ${statement.sql}`)
+                                .all(...statement.parameters)
+                        }))
+                    }))
+                };
+                scenario.results.push(result);
+                console.log(
+                    JSON.stringify({
+                        scenario: definition.name,
+                        case: test.name,
+                        parity: result.parity,
+                        medianMs: Object.fromEntries(
+                            result.implementations.map((item) => [
+                                item.name,
+                                Math.round(item.medianMs)
+                            ])
+                        )
+                    })
+                );
+            }
+        } finally {
+            handle.close();
+        }
+    }
+    if (options.json) {
+        const target = resolve(options.json);
+        mkdirSync(dirname(target), { recursive: true });
+        writeFileSync(target, JSON.stringify(report, null, 2));
+        console.log(`Detailed results: ${target}`);
+    }
+} finally {
+    rmSync(temporary, { recursive: true, force: true });
+}

scripts/check-ui-styles.mjs

@@ -0,0 +1,204 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+import { parse } from 'svelte/compiler';
+
+const sourceRoot = fileURLToPath(new URL('../src/', import.meta.url));
+const tokens = await readFile(path.join(sourceRoot, 'lib/ui/styles/tokens.css'), 'utf8');
+const sharedBreakpointWidths = new Set(
+    [...tokens.matchAll(/--breakpoint-[\w-]+:\s*([\d.]+px)\s*;/g)].map((match) => match[1])
+);
+
+// Existing layouts awaiting migration (Refs #130). Remove widths as each file
+// moves to the shared scale; a listed file cannot introduce other legacy widths.
+/** @type {Record<string, number[]>} */
+const legacyBreakpointWidths = {
+    'src/lib/ui/components/ArtifactDetail.svelte': [720, 1000],
+    'src/lib/ui/components/BetaPlanBuilder.svelte': [480, 850],
+    'src/lib/ui/components/BetaTestProgress.svelte': [720],
+    'src/lib/ui/components/BlueprintBuilder.svelte': [620],
+    'src/lib/ui/components/ConfigEditor.svelte': [600],
+    'src/lib/ui/components/ConfigInspector.svelte': [600],
+    'src/lib/ui/components/ConfigLibrary.svelte': [600],
+    'src/lib/ui/components/ConfigTextEditor.svelte': [600],
+    'src/lib/ui/components/DeployCollectionDialog.svelte': [440, 600],
+    'src/lib/ui/components/DeployServerDialog.svelte': [440],
+    'src/lib/ui/components/ProvisioningActivity.svelte': [600],
+    'src/lib/ui/components/ProvisioningProgress.svelte': [600],
+    'src/lib/ui/components/SettingsEditor.svelte': [720],
+    'src/lib/ui/components/capacity/RebalancePanel.svelte': [600],
+    'src/lib/ui/components/capacity/RebalanceRunView.svelte': [600],
+    'src/lib/ui/components/metrics/MetricsExplore.svelte': [540],
+    'src/lib/ui/components/metrics/MetricsSummary.svelte': [760],
+    'src/lib/ui/components/primitives/Button.svelte': [720],
+    'src/lib/ui/components/primitives/Card.svelte': [480],
+    'src/lib/ui/components/primitives/Dialog.svelte': [600],
+    'src/lib/ui/styles/components.css': [600],
+    'src/routes/account/+page.svelte': [540],
+    'src/routes/autoscaling/+page.svelte': [500],
+    'src/routes/betas/+page.svelte': [720],
+    'src/routes/betas/[id]/+page.svelte': [720],
+    'src/routes/betas/new/+page.svelte': [700],
+    'src/routes/campaigns/+page.svelte': [720],
+    'src/routes/campaigns/[id]/+page.svelte': [720],
+    'src/routes/capacity/+page.svelte': [600],
+    'src/routes/collections/launches/[id]/+page.svelte': [600],
+    'src/routes/hosts/operations/+page.svelte': [600],
+    'src/routes/leaderboard/+page.svelte': [800, 1000],
+    'src/routes/login/+page.svelte': [540],
+    'src/routes/matches/[id]/+page.svelte': [480, 800],
+    'src/routes/metrics/+page.svelte': [540],
+    'src/routes/plugins/+page.svelte': [720],
+    'src/routes/provisioning/+page.svelte': [720],
+    'src/routes/provisioning/[id]/+page.svelte': [600, 1000],
+    'src/routes/setup/+page.svelte': [540]
+};
+
+/**
+ * @param {string} source
+ * @param {string} filename
+ */
+export function checkMediaWidths(source, filename) {
+    // Parse styles so comments, script strings and CSS declaration values cannot
+    // be mistaken for media queries. Wrap standalone CSS without adding a line.
+    const cssSource = filename.endsWith('.css') ? `<style>${source}</style>` : source;
+    const stylesheet = parse(cssSource, { modern: true }).css;
+    const legacyWidths = legacyBreakpointWidths[filename.replaceAll('\\', '/')] ?? [];
+    const allowed = new Set([...sharedBreakpointWidths, ...legacyWidths.map((n) => `${n}px`)]);
+    /** @type {string[]} */
+    const failures = [];
+    /** @param {import('svelte/compiler').AST.CSS.StyleSheet | import('svelte/compiler').AST.CSS.Block | null} block */
+    function visit(block) {
+        for (const node of block?.children ?? []) {
+            if (node.type === 'Atrule' && node.name.toLowerCase() === 'media') {
+                const query = node.prelude.replace(/\/\*[\s\S]*?\*\//g, ' ');
+                // Check both sides independently to cover chained range syntax.
+                const widths = [
+                    ...query.matchAll(
+                        /(?<![\w-])(?:min-|max-)?width\s*(?::|[<>]=?|=)\s*([^\s)]+)/gi
+                    ),
+                    ...query.matchAll(/([\d.]+(?:[a-z%]+)?)\s*(?:[<>]=?|=)\s*width\b/gi)
+                ];
+                for (const match of widths) {
+                    if (allowed.has(match[1].toLowerCase())) continue;
+                    const line = cssSource.slice(0, node.start).split('\n').length;
+                    failures.push(
+                        `${filename}:${line}: use a shared media width (${[...sharedBreakpointWidths].join(', ')}): ${match[1]}`
+                    );
+                }
+            }
+            if ('block' in node) visit(node.block);
+        }
+    }
+    visit(stylesheet);
+    return failures;
+}
+
+const nativeInputTypes = new Set([
+    'checkbox',
+    'radio',
+    'file',
+    'hidden',
+    'range',
+    'color',
+    'button',
+    'submit',
+    'reset',
+    'image'
+]);
+
+/**
+ * @param {string} source
+ * @param {string} filename
+ */
+export function checkFormControls(source, filename) {
+    if (
+        !filename.endsWith('.svelte') ||
+        /(?:^|\/)primitives\/(?:Input|Select)\.svelte$/.test(filename)
+    )
+        return [];
+    /** @type {string[]} */
+    const failures = [];
+    /** @param {unknown} node */
+    function visit(node) {
+        if (!node || typeof node !== 'object') return;
+        if ('type' in node && node.type === 'RegularElement') {
+            const element = /** @type {import('svelte/compiler').AST.RegularElement} */ (node);
+            const type = element.attributes.find(
+                (attribute) => attribute.type === 'Attribute' && attribute.name === 'type'
+            );
+            const value =
+                type?.type === 'Attribute' && type.value !== true
+                    ? Array.isArray(type.value)
+                        ? type.value[0]
+                        : type.value
+                    : undefined;
+            const inputType =
+                value?.type === 'Text'
+                    ? value.data
+                    : value?.expression.type === 'Literal'
+                      ? value.expression.value
+                      : undefined;
+            if (
+                element.name === 'select' ||
+                (element.name === 'input' &&
+                    (typeof inputType !== 'string' || !nativeInputTypes.has(inputType)))
+            ) {
+                const line = source.slice(0, element.start).split('\n').length;
+                failures.push(
+                    `${filename}:${line}: use Field with ${element.name === 'select' ? 'Select' : 'Input'} instead of raw <${element.name}>`
+                );
+            }
+        }
+        for (const value of Object.values(node)) {
+            if (Array.isArray(value)) value.forEach(visit);
+            else if (value && typeof value === 'object') visit(value);
+        }
+    }
+    visit(parse(source, { modern: true }).fragment);
+    return failures;
+}
+
+async function main() {
+    /** @type {string[]} */
+    const failures = [];
+    /** @param {string} directory */
+    async function visit(directory) {
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const file = path.join(directory, entry.name);
+            if (entry.isDirectory()) await visit(file);
+            else if (/\.(svelte|css)$/.test(entry.name)) {
+                const source = await readFile(file, 'utf8');
+                const filename = path
+                    .relative(path.dirname(sourceRoot), file)
+                    .split(path.sep)
+                    .join('/');
+                failures.push(...checkMediaWidths(source, filename));
+                if (file === path.join(sourceRoot, 'lib/ui/styles/tokens.css')) continue;
+                failures.push(...checkFormControls(source, filename));
+                for (const match of source.matchAll(/font-size\s*:\s*[\d.]+px\b/g)) {
+                    const line = source.slice(0, match.index).split('\n').length;
+                    failures.push(
+                        `${filename}:${line}: use a shared type token (12 px minimum): ${match[0]}`
+                    );
+                }
+            }
+        }
+    }
+    await visit(sourceRoot);
+    for (const match of tokens.matchAll(
+        /--text-(?:xs|sm|base|md|lg|xl|2xl|3xl|input):\s*([\d.]+)px/g
+    )) {
+        if (Number(match[1]) < 12) failures.push(`Type tokens must be at least 12 px: ${match[0]}`);
+    }
+    if (failures.length) {
+        console.error(failures.join('\n'));
+        process.exitCode = 1;
+    } else
+        console.log(
+            'UI styles: shared form controls, media widths, font sizes and 12 px minimum verified.'
+        );
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url))
+    await main();

scripts/database-diagnostics.mjs

@@ -0,0 +1,33 @@
+import Database from 'better-sqlite3';
+import { statSync } from 'node:fs';
+
+const path = process.argv[2] ?? process.env.DB_PATH ?? './data/cherry.db';
+const db = new Database(path, { readonly: true, fileMustExist: true });
+try {
+    console.table(
+        ['', '-wal', '-shm'].map((suffix) => {
+            try {
+                return { file: path + suffix, bytes: statSync(path + suffix).size };
+            } catch (error) {
+                if (error.code !== 'ENOENT') throw error;
+                return { file: path + suffix, bytes: 0 };
+            }
+        })
+    );
+    console.table(
+        db
+            .prepare(
+                `SELECT name, ROUND(SUM(pgsize)/1048576.0, 2) AS mib
+        FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC LIMIT 20`
+            )
+            .all()
+    );
+    console.table(
+        ['page_size', 'page_count', 'freelist_count', 'auto_vacuum'].map((pragma) => ({
+            pragma,
+            value: db.pragma(pragma, { simple: true })
+        }))
+    );
+} finally {
+    db.close();
+}

scripts/fixture-data.ts

@@ -0,0 +1,625 @@
+import { createHash } from 'node:crypto';
+import type Database from 'better-sqlite3';
+import { zombieMapBySlug } from '../src/lib/map-aliases.js';
+
+export const FIXTURE_KEY = 'development_fixture_v1';
+export const FIXTURE_VERSION = 'electric-cherry-ui-v1';
+const MINUTE = 60_000;
+const DAY = 24 * 60 * MINUTE;
+const GIB = 1024 ** 3;
+const image = 'fixture.invalid/electric-cherry/preview:local';
+
+type Scenario = {
+    game: 't5' | 't6';
+    slug: string;
+    players: number;
+    round: number;
+    minutes: number;
+    state: 'active' | 'idle' | 'stopped' | 'stale' | 'starting' | 'host-offline';
+};
+const scenarios: Scenario[] = [
+    { game: 't5', slug: 'theater', players: 4, round: 27, minutes: 76, state: 'active' },
+    { game: 't5', slug: 'cosmodrome', players: 2, round: 16, minutes: 39, state: 'active' },
+    { game: 't5', slug: 'moon', players: 0, round: 0, minutes: 0, state: 'idle' },
+    { game: 't5', slug: 'temple', players: 3, round: 32, minutes: 84, state: 'active' },
+    { game: 't5', slug: 'pentagon', players: 0, round: 0, minutes: 0, state: 'stopped' },
+    { game: 't5', slug: 'coast', players: 0, round: 0, minutes: 0, state: 'stale' },
+    { game: 't6', slug: 'tomb', players: 3, round: 22, minutes: 58, state: 'active' },
+    { game: 't6', slug: 'town', players: 2, round: 12, minutes: 23, state: 'active' },
+    { game: 't6', slug: 'buried', players: 0, round: 0, minutes: 0, state: 'idle' },
+    { game: 't6', slug: 'prison', players: 4, round: 41, minutes: 119, state: 'active' },
+    { game: 't6', slug: 'nuked', players: 0, round: 0, minutes: 0, state: 'starting' },
+    { game: 't6', slug: 'highrise', players: 0, round: 0, minutes: 0, state: 'host-offline' }
+];
+
+export const fixtureServers = scenarios.map((scenario, index) => ({
+    ...scenario,
+    name: `${scenario.game}_zombie_${scenario.slug}_1`,
+    host: `fixture-host-${index === 11 ? 3 : (index % 2) + 1}`,
+    instance: `fixture-instance-${index + 1}`,
+    port: 28960 + index,
+    map: zombieMapBySlug(scenario.game, scenario.slug)!
+}));
+const names = [
+    'Jamie Cross',
+    'Alex',
+    'Sam',
+    'Morgan',
+    'Riley',
+    'Taylor',
+    'Jordan',
+    'Casey',
+    'Quinn',
+    'Avery',
+    'Drew',
+    'Kai'
+];
+const iso = (at: number) => new Date(at).toISOString();
+const player = (index: number) => ({
+    guid: index === 0 ? 'fixture-jamie' : `fixture-player-${index}`,
+    name:
+        index === 0
+            ? names[0]
+            : `${names[index % names.length]} ${Math.floor(index / names.length) + 1}`
+});
+const rosterFor = (server: (typeof fixtureServers)[number], index: number) =>
+    Array.from({ length: server.players }, (_, slot) => ({
+        ...player(slot === 0 ? 0 : index * 4 + slot),
+        ping: 29 + slot * 7,
+        kills: (slot + 1) * 137,
+        headshots: (slot + 1) * 41,
+        downs: slot % 2,
+        revives: slot,
+        score: (slot + 1) * 4100
+    }));
+const seenAt = (server: Scenario, now: number) =>
+    iso(
+        now -
+            (['stopped', 'host-offline'].includes(server.state)
+                ? 15 * MINUTE
+                : server.state === 'stale'
+                  ? 105_000
+                  : 0)
+    );
+const containerState = (server: Scenario) =>
+    server.state === 'stopped' ? 'exited' : server.state === 'starting' ? 'starting' : 'running';
+const instanceStates = (host: string, now: number) =>
+    fixtureServers
+        .filter((server) => server.host === host)
+        .map((server) => ({
+            instanceId: server.instance,
+            generation: 1,
+            state: containerState(server),
+            healthy: !['stopped', 'starting'].includes(server.state),
+            observedAt: iso(now - (host === 'fixture-host-3' ? 10 * MINUTE : 0))
+        }));
+
+/** Uses only a caller-owned, fully migrated database. No host or worker code is imported. */
+export function seedFixtureData(handle: Database.Database, now: number): void {
+    if (!Number.isFinite(now)) throw new Error('Invalid fixture time.');
+    const occupied = [
+        'servers',
+        'leaderboard_matches',
+        'ec_hosts',
+        'auth_users',
+        'scripts',
+        'server_deployments',
+        'deployments',
+        'betas',
+        'game_file_libraries'
+    ].some((table) => handle.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get());
+    if (occupied || handle.prepare('SELECT 1 FROM settings WHERE key=?').get(FIXTURE_KEY))
+        throw new Error('Fixture seeding requires a new database.');
+    const insert = (table: string, values: Record<string, unknown>) => {
+        const keys = Object.keys(values);
+        handle
+            .prepare(
+                `INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})`
+            )
+            .run(...Object.values(values));
+    };
+    const started = iso(now - 30 * DAY);
+    const rememberPlayer =
+        handle.prepare(`INSERT INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?)
+        ON CONFLICT(game,guid) DO UPDATE SET first_seen_at=MIN(first_seen_at,excluded.first_seen_at),last_seen_at=MAX(last_seen_at,excluded.last_seen_at)`);
+    handle.transaction(() => {
+        insert('settings', { key: FIXTURE_KEY, value: FIXTURE_VERSION });
+        // Preview observations must never opt into automatic lifecycle changes.
+        for (const [key, value] of Object.entries({
+            maintenance_paused: '1',
+            maintenance_health: '0',
+            maintenance_drift: '0',
+            maintenance_updates: '0',
+            watchdog_enabled: '0',
+            autoscaling_paused: '1'
+        })) {
+            handle
+                .prepare(
+                    'INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value'
+                )
+                .run(key, value);
+        }
+        for (const [index, name] of [
+            'Chicago · Primary',
+            'Dallas · Overflow',
+            'Frankfurt · Europe'
+        ].entries()) {
+            const id = `fixture-host-${index + 1}`;
+            insert('ec_hosts', {
+                id,
+                name,
+                hostname: `host-${index + 1}.fixture.invalid`,
+                created_at: started,
+                enrolled_at: started,
+                last_heartbeat: iso(now - (index === 2 ? 10 * MINUTE : 0)),
+                inventory: JSON.stringify({
+                    protocolVersion: 1,
+                    agentVersion: '1.7.0',
+                    cpuCount: 16,
+                    cpuUsagePercent: 24 + index * 9,
+                    memoryTotal: 32 * GIB,
+                    memoryAvailable: (20 - index * 3) * GIB,
+                    storageTotal: 500 * GIB,
+                    storageAvailable: 240 * GIB,
+                    dockerRunning: true,
+                    directoriesWritable: true,
+                    preparedRuntimeImages: [image],
+                    libraries: [
+                        { id: 'fixture-t5', state: 'ready' },
+                        { id: 'fixture-t6', state: 'ready' }
+                    ],
+                    capabilities: ['instance.heartbeat-state'],
+                    instanceStates: instanceStates(id, now)
+                })
+            });
+            // Deliberately not a token hash; no bearer token can authenticate as these agents.
+            insert('ec_host_credentials', {
+                id: `fixture-credential-${index + 1}`,
+                host_id: id,
+                token_hash: `invalid-fixture-${index + 1}`,
+                created_at: started
+            });
+            for (let minute = 0; minute < 7 * 1440; minute += 15) {
+                insert('ec_host_resource_samples', {
+                    host_id: id,
+                    sampled_at: iso(now - minute * MINUTE),
+                    cpu_count: 16,
+                    cpu_usage_percent: 28 + index * 6 + 10 * Math.sin(minute / 95),
+                    memory_total: 32 * GIB,
+                    memory_available: Math.round((18 - index * 2 + Math.sin(minute / 110)) * GIB)
+                });
+            }
+        }
+        let liveId = 495;
+        for (const [index, server] of fixtureServers.entries()) {
+            const beginning = iso(now - server.minutes * MINUTE - 12_000);
+            const roster = rosterFor(server, index);
+            insert('servers', {
+                name: server.name,
+                game: server.game,
+                map: server.map.canonical,
+                map_slug: server.slug,
+                game_mode: 'zombie',
+                game_port: server.port,
+                game_id: server.game === 't5' ? 't5sp' : 't6zm',
+                image,
+                created_by: 'development fixture',
+                created_at: started,
+                last_players: server.players,
+                last_map: server.map.canonical,
+                last_gametype: 'zclassic',
+                last_roster: JSON.stringify(roster),
+                last_seen_at: seenAt(server, now),
+                last_checked_at: iso(now),
+                last_error: server.state === 'stale' ? 'timeout: no reply from game server' : null,
+                rcon_failures: server.state === 'stale' ? 2 : 0,
+                container_state: containerState(server),
+                container_checked_at: iso(now),
+                desired_state: server.state === 'stopped' ? 'stopped' : 'running',
+                active_since: server.players ? beginning : null,
+                inactive_since: server.players ? null : iso(now - 5 * MINUTE),
+                game_started_at: server.players ? beginning : null
+            });
+            insert('native_instances', {
+                id: server.instance,
+                server: server.name,
+                host_id: server.host,
+                library_id: `fixture-${server.game}`,
+                image,
+                port: server.port,
+                created_at: started
+            });
+            insert('plutonium_versions', {
+                server: server.name,
+                revision: 4878,
+                checked_at: iso(now),
+                error: null
+            });
+            if (server.state === 'active' || server.state === 'idle') {
+                const matchId = server.players ? liveId++ : null;
+                const payload = {
+                    server: server.name,
+                    map: server.map.canonical,
+                    event: 'heartbeat',
+                    round: server.round,
+                    players: roster,
+                    receivedAt: iso(now),
+                    timing: {
+                        matchId,
+                        players: Object.fromEntries(
+                            roster.map((p) => [
+                                p.guid,
+                                {
+                                    serverJoinedAt: beginning,
+                                    matchJoinedAt: beginning,
+                                    roundStarted: 1
+                                }
+                            ])
+                        )
+                    }
+                };
+                insert('server_telemetry', {
+                    server: server.name,
+                    round: server.round,
+                    players: server.players,
+                    received_at: iso(now),
+                    payload: JSON.stringify(payload)
+                });
+                if (matchId !== null) {
+                    insert('leaderboard_matches', {
+                        id: matchId,
+                        server: server.name,
+                        game: server.game,
+                        map: server.map.canonical,
+                        map_key: server.slug,
+                        map_name: server.map.friendly,
+                        started_at: beginning,
+                        last_received_at: iso(now),
+                        status: 'active',
+                        start_observed: 1,
+                        start_players: server.players,
+                        highest_round: server.round,
+                        last_event: 'heartbeat',
+                        last_fingerprint: `fixture-live-${index}`
+                    });
+                    for (const p of roster) {
+                        rememberPlayer.run(server.game, p.guid, p.name, beginning, iso(now));
+                        insert('leaderboard_match_players', {
+                            match_id: matchId,
+                            guid: p.guid,
+                            highest_round: server.round,
+                            rounds_survived: server.round,
+                            kills: p.kills,
+                            headshots: p.headshots,
+                            downs: p.downs,
+                            revives: p.revives,
+                            best_score: p.score,
+                            last_round: server.round,
+                            present: 1,
+                            has_stats: 1,
+                            completed: 0,
+                            first_seen_at: beginning,
+                            last_seen_at: iso(now),
+                            first_observed_round: 1,
+                            present_at_start: 1
+                        });
+                    }
+                }
+            }
+            for (const phase of ['startup', 'running']) {
+                for (let minute = 0; minute < 60; minute++)
+                    insert('ec_instance_resource_samples', {
+                        host_id: server.host,
+                        instance_id: server.instance,
+                        generation: 1,
+                        phase,
+                        bucket: iso(now - minute * MINUTE),
+                        cpu_cores: phase === 'startup' ? 0.7 + index / 40 : 0.25 + index / 50,
+                        memory_bytes: Math.round((phase === 'startup' ? 1.1 : 0.7) * GIB)
+                    });
+            }
+        }
+        for (let index = 0; index < 494; index++) {
+            // A large Kino duo cohort exercises ranking and profile pagination.
+            const server = fixtureServers[index < 64 ? 0 : index % fixtureServers.length];
+            const squad = index < 64 ? 2 : (Math.floor(index / 12) % 4) + 1;
+            const unknownStart = index >= 64 && index % 37 === 0;
+            const interrupted = index >= 64 && index % 31 === 0;
+            const end = now - (180 + index * 37) * MINUTE;
+            const beginning = end - (30 + (index % 100)) * MINUTE;
+            const round = 10 + ((index * 7) % 56);
+            insert('leaderboard_matches', {
+                id: index + 1,
+                server: server.name,
+                game: server.game,
+                map: server.map.canonical,
+                map_key: server.slug,
+                map_name: server.map.friendly,
+                started_at: iso(beginning),
+                ended_at: iso(end),
+                last_received_at: iso(end),
+                status: interrupted ? 'interrupted' : 'completed',
+                start_observed: unknownStart ? 0 : 1,
+                start_players: unknownStart ? null : squad,
+                highest_round: round,
+                last_event: interrupted ? 'interrupted' : 'game_end',
+                last_fingerprint: `fixture-match-${index + 1}`
+            });
+            for (let slot = 0; slot < squad; slot++) {
+                const p = player(slot === 0 ? 0 : ((index + slot * 19) % 80) + 1);
+                rememberPlayer.run(server.game, p.guid, p.name, iso(beginning), iso(end));
+                insert('leaderboard_match_players', {
+                    match_id: index + 1,
+                    guid: p.guid,
+                    highest_round: round,
+                    rounds_survived: round - (unknownStart ? 4 : 0),
+                    kills: round * 35 + slot * 71,
+                    headshots: round * 9,
+                    downs: index % 4,
+                    revives: slot + (index % 3),
+                    best_score: round * 2100 + slot * 1000,
+                    last_round: round,
+                    present: slot > 0 && index % 9 === 0 ? 0 : 1,
+                    has_stats: 1,
+                    completed: !interrupted && !(slot > 0 && index % 9 === 0) ? 1 : 0,
+                    first_seen_at: iso(beginning),
+                    last_seen_at: iso(end),
+                    first_observed_round: unknownStart ? 5 : 1,
+                    present_at_start: unknownStart ? null : 1
+                });
+            }
+        }
+        const metric = handle.prepare(
+            'INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,?,?,?,?,NULL)'
+        );
+        for (let minute = 0; minute < 7 * 1440; minute++) {
+            for (const [index, server] of fixtureServers.entries()) {
+                const count = Math.floor((Math.sin(minute / 90 + index) + 1) * 2.49);
+                metric.run(
+                    now - minute * MINUTE,
+                    server.name,
+                    server.game,
+                    server.map.friendly,
+                    minute % 720 < 8 && index === 5 ? null : count
+                );
+            }
+        }
+        for (const scope of ['*', ...[1, 2, 3].map((index) => `fixture-host-${index}`)])
+            for (const phase of ['startup', 'running']) {
+                insert('ec_resource_allowances', {
+                    scope,
+                    phase,
+                    cpu_cores: phase === 'startup' ? 1 : 0.5,
+                    memory_bytes: Math.round((phase === 'startup' ? 1.2 : 0.8) * GIB),
+                    samples: 240,
+                    window_start: iso(now - DAY),
+                    observed_at: iso(now),
+                    computed_at: iso(now),
+                    usage_margin: 1
+                });
+            }
+        // Real artifact blobs, so details and downloads work without game files.
+        for (const [index, kind] of ['script', 'baseline', 'config'].entries()) {
+            const content = Buffer.from(
+                kind === 'script'
+                    ? '// Synthetic UI fixture; no gameplay behavior.\n'
+                    : '// Synthetic UI fixture configuration.\nset sv_maxclients 4\n'
+            );
+            const hash = createHash('sha256').update(content).digest('hex');
+            handle
+                .prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)')
+                .run(hash, content.length, content);
+            insert('scripts', {
+                id: index + 1,
+                name: [
+                    'Fixture telemetry',
+                    'Fixture Black Ops I baseline',
+                    'Fixture lobby settings'
+                ][index],
+                target_rel_path:
+                    index === 0 ? 'fixture.gsc' : index === 1 ? 'dedicated_sp.cfg' : 'fixture.cfg',
+                scope_type: 'game',
+                scope_value: 't5',
+                current_version_id: index + 1,
+                kind,
+                created_at: started
+            });
+            insert('script_versions', {
+                id: index + 1,
+                script_id: index + 1,
+                version: 1,
+                sha256: hash,
+                content_hash: hash,
+                size: content.length,
+                notes: 'Synthetic preview content.',
+                state: 'tested',
+                kind,
+                uploaded_at: started
+            });
+        }
+        for (let index = 0; index < 36; index++) {
+            const failed = index % 7 === 0;
+            const server = fixtureServers[failed ? 5 : index % fixtureServers.length];
+            const id = `fixture-operation-${index + 1}`;
+            const state = failed ? 'failed' : index % 11 === 0 ? 'cancelled' : 'done';
+            insert('server_deployments', {
+                id,
+                server: server.name,
+                action: ['restart', 'start', 'rebuild', 'stop'][index % 4],
+                state,
+                stage: failed
+                    ? 'Verification failed'
+                    : state === 'done'
+                      ? 'Completed'
+                      : 'Cancelled before dispatch',
+                started_at: iso(now - (60 + index * 2) * MINUTE),
+                finished_at: iso(now - (59 + index * 2) * MINUTE),
+                triggered_by: 'Fixture operator',
+                error: failed ? 'Synthetic verification timeout. No host command was sent.' : null,
+                warnings: '[]'
+            });
+            insert('native_provisioning', {
+                deployment_id: id,
+                instance_id: server.instance,
+                generation: 1
+            });
+            insert('server_deployment_events', {
+                deployment_id: id,
+                created_at: iso(now - (59 + index * 2) * MINUTE),
+                message: 'Synthetic retained history; no command was sent.'
+            });
+        }
+        for (let index = 0; index < 8; index++) {
+            const server = fixtureServers[index % fixtureServers.length];
+            insert('ec_host_operations', {
+                id: `fixture-host-command-${index + 1}`,
+                host_id: server.host,
+                instance_id: server.instance,
+                server_name: server.name,
+                kind: index % 2 ? 'instance.inspect' : 'instance.restart',
+                payload: '{}',
+                state: index % 3 ? 'completed' : 'failed',
+                result: '{}',
+                error: index % 3 ? null : 'Synthetic command failure.',
+                created_at: iso(now - (90 + index * 3) * MINUTE),
+                updated_at: iso(now - (89 + index * 3) * MINUTE),
+                source: 'Fixture operator'
+            });
+        }
+        for (let index = 0; index < 3; index++) {
+            const server = fixtureServers[index === 0 ? 1 : index === 1 ? 5 : index];
+            insert('maintenance_jobs', {
+                id: index + 1,
+                server: server.name,
+                state: index === 1 ? 'failed' : 'done',
+                target_revision: 4878,
+                requested_at: iso(now - (120 + index * 4) * MINUTE),
+                finished_at: iso(now - (119 + index * 4) * MINUTE),
+                detail:
+                    index === 1
+                        ? 'Synthetic update could not be verified.'
+                        : 'Synthetic scheduled restart completed.',
+                kind: 'schedule',
+                origin: 'manual',
+                provisioning_id: index === 0 ? 'fixture-operation-2' : null
+            });
+            insert('watchdog_incidents', {
+                id: `fixture-recovery-${index + 1}`,
+                server: server.name,
+                kind: 'health',
+                state: 'resolved',
+                summary: 'Synthetic recovery finding resolved.',
+                details: '{}',
+                first_seen_at: iso(now - (130 + index * 4) * MINUTE),
+                last_seen_at: iso(now - (129 + index * 4) * MINUTE),
+                finished_at: iso(now - (129 + index * 4) * MINUTE)
+            });
+            insert('deployments', {
+                id: `fixture-campaign-${index + 1}`,
+                action: 'drift',
+                started_at: iso(now - (140 + index * 4) * MINUTE),
+                finished_at: iso(now - (139 + index * 4) * MINUTE),
+                triggered_by: 'Fixture operator'
+            });
+            insert('deployment_servers', {
+                deployment_id: `fixture-campaign-${index + 1}`,
+                server: server.name,
+                files_changed: 1,
+                restart_state: 'done',
+                players: 0
+            });
+        }
+        for (let index = 0; index < 2; index++) {
+            insert('betas', {
+                id: index + 1,
+                script_id: 1,
+                script_version_id: 1,
+                trigger_kind: 'ui',
+                state: index ? 'rolled-back' : 'promoted',
+                opened_at: iso(now - (150 + index * 3) * MINUTE),
+                closed_at: iso(now - (149 + index * 3) * MINUTE),
+                notes: 'Synthetic completed beta.'
+            });
+            insert('beta_scripts', {
+                beta_id: index + 1,
+                script_id: 1,
+                script_version_id: 1,
+                kind: 'script'
+            });
+            insert('beta_servers', {
+                beta_id: index + 1,
+                server: fixtureServers[index].name,
+                had_original: 0,
+                stage: 'ready',
+                updated_at: iso(now - (149 + index * 3) * MINUTE)
+            });
+        }
+        insert('provisioning_sources', {
+            operation_id: 'fixture-operation-2',
+            kind: 'maintenance',
+            source_id: '1',
+            server: fixtureServers[1].name
+        });
+        insert('provisioning_sources', {
+            operation_id: 'fixture-operation-3',
+            kind: 'autoscaling',
+            source_id: 'fixture-paused-plan',
+            server: fixtureServers[2].name
+        });
+    })();
+}
+
+/** Refreshes existing synthetic observations only; never queues work or writes real files. */
+export function refreshFixtureData(handle: Database.Database, now: number): void {
+    const marker = handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY) as
+        { value: string } | undefined;
+    if (marker?.value !== FIXTURE_VERSION)
+        throw new Error('Refusing to refresh an unmarked database.');
+    if (!Number.isFinite(now)) throw new Error('Invalid fixture time.');
+    handle.transaction(() => {
+        for (const [index, server] of fixtureServers.entries()) {
+            handle
+                .prepare(
+                    `UPDATE servers SET last_seen_at=?,last_checked_at=?,container_checked_at=?,container_state=?,
+                last_players=?,last_map=?,last_roster=?,last_error=?,rcon_failures=?,last_reply_snippet=NULL WHERE name=?`
+                )
+                .run(
+                    seenAt(server, now),
+                    iso(now),
+                    iso(now),
+                    containerState(server),
+                    server.players,
+                    server.map.canonical,
+                    JSON.stringify(rosterFor(server, index)),
+                    server.state === 'stale' ? 'timeout: no reply from game server' : null,
+                    server.state === 'stale' ? 2 : 0,
+                    server.name
+                );
+            handle
+                .prepare(
+                    `UPDATE server_telemetry SET received_at=?,payload=json_set(payload,'$.receivedAt',?) WHERE server=?`
+                )
+                .run(iso(now), iso(now), server.name);
+            handle
+                .prepare(
+                    "UPDATE leaderboard_matches SET last_received_at=? WHERE server=? AND status='active' AND id BETWEEN 495 AND 500"
+                )
+                .run(iso(now), server.name);
+            handle
+                .prepare('UPDATE plutonium_versions SET checked_at=? WHERE server=?')
+                .run(iso(now), server.name);
+        }
+        for (let index = 1; index <= 3; index++) {
+            const id = `fixture-host-${index}`;
+            handle
+                .prepare(
+                    "UPDATE ec_hosts SET last_heartbeat=?,inventory=json_set(inventory,'$.instanceStates',json(?)) WHERE id=?"
+                )
+                .run(
+                    iso(now - (index === 3 ? 10 * MINUTE : 0)),
+                    JSON.stringify(instanceStates(id, now)),
+                    id
+                );
+        }
+    })();
+}

scripts/fixture-options.ts

@@ -0,0 +1,78 @@
+import { mkdir, open, realpath, rm } from 'node:fs/promises';
+import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
+
+export interface FixtureOptions {
+    dbPath: string;
+    now: number;
+    watch: boolean;
+}
+
+export function parseFixtureOptions(args: string[], clock = Date.now()): FixtureOptions {
+    let dbPath = '',
+        now = clock,
+        watch = false,
+        explicitTime = false;
+    const seen = new Set<string>();
+    for (let index = 0; index < args.length; index++) {
+        const option = args[index];
+        if (seen.has(option)) throw new Error(`Duplicate option: ${option}`);
+        seen.add(option);
+        if (option === '--watch') {
+            watch = true;
+            continue;
+        }
+        if (option !== '--db' && option !== '--now') throw new Error(`Unknown option: ${option}`);
+        const value = args[++index];
+        if (!value || value.startsWith('--')) throw new Error(`Missing value for ${option}.`);
+        if (option === '--db') dbPath = value;
+        else {
+            if (
+                !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(
+                    value
+                ) ||
+                !Number.isFinite(Date.parse(value))
+            )
+                throw new Error('--now must be an ISO timestamp with a timezone.');
+            now = Date.parse(value);
+            explicitTime = true;
+        }
+    }
+    if (!dbPath || !isAbsolute(dbPath))
+        throw new Error(
+            'Provide --db with an absolute path inside a new fixture directory. DB_PATH is never used.'
+        );
+    if (!Number.isFinite(now)) throw new Error('Invalid fixture clock.');
+    if (watch && explicitTime)
+        throw new Error('Use --watch for live time or --now for a fixed snapshot, separately.');
+    return { dbPath: resolve(dbPath), now, watch };
+}
+
+/** An atomic new-directory reservation rejects existing DBs, sidecars and symlinks together. */
+export async function reserveFixture(dbPath: string): Promise<{ root: string; dbPath: string }> {
+    if (!isAbsolute(dbPath)) throw new Error('Fixture database path must be absolute.');
+    const requested = resolve(dbPath);
+    const requestedRoot = dirname(requested);
+    // The destination's parent must already exist. Canonicalize /tmp on macOS,
+    // but never follow a pre-existing fixture directory or database symlink.
+    const parent = await realpath(dirname(requestedRoot));
+    const root = join(parent, basename(requestedRoot));
+    try {
+        await mkdir(root, { mode: 0o700 });
+    } catch (cause) {
+        if ((cause as NodeJS.ErrnoException).code === 'EEXIST')
+            throw new Error(
+                'Fixture directory already exists. Choose a new directory; existing data is never overwritten.',
+                { cause }
+            );
+        throw cause;
+    }
+    const target = join(root, basename(requested));
+    try {
+        const file = await open(target, 'wx', 0o600);
+        await file.close();
+    } catch (error) {
+        await rm(root, { recursive: true, force: true });
+        throw error;
+    }
+    return { root, dbPath: target };
+}

scripts/reset-admin.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+// Local recovery only: requires filesystem/container access to the controller.
+import Database from 'better-sqlite3';
+import { createHash } from 'node:crypto';
+import { resolve } from 'node:path';
+import { emitKeypressEvents } from 'node:readline';
+import { fileURLToPath } from 'node:url';
+import { hashPassword } from '../src/lib/server/auth/passwords.js';
+
+/** @param {string} prompt @returns {Promise<string>} */
+function secret(prompt) {
+    return new Promise((resolve, reject) => {
+        let value = '';
+        process.stdout.write(prompt);
+        emitKeypressEvents(process.stdin);
+        process.stdin.setRawMode(true);
+        process.stdin.resume();
+        const cleanup = () => {
+            process.stdin.off('keypress', listener);
+            process.stdin.setRawMode(false);
+            process.stdin.pause();
+            process.stdout.write('\n');
+        };
+        /** @param {string | undefined} text @param {import('node:readline').Key | undefined} key */
+        const listener = (text, key) => {
+            if (key?.ctrl && key.name === 'c') {
+                cleanup();
+                reject(new Error('Cancelled.'));
+            } else if (key?.name === 'return' || key?.name === 'enter') {
+                cleanup();
+                resolve(value);
+            } else if (key?.name === 'backspace') value = [...value].slice(0, -1).join('');
+            else if (
+                text &&
+                !key?.ctrl &&
+                !key?.meta &&
+                // eslint-disable-next-line no-control-regex -- Reject terminal control characters while collecting password input.
+                !/[\x00-\x1f\x7f]/.test(text) &&
+                [...value].length < 128
+            )
+                value += text;
+        };
+        process.stdin.on('keypress', listener);
+    });
+}
+
+/** @param {import('better-sqlite3').Database} database @param {string} username */
+function administrator(database, username) {
+    const user = /** @type {{ id: string, username: string } | undefined} */ (
+        database
+            .prepare(
+                "SELECT id,username FROM auth_users WHERE username=? AND role='admin' AND disabled=0"
+            )
+            .get(username)
+    );
+    if (!user) throw new Error('An enabled administrator with that username was not found.');
+    return user;
+}
+
+/**
+ * Local recovery shared with regression tests; the CLI only accepts passwords on its TTY.
+ * @param {import('better-sqlite3').Database} database
+ * @param {string} username
+ * @param {string} password
+ */
+export async function recoverAdministrator(database, username, password) {
+    const user = administrator(database, username);
+    const passwordHash = await hashPassword(password);
+    const digest = (/** @type {string} */ value) =>
+        createHash('sha256').update(value).digest('hex');
+    database
+        .transaction(() => {
+            const changed = database
+                .prepare(
+                    "UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0"
+                )
+                .run(passwordHash, Date.now(), user.id);
+            if (changed.changes !== 1)
+                throw new Error('Administrator changed during recovery. Try again.');
+            database.prepare('DELETE FROM auth_sessions WHERE user_id=?').run(user.id);
+            // Older databases may not have device credentials yet; recovery still works.
+            if (
+                database
+                    .prepare(
+                        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='auth_login_devices'"
+                    )
+                    .get()
+            ) {
+                database.prepare('DELETE FROM auth_login_devices WHERE user_id=?').run(user.id);
+            }
+            database
+                .prepare('DELETE FROM auth_rate_limits WHERE key GLOB ? OR key IN (?,?)')
+                .run(
+                    `login:account:${digest(user.username.toLowerCase())}:ip:*`,
+                    digest(`login:account:${user.username.toLowerCase()}`),
+                    digest(`password:account:${user.id.toLowerCase()}`)
+                );
+            database
+                .prepare(
+                    'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)'
+                )
+                .run('password.recovered', user.id, '{"source":"local-cli"}', Date.now());
+        })
+        .immediate();
+}
+
+async function main() {
+    const username = process.argv[2]?.toLowerCase();
+    if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username))
+        throw new Error('Usage: node scripts/reset-admin.mjs <administrator-username>');
+    if (!process.stdin.isTTY)
+        throw new Error(
+            'Run this command in an interactive terminal. Passwords are never accepted as command arguments.'
+        );
+    const database = new Database(process.env.DB_PATH || './data/cherry.db', {
+        fileMustExist: true
+    });
+    database.pragma('busy_timeout=5000');
+    try {
+        administrator(database, username);
+        const password = await secret('New administrator password (8–128 characters): ');
+        const confirmation = await secret('Repeat password: ');
+        if (password !== confirmation) throw new Error('Passwords did not match.');
+        await recoverAdministrator(database, username, password);
+        console.log(
+            'Administrator password reset. Sessions and trusted login devices revoked; account rate limits cleared.'
+        );
+    } finally {
+        database.close();
+    }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+    try {
+        await main();
+    } catch (cause) {
+        console.error(cause instanceof Error ? cause.message : 'Administrator recovery failed.');
+        process.exitCode = 1;
+    }
+}

scripts/seed-fixture.ts

@@ -0,0 +1,186 @@
+import { randomBytes } from 'node:crypto';
+import { createServer as createHttpServer } from 'node:http';
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createServer, type ViteDevServer } from 'vite';
+import type Database from 'better-sqlite3';
+import {
+    FIXTURE_KEY,
+    FIXTURE_VERSION,
+    fixtureServers,
+    refreshFixtureData,
+    seedFixtureData
+} from './fixture-data.js';
+import { parseFixtureOptions, reserveFixture } from './fixture-options.js';
+
+const help = `Create an isolated synthetic UI preview database.
+
+  npm run seed:fixture -- --db /tmp/ec-preview/cherry.db [--now 2026-10-07T12:00:00Z]
+  npm run seed:fixture -- --db /tmp/ec-live-preview/cherry.db --watch
+
+The database's directory must not exist; its parent must exist. Existing files,
+databases, symlinks and environment DB_PATH are never reused. --watch refreshes
+only this new fixture's observations every 15 seconds until Ctrl+C. --now fixes
+all sample times for reproducible snapshots and cannot be combined with --watch.
+No web server, worker, game agent or network request is started by this command.
+`;
+
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length === 1 && ['--help', '-h'].includes(args[0])) {
+        console.log(help);
+        return;
+    }
+    const options = parseFixtureOptions(args);
+    process.umask(0o077);
+    const destination = await reserveFixture(options.dbPath);
+    let vite: ViteDevServer | undefined;
+    let closeDb: (() => void) | undefined;
+    let complete = false;
+    try {
+        const paths = {
+            DB_PATH: destination.dbPath,
+            SERVERS_ROOT: join(destination.root, 't5'),
+            T6_SERVERS_ROOT: join(destination.root, 't6'),
+            BETA_ARCHIVE_DIR: join(destination.root, 'beta-archive'),
+            GAME_FILES_ROOT: join(destination.root, 'game-files'),
+            SQLITE_TMPDIR: join(destination.root, 'sqlite-tmp')
+        };
+        for (const [key, value] of Object.entries(paths)) {
+            process.env[key] = value;
+            if (key !== 'DB_PATH') await mkdir(value, { mode: 0o700 });
+        }
+        Object.assign(process.env, {
+            RCON_PASSWORD: '',
+            RCON_HOST: '127.0.0.1',
+            EC_AUTH_ALLOW_HTTP: 'true',
+            DEPLOY_KEEP_DAYS: '7',
+            DEPLOYMENT_RESTART_CONCURRENCY: '1',
+            DEPLOYMENT_CREATE_CONCURRENCY: '1',
+            EC_SQLITE_MMAP_MB: '0',
+            PLUTAINER_IMAGE: 'fixture.invalid/electric-cherry/preview:local'
+        });
+        const repository = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+        vite = await createServer({
+            root: repository,
+            cacheDir: join(destination.root, 'vite-cache'),
+            configFile: false,
+            envFile: false,
+            appType: 'custom',
+            // Vite 5 otherwise opens its HMR WebSocket port even with hmr:false.
+            // Attach it to an HTTP server that is deliberately never listened on.
+            server: { middlewareMode: true, hmr: { server: createHttpServer() }, watch: null },
+            optimizeDeps: { noDiscovery: true },
+            resolve: {
+                alias: { $lib: join(repository, 'src/lib'), $ui: join(repository, 'src/lib/ui') }
+            },
+            logLevel: 'error'
+        });
+        // Load the real migrations through Vite's ?raw support. Importing hooks
+        // would start queues and agents; this script never loads the app router.
+        const database = (await vite.ssrLoadModule('/src/lib/server/db/index.ts')) as {
+            db: () => Database.Database;
+            closeDb: () => void;
+        };
+        closeDb = database.closeDb;
+        const handle = database.db();
+        seedFixtureData(handle, options.now);
+        const password = randomBytes(24).toString('base64url');
+        const { hashPassword } = await vite.ssrLoadModule('/src/lib/server/auth/passwords.js');
+        handle
+            .prepare(
+                `INSERT INTO auth_users(id,username,display_name,password_hash,role,created_at,updated_at)
+            VALUES('fixture-admin','demo','Demo operator',?,'admin',?,?)`
+            )
+            .run(await hashPassword(password), options.now, options.now);
+        if (
+            (handle.pragma('foreign_key_check') as unknown[]).length ||
+            handle.pragma('integrity_check', { simple: true }) !== 'ok'
+        ) {
+            throw new Error('Fixture integrity validation failed.');
+        }
+        const environment = {
+            ...paths,
+            RCON_PASSWORD: '',
+            RCON_HOST: '127.0.0.1',
+            EC_AUTH_ALLOW_HTTP: 'true',
+            PLUTAINER_IMAGE: process.env.PLUTAINER_IMAGE!
+        };
+        const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
+        await writeFile(
+            join(destination.root, 'preview.env'),
+            Object.entries(environment)
+                .map(([key, value]) => `export ${key}=${shellQuote(value)}`)
+                .join('\n') + '\n',
+            { flag: 'wx', mode: 0o600 }
+        );
+        await writeFile(
+            join(destination.root, 'fixture.json'),
+            JSON.stringify(
+                {
+                    format: FIXTURE_VERSION,
+                    marker: FIXTURE_KEY,
+                    dbPath: destination.dbPath,
+                    createdAt: new Date(options.now).toISOString(),
+                    username: 'demo',
+                    password,
+                    hosts: 3,
+                    servers: fixtureServers.map((server) => server.name),
+                    matches: 500,
+                    note: 'Synthetic local data only. Host credentials cannot authenticate; game images and library IDs are fictitious.'
+                },
+                null,
+                2
+            ) + '\n',
+            { flag: 'wx', mode: 0o600 }
+        );
+        handle.pragma('wal_checkpoint(TRUNCATE)');
+        complete = true;
+        console.log(
+            `Created ${destination.dbPath}\n3 hosts · 12 servers · 500 matches · 7 days of metrics\nLocal sign-in details: ${join(destination.root, 'fixture.json')}\nPreview environment: ${join(destination.root, 'preview.env')}`
+        );
+        if (options.watch) {
+            refreshFixtureData(handle, Date.now());
+            console.log(
+                'Refreshing synthetic observations every 15 seconds. Leave this terminal open; Ctrl+C stops the feeder.'
+            );
+            await new Promise<void>((resolvePromise, reject) => {
+                const finish = (error?: unknown) => {
+                    clearInterval(timer);
+                    process.off('SIGINT', stop);
+                    process.off('SIGTERM', stop);
+                    if (error) reject(error);
+                    else resolvePromise();
+                };
+                const stop = () => finish();
+                const timer = setInterval(() => {
+                    try {
+                        refreshFixtureData(handle, Date.now());
+                    } catch (error) {
+                        finish(error);
+                    }
+                }, 15_000);
+                process.on('SIGINT', stop);
+                process.on('SIGTERM', stop);
+            });
+        }
+    } finally {
+        try {
+            closeDb?.();
+        } finally {
+            try {
+                await vite?.close();
+            } finally {
+                if (!complete) await rm(destination.root, { recursive: true, force: true });
+            }
+        }
+    }
+}
+
+main().catch((error) => {
+    console.error(
+        `Fixture creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`
+    );
+    process.exitCode = 1;
+});

src/app.d.ts

@@ -1,5 +1,10 @@
 declare global {
-    namespace App {}
+    namespace App {
+        interface Locals {
+            user: import('./lib/auth.js').AuthUser | null;
+            sessionHash: string | null;
+        }
+    }
 }
 
 export {};

src/hooks.server.ts

@@ -2,20 +2,61 @@ import type { Handle } from '@sveltejs/kit';
 import { migrate } from '$lib/server/db/migrate.js';
 import { startWorker } from '$lib/server/pipeline/worker.js';
 import { interruptDeployments } from '$lib/server/provisioning/history.js';
+import { authenticationHandle } from '$lib/server/auth/guard.js';
+import { invalidateFleet } from '$lib/server/pipeline/bus.js';
+import { startPerformanceMonitor } from '$lib/server/performance.js';
+import { withOperationSource } from '$lib/server/hosts/operation-context.js';
+import { withHostRequestContext } from '$lib/server/hosts/request-context.js';
+import { startDiscordWorker } from '$lib/server/discord/worker.js';
 
 let booted = false;
+let rejectionLoggerRegistered = false;
 function boot(): void {
     if (booted) return;
-    booted = true;
+    if (!rejectionLoggerRegistered) {
+        process.on('unhandledRejection', (reason) => {
+            console.error('[process] unhandled rejection:', reason);
+        });
+        rejectionLoggerRegistered = true;
+    }
     migrate();
     interruptDeployments();
     startWorker();
+    startDiscordWorker();
+    startPerformanceMonitor();
+    booted = true;
 }
 
-// SvelteKit calls handle() for every request; using the first request as the
+// Boot synchronously before serving requests. Failed startup stays unbooted so
-// boot trigger keeps the whole app single-process without needing an
+// the next request retries migrations and worker startup before reaching the UI.
-// out-of-band init hook.
+export const handle: Handle = async (input) => {
-export const handle: Handle = async ({ event, resolve }) => {
     boot();
-    return resolve(event);
+    const started = performance.now();
+    const response = await withHostRequestContext(() =>
+        withOperationSource(
+            `${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
+            () => authenticationHandle(input)
+        )
+    );
+    const elapsed = performance.now() - started;
+    response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
+    if (
+        process.env.EC_PERF_LOG === 'true' &&
+        elapsed > 250 &&
+        !input.event.route.id?.startsWith('/api/agent/')
+    ) {
+        console.info('[request]', {
+            route: input.event.route.id,
+            method: input.event.request.method,
+            status: response.status,
+            elapsedMs: Math.round(elapsed)
+        });
+    }
+    if (
+        input.event.locals.user &&
+        !['GET', 'HEAD', 'OPTIONS'].includes(input.event.request.method) &&
+        response.status < 400
+    )
+        invalidateFleet();
+    return response;
 };

src/lib/activity-redirect.ts

@@ -0,0 +1,44 @@
+const historyFilters = ['host', 'type', 'source', 'state', 'days', 'q', 'limit', 'offset'];
+
+/** Route retired activity bookmarks to the existing history that owns their scope. */
+export function activityRedirectHref(url: URL): string {
+    const params = new URLSearchParams();
+    const copy = (keys: string[]) => {
+        for (const key of keys) {
+            const value = url.searchParams.get(key);
+            if (value) params.set(key, value);
+        }
+    };
+    const server = url.searchParams.get('server')?.trim();
+    const host = url.searchParams.get('host')?.trim();
+    const source = url.searchParams.get('source');
+    let path: string;
+    let hash = '';
+    if (server) {
+        path = `/servers/${encodeURIComponent(server)}`;
+        params.set('tab', 'history');
+        copy(historyFilters);
+        hash = url.hash;
+    } else if (host || source === 'host') {
+        path = '/hosts/operations';
+        if (host) params.set('host', host);
+    } else if (source === 'maintenance' || source === 'recovery') {
+        path = '/maintenance';
+        params.set('view', 'activity');
+        copy(historyFilters);
+        hash = url.hash;
+    } else if (source === 'campaign') {
+        path = '/campaigns';
+        copy(['q']);
+    } else if (source === 'beta') {
+        path = '/betas';
+        params.set('view', 'history');
+        copy(['q']);
+    } else {
+        path = '/provisioning';
+        params.set('view', 'history');
+        if (source === 'autoscaling') params.set('source', source);
+    }
+    copy(['returnTo']);
+    return path + (params.size ? `?${params}` : '') + hash;
+}

src/lib/activity.ts

@@ -0,0 +1,118 @@
+export const activityTypes = {
+    create: 'Create server',
+    start: 'Start server',
+    restart: 'Restart server',
+    rebuild: 'Rebuild server',
+    stop: 'Stop server',
+    remove: 'Destroy server',
+    update: 'Game update',
+    files: 'File changes',
+    health: 'Health finding',
+    drift: 'File drift',
+    host: 'Host operation',
+    beta: 'Beta test'
+} as const;
+export type ActivityType = keyof typeof activityTypes;
+export const activitySources = {
+    provisioning: 'Server operations',
+    maintenance: 'Maintenance & recovery',
+    recovery: 'Recovery',
+    campaign: 'Campaigns',
+    beta: 'Betas',
+    host: 'Hosts',
+    autoscaling: 'Autoscaling'
+} as const;
+export type ActivitySource = keyof typeof activitySources;
+export const activityStates = {
+    queued: 'Queued',
+    running: 'In progress',
+    succeeded: 'Completed',
+    warning: 'Completed with warnings',
+    failed: 'Failed',
+    cancelled: 'Cancelled',
+    attention: 'Needs attention',
+    unknown: 'Unknown'
+} as const;
+export type ActivityState = keyof typeof activityStates;
+
+export interface ActivityRow {
+    id: string;
+    at: string;
+    type: ActivityType;
+    source: ActivitySource;
+    /** A shared operation can have more than one caller; filters match any membership. */
+    sources: ActivitySource[];
+    state: ActivityState;
+    rawState: string;
+    title: string;
+    detail: string | null;
+    server: string | null;
+    serverDeletedAt: string | null;
+    hostId: string | null;
+    hostName: string | null;
+    href: string;
+}
+
+export interface ActivityQuery {
+    excludeProvisioning?: boolean;
+    excludeMaintenance?: boolean;
+    server?: string;
+    host?: string;
+    type?: ActivityType;
+    source?: ActivitySource;
+    state?: ActivityState;
+    days?: number;
+    q?: string;
+    limit?: number;
+    offset?: number;
+}
+export interface ActivityPage {
+    rows: ActivityRow[];
+    total: number;
+    limit: number;
+    offset: number;
+    query: ActivityQuery;
+}
+
+function selection<T extends string>(
+    value: string | null,
+    labels: Record<T, string>
+): T | undefined {
+    return value !== null && Object.hasOwn(labels, value) ? (value as T) : undefined;
+}
+function integer(value: string | null, fallback: number, min: number, max: number): number {
+    if (value === null || !/^\d+$/.test(value)) return fallback;
+    const parsed = Number(value);
+    return Number.isSafeInteger(parsed) ? Math.min(max, Math.max(min, parsed)) : fallback;
+}
+
+/** Server and host are exact identities; q searches friendly names and safe summaries. */
+export function parseActivityQuery(params: URLSearchParams): ActivityQuery {
+    const query: ActivityQuery = {
+        limit: integer(params.get('limit'), 25, 1, 100),
+        offset: integer(params.get('offset'), 0, 0, 1_000_000_000)
+    };
+    if (params.get('excludeProvisioning') === 'true') query.excludeProvisioning = true;
+    if (params.get('excludeMaintenance') === 'true') query.excludeMaintenance = true;
+    for (const key of ['server', 'host', 'q'] as const) {
+        const value = params
+            .get(key)
+            ?.trim()
+            .slice(0, key === 'q' ? 200 : 255);
+        if (value) query[key] = value;
+    }
+    query.type = selection(params.get('type'), activityTypes);
+    query.source = selection(params.get('source'), activitySources);
+    query.state = selection(params.get('state'), activityStates);
+    // Missing or zero days means all retained history, including removed servers.
+    const days = integer(params.get('days'), 0, 0, 36_500);
+    if (days) query.days = days;
+    return query;
+}
+
+export function normalizeActivityQuery(query: ActivityQuery): ActivityQuery {
+    const params = new URLSearchParams();
+    for (const [key, value] of Object.entries(query))
+        if (value !== undefined) params.set(key, String(value));
+    return parseActivityQuery(params);
+}

src/lib/afk.ts

@@ -0,0 +1,143 @@
+export const afkPageSize = 25;
+export const afkLedgerKinds = [
+    'free_grant',
+    'purchase',
+    'reward',
+    'reserve',
+    'start',
+    'release',
+    'unresolved',
+    'consume',
+    'correction'
+] as const;
+export type AfkLedgerKind = (typeof afkLedgerKinds)[number];
+export type AfkSessionState = 'reserved' | 'started' | 'settled' | 'unresolved' | 'corrected';
+
+export interface AfkQuery {
+    view: 'accounts' | 'sessions' | 'ledger';
+    q: string;
+    balance: '' | 'saved' | 'reserved' | 'unresolved';
+    sort: 'saved' | 'available' | 'recent';
+    game: '' | 't5' | 't6';
+    server: string;
+    state: '' | AfkSessionState;
+    kind: '' | AfkLedgerKind;
+    from: string;
+    to: string;
+    offset: number;
+    player: string;
+}
+
+export function readAfkQuery(params: URLSearchParams): AfkQuery {
+    function choice<T extends string>(key: string, values: readonly T[], fallback: T): T {
+        const value = params.get(key) || fallback;
+        if (!values.includes(value as T)) throw new Error(`Invalid ${key} filter.`);
+        return value as T;
+    }
+    function text(key: string, max: number) {
+        const value = (params.get(key) ?? '').trim();
+        if (value.length > max) throw new Error(`${key} is too long.`);
+        return value;
+    }
+    function date(key: string) {
+        const value = text(key, 10);
+        if (
+            value &&
+            (!/^\d{4}-\d{2}-\d{2}$/.test(value) ||
+                !Number.isFinite(Date.parse(value)) ||
+                new Date(value).toISOString().slice(0, 10) !== value)
+        )
+            throw new Error('Use a valid date in YYYY-MM-DD format.');
+        return value;
+    }
+    const offset = Number(params.get('offset') || 0);
+    if (!Number.isSafeInteger(offset) || offset < 0) throw new Error('Invalid page offset.');
+    const from = date('from');
+    const to = date('to');
+    if (from && to && from > to) throw new Error('The end date must follow the start date.');
+    return {
+        view: choice('view', ['accounts', 'sessions', 'ledger'], 'accounts'),
+        q: text('q', 100),
+        balance: choice('balance', ['', 'saved', 'reserved', 'unresolved'], ''),
+        sort: choice('sort', ['saved', 'available', 'recent'], 'saved'),
+        game: choice('game', ['', 't5', 't6'], ''),
+        server: text('server', 256),
+        state: choice(
+            'state',
+            ['', 'reserved', 'started', 'settled', 'unresolved', 'corrected'],
+            ''
+        ),
+        kind: choice('kind', ['', ...afkLedgerKinds], ''),
+        from,
+        to,
+        offset: Math.floor(offset / afkPageSize) * afkPageSize,
+        player: text('player', 128)
+    };
+}
+
+export interface AfkIdentity {
+    guid: string;
+    name: string | null;
+}
+export interface AfkAccount extends AfkIdentity {
+    freeSeconds: number;
+    savedSeconds: number;
+    availableSeconds: number;
+    reservedSeconds: number;
+    reservedSavedSeconds: number;
+    unresolvedSessions: number;
+    lastActivity: string | null;
+    freeRenewsAt: string | null;
+}
+export interface AfkSession extends AfkIdentity {
+    sessionId: string;
+    server: string;
+    game: string | null;
+    matchId: number | null;
+    matchAvailable: number;
+    state: AfkSessionState;
+    authorizedSeconds: number;
+    consumedSeconds: number | null;
+    chargedSeconds: number | null;
+    confirmedSeconds: number;
+    endReason: string | null;
+    createdAt: string;
+    startedAt: string | null;
+    endedAt: string | null;
+}
+export interface AfkLedgerEntry extends AfkIdentity {
+    id: number;
+    server: string;
+    game: string | null;
+    matchId: number | null;
+    matchAvailable: number;
+    kind: AfkLedgerKind;
+    freeDelta: number;
+    savedDelta: number;
+    freeSeconds: number;
+    savedSeconds: number;
+    operationId: string;
+    sessionId: string | null;
+    eventAt: string;
+    createdAt: string;
+}
+export interface AfkPage<T> {
+    entries: T[];
+    total: number;
+    offset: number;
+}
+export interface AfkOverview {
+    query: AfkQuery;
+    checkedAt: string;
+    capacitySeconds: number;
+    summary: {
+        accounts: number;
+        savedSeconds: number;
+        reservedSeconds: number;
+        unresolvedSessions: number;
+    };
+    servers: string[];
+    accounts: AfkPage<AfkAccount>;
+    sessions: AfkPage<AfkSession>;
+    ledger: AfkPage<AfkLedgerEntry>;
+}

src/lib/artifact-limits.ts

@@ -0,0 +1,2 @@
+// Keep managed artifacts within the host agent's MAX_FILE limit.
+export const MAX_ARTIFACT_BYTES = 32 * 1024 * 1024;

src/lib/artifact-version.ts

@@ -1,7 +1,25 @@
+import type { LaunchFileSummary } from './deployment-plan.js';
+
 /** Keep the upload number visible: two builds may share a publisher's release tag. */
-export function artifactVersionLabel(version: { version: number; release_version?: string | null } | null | undefined): string {
+export function artifactVersionLabel(
+    version: { version: number; release_version?: string | null } | null | undefined,
+    kind?: string
+): string {
     if (!version) return '—';
     return version.release_version
         ? `${version.release_version} (upload ${version.version})`
+        : kind === 'script'
+          ? `Unversioned (upload ${version.version})`
           : `v${version.version}`;
 }
+
+export function launchFileVersionLabel(
+    file: Pick<LaunchFileSummary, 'version' | 'releaseVersion' | 'kind'>
+): string {
+    return file.version === null
+        ? 'Generated'
+        : artifactVersionLabel(
+              { version: file.version, release_version: file.releaseVersion },
+              file.kind
+          );
+}

src/lib/auth.ts

@@ -0,0 +1,46 @@
+export type UserRole = 'admin' | 'super_viewer' | 'operator' | 'moderator' | 'viewer';
+
+export const moderatorMaximumRestrictionMs = 7 * 24 * 60 * 60 * 1000;
+
+/** Reserved identity for work requested by the controller. */
+export const systemUsername = 'Maxis';
+
+export interface AuthUser {
+    id: string;
+    username: string;
+    displayName: string;
+    role: UserRole;
+    disabled: boolean;
+    mustChangePassword: boolean;
+    createdAt: string;
+    updatedAt: string;
+    lastLoginAt: string | null;
+}
+
+export const roleLabels: Record<UserRole, string> = {
+    admin: 'Administrator',
+    super_viewer: 'Super Viewer',
+    operator: 'Operator',
+    moderator: 'Moderator',
+    viewer: 'Viewer'
+};
+
+export function canOperate(role: UserRole | null | undefined): boolean {
+    return role === 'admin' || role === 'operator';
+}
+
+export function canReadAdmin(role: UserRole | null | undefined): boolean {
+    return role === 'admin' || role === 'super_viewer';
+}
+
+export function canReadSensitive(role: UserRole | null | undefined): boolean {
+    return canOperate(role) || role === 'super_viewer';
+}
+
+export function canModerate(role: UserRole | null | undefined): boolean {
+    return role === 'admin' || role === 'moderator';
+}
+
+export function canReadModeration(role: UserRole | null | undefined): boolean {
+    return canModerate(role) || role === 'super_viewer';
+}

src/lib/autoscaling.ts

@@ -0,0 +1,130 @@
+import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
+
+export const autoscalingModes = ['off', 'create-destroy'] as const;
+export type AutoscalingMode = (typeof autoscalingModes)[number];
+export interface ScalingSettings {
+    game: ZombieGame;
+    /** '*' applies the targets independently to every map. */
+    mapSlug: string;
+    mode: AutoscalingMode;
+    minimum: number;
+    maximum: number;
+    spares: number;
+    emptyMinutes: number;
+}
+export interface ScalingPolicy extends ScalingSettings {
+    id: string;
+    revision: number;
+    pausedReason: string | null;
+    updatedAt: string;
+}
+export interface ScalingMember {
+    name: string;
+    mapSlug?: string;
+    identity: string;
+    state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
+    players: number | null;
+    /** Current occupied match round from fresh telemetry; null when unavailable. */
+    round: number | null;
+    observedAt: number | null;
+    source: string | null;
+    emptySince: number | null;
+    blocker: string | null;
+    operationId: string | null;
+    emptyStartedAt?: number | null;
+}
+export interface ScalingSnapshot {
+    members: ScalingMember[];
+    reservedNames: string[];
+    keyPool: { total: number; available: number };
+    blockers: string[];
+}
+export interface ScalingDecision {
+    action: 'create' | 'destroy' | 'wait';
+    reason: string;
+    server: string | null;
+    desired: number;
+    deficit: number;
+    occupied: number;
+    empty: number;
+    pending: number;
+    unknown: number;
+}
+export interface ScalingEvaluation {
+    policy: ScalingPolicy;
+    snapshot: ScalingSnapshot;
+    decision: ScalingDecision;
+    blockers: string[];
+}
+export interface ScalingHistory {
+    id: string;
+    at: number;
+    lastSeenAt: number;
+    decision: ScalingDecision;
+    operationId: string | null;
+    operationState: string | null;
+}
+export interface ScalingView extends ScalingEvaluation {
+    history: ScalingHistory[];
+    lastEvaluatedAt: number | null;
+    totals: {
+        observedMinutes: number;
+        emptyLobbyHours: number;
+        deficitMinutes: number;
+        unknownMinutes: number;
+    };
+    operations: {
+        creates: number;
+        removals: number;
+        cancelled: number;
+        failed: number;
+        readinessSamples: number;
+        readinessMeanSeconds: number | null;
+        readinessMaxSeconds: number | null;
+    };
+    ownsMap: boolean;
+    unsettled: { id: string; action: string; state: string; stage: string }[];
+}
+export interface ScalingReport {
+    policies: ScalingView[];
+    globalPaused: boolean;
+    liveAvailable: boolean;
+    removalAvailable: boolean;
+}
+export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-destroy';
+
+export function scalingSettings(input: unknown): ScalingSettings {
+    if (!input || typeof input !== 'object' || Array.isArray(input))
+        throw new Error('Provide an autoscaling policy.');
+    const value = input as Record<string, unknown>;
+    const { game, mapSlug } = value;
+    if (
+        (game !== 't5' && game !== 't6') ||
+        typeof mapSlug !== 'string' ||
+        (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)
+    )
+        throw new Error('Choose a supported game and map variant, or all maps.');
+    const mode = value.mode ?? 'off';
+    if (!autoscalingModes.includes(mode as AutoscalingMode))
+        throw new Error('Choose a valid autoscaling mode.');
+    const integer = (key: string, fallback: number | undefined, max: number) => {
+        const n = value[key] ?? fallback;
+        if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max)
+            throw new Error(`${key} must be an integer from 1 to ${max}.`);
+        return n;
+    };
+    const minimum = integer('minimum', 1, 20),
+        maximum = integer('maximum', undefined, 20),
+        spares = integer('spares', 1, 20);
+    if (minimum > maximum || spares > maximum)
+        throw new Error('Minimum and spare target cannot exceed the maximum.');
+    return {
+        game,
+        mapSlug,
+        mode: mode as AutoscalingMode,
+        minimum,
+        maximum,
+        spares,
+        emptyMinutes: integer('emptyMinutes', 15, 1440)
+    };
+}

src/lib/bank.ts

@@ -0,0 +1,116 @@
+export const bankPageSize = 25;
+export const bankAttentionAgeMs = 5 * 60_000;
+
+export interface BankQuery {
+    view: 'accounts' | 'transactions' | 'pending';
+    q: string;
+    balance: '' | 'funded' | 'limit' | 'pending';
+    sort: 'balance' | 'recent';
+    game: '' | 't5' | 't6';
+    server: string;
+    operation: '' | 'deposit' | 'withdraw';
+    from: string;
+    to: string;
+    offset: number;
+    player: string;
+    historyOffset: number;
+}
+
+export function readBankQuery(params: URLSearchParams): BankQuery {
+    function choice<T extends string>(key: string, values: readonly T[], fallback: T): T {
+        const value = params.get(key) || fallback;
+        if (!values.includes(value as T)) throw new Error(`Invalid ${key} filter.`);
+        return value as T;
+    }
+    function text(key: string, max: number) {
+        const value = (params.get(key) ?? '').trim();
+        if (value.length > max) throw new Error(`${key} is too long.`);
+        return value;
+    }
+    function offset(key: string) {
+        const value = Number(params.get(key) || 0);
+        if (!Number.isSafeInteger(value) || value < 0) throw new Error('Invalid page offset.');
+        return Math.floor(value / bankPageSize) * bankPageSize;
+    }
+    function date(key: string) {
+        const value = text(key, 10);
+        if (
+            value &&
+            (!/^\d{4}-\d{2}-\d{2}$/.test(value) ||
+                !Number.isFinite(Date.parse(value)) ||
+                new Date(value).toISOString().slice(0, 10) !== value)
+        )
+            throw new Error('Use a valid date in YYYY-MM-DD format.');
+        return value;
+    }
+    const from = date('from');
+    const to = date('to');
+    if (from && to && from > to) throw new Error('The end date must follow the start date.');
+    return {
+        view: choice('view', ['accounts', 'transactions', 'pending'], 'accounts'),
+        q: text('q', 100),
+        balance: choice('balance', ['', 'funded', 'limit', 'pending'], ''),
+        sort: choice('sort', ['balance', 'recent'], 'balance'),
+        game: choice('game', ['', 't5', 't6'], ''),
+        server: text('server', 256),
+        operation: choice('operation', ['', 'deposit', 'withdraw'], ''),
+        from,
+        to,
+        offset: offset('offset'),
+        player: text('player', 128),
+        historyOffset: offset('historyOffset')
+    };
+}
+
+export interface BankIdentity {
+    guid: string;
+    name: string | null;
+}
+export interface BankAccount extends BankIdentity {
+    balance: number;
+    available: number;
+    pendingOperation: 'deposit' | 'withdraw' | null;
+    pendingAmount: number | null;
+    lastActivity: string | null;
+    lastAmount: number | null;
+}
+export interface BankTransfer extends BankIdentity {
+    id: number;
+    server: string;
+    game: string | null;
+    matchId: number | null;
+    matchAvailable: number;
+    amount: number;
+    balance: number;
+    createdAt: string;
+}
+export interface BankPending extends BankIdentity {
+    requestId: string;
+    server: string;
+    game: string | null;
+    matchId: number | null;
+    matchAvailable: number;
+    operation: 'deposit' | 'withdraw';
+    amount: number;
+    createdAt: string;
+}
+export interface BankPage<T> {
+    entries: T[];
+    total: number;
+    offset: number;
+}
+export interface BankOverview {
+    query: BankQuery;
+    maximum: number | null;
+    checkedAt: string;
+    summary: { totalBanked: number; fundedAccounts: number; transfers24h: number; pending: number };
+    servers: string[];
+    accounts: BankPage<BankAccount>;
+    transactions: BankPage<BankTransfer>;
+    pending: BankPage<BankPending>;
+    detail: {
+        account: BankAccount;
+        pending: BankPending | null;
+        history: BankPage<BankTransfer>;
+    } | null;
+}

src/lib/beta-draft.ts

@@ -2,9 +2,14 @@ import { requirementKinds, type BetaPlan } from './beta-plan.js';
 
 export const betaDraftKey = 'electric-cherry:beta-draft:v1';
 export interface BetaDraft {
-    version: 1; savedAt: string; step: number;
+    version: 1;
+    savedAt: string;
+    step: number;
     picks: Array<{ scriptId: number | null; versionId: number | null }>;
-    servers: string[]; testPlan: BetaPlan;
+    servers: string[];
+    testPlan: BetaPlan;
+    selectionMode: 'automatic' | 'manual';
+    instanceCount: number | null;
     label?: string;
 }
 /** Validate draft structure, preserving incomplete numeric fields for correction. */
@@ -12,16 +17,138 @@ export function readBetaDraft(raw: string | null): BetaDraft | null {
     if (!raw) return null;
     try {
         const d = JSON.parse(raw);
-        if (d?.version !== 1 || !Array.isArray(d.picks) || !d.picks.length || d.picks.length > 100
+        if (
-            || !d.picks.every((p: BetaDraft['picks'][number]) => p && [p.scriptId, p.versionId].every(v => v === null || Number.isSafeInteger(v) && v! > 0))
+            d?.version !== 1 ||
-            || !Array.isArray(d.servers) || !d.servers.every((s: unknown) => typeof s === 'string')
+            !Array.isArray(d.picks) ||
-            || d.testPlan?.version !== 1 || !Array.isArray(d.testPlan.requirements) || d.testPlan.requirements.length > 30
+            !d.picks.length ||
-            || !d.testPlan.requirements.every((r: BetaPlan['requirements'][number]) => r && requirementKinds.includes(r.kind) && typeof r.id === 'string' && /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) && typeof r.label === 'string')) return null;
+            d.picks.length > 100 ||
-        return { ...d, step: Math.max(0, Math.min(3, Number(d.step) || 0)) };
+            !d.picks.every(
-    } catch { return null; }
+                (p: BetaDraft['picks'][number]) =>
+                    p &&
+                    [p.scriptId, p.versionId].every(
+                        (v) => v === null || (Number.isSafeInteger(v) && v! > 0)
+                    )
+            ) ||
+            !Array.isArray(d.servers) ||
+            !d.servers.every((s: unknown) => typeof s === 'string') ||
+            d.testPlan?.version !== 1 ||
+            !Array.isArray(d.testPlan.requirements) ||
+            d.testPlan.requirements.length > 30 ||
+            !d.testPlan.requirements.every(
+                (r: BetaPlan['requirements'][number]) =>
+                    r &&
+                    requirementKinds.includes(r.kind) &&
+                    typeof r.id === 'string' &&
+                    /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) &&
+                    typeof r.label === 'string'
+            )
+        )
+            return null;
+        if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode))
+            return null;
+        if (
+            d.instancesPerGame !== undefined &&
+            (!d.instancesPerGame ||
+                typeof d.instancesPerGame !== 'object' ||
+                Array.isArray(d.instancesPerGame) ||
+                !Object.entries(d.instancesPerGame).every(
+                    ([game, count]) =>
+                        ['t5', 't6'].includes(game) &&
+                        (count === null || (typeof count === 'number' && Number.isFinite(count)))
+                ))
+        )
+            return null;
+        if (
+            d.instanceCount !== undefined &&
+            d.instanceCount !== null &&
+            (typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))
+        )
+            return null;
+        const previousCounts = Object.values(d.instancesPerGame ?? {}) as Array<number | null>;
+        const instanceCount =
+            d.instanceCount !== undefined
+                ? d.instanceCount
+                : previousCounts.includes(null)
+                  ? null
+                  : previousCounts.length
+                    ? Math.max(...(previousCounts as number[]))
+                    : 1;
+        // Existing drafts represent explicit selections and must not silently switch modes.
+        const selectionMode = d.selectionMode ?? (d.servers.length ? 'manual' : 'automatic');
+        // Revisit server setup when old per-game counts cannot be retained as one value.
+        const changedCounts =
+            selectionMode === 'automatic' &&
+            d.instanceCount === undefined &&
+            new Set(previousCounts).size > 1;
+        return {
+            ...d,
+            selectionMode,
+            instanceCount,
+            step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0))
+        };
+    } catch {
+        return null;
+    }
 }
 
-export function compatibleBetaServers(selected: Iterable<string>, candidates: readonly string[]): { kept: Set<string>; removed: string[] } {
+interface BetaFileScope {
-    const allowed = new Set(candidates), names = [...selected];
+    scope_type: 'game' | 'map' | null;
-    return { kept: new Set(names.filter(name => allowed.has(name))), removed: names.filter(name => !allowed.has(name)) };
+    scope_value: string | null;
+}
+interface BetaServerScope {
+    name: string;
+    game: string | null;
+    map: string | null;
+}
+
+export function betaScopeMatches(file: BetaFileScope, server: BetaServerScope): boolean {
+    return (
+        !file.scope_type ||
+        (file.scope_type === 'game' ? server.game : server.map) === file.scope_value
+    );
+}
+
+/** A bundle installs each file only on its own matching participants. */
+export function betaCandidateServers<T extends BetaServerScope>(
+    files: readonly BetaFileScope[],
+    servers: readonly T[]
+): T[] {
+    return servers.filter((server) => files.some((file) => betaScopeMatches(file, server)));
+}
+
+export function betaApplicableGames(
+    files: readonly BetaFileScope[],
+    servers: readonly BetaServerScope[]
+): string[] {
+    return [
+        ...new Set(
+            [
+                ...files
+                    .filter((file) => file.scope_type === 'game')
+                    .map((file) => file.scope_value),
+                ...betaCandidateServers(files, servers).map((server) => server.game)
+            ].filter((game): game is string => game === 't5' || game === 't6')
+        )
+    ].sort();
+}
+
+/** Expand the shared count only to games represented by the selected files. */
+export function betaInstanceCounts(
+    games: readonly string[],
+    count: number | null
+): Record<string, number> {
+    if (count === null || !Number.isSafeInteger(count) || count < 1 || count > 1000) return {};
+    return Object.fromEntries(games.map((game) => [game, count]));
+}
+
+export function compatibleBetaServers(
+    selected: Iterable<string>,
+    candidates: readonly string[]
+): { kept: Set<string>; removed: string[] } {
+    const allowed = new Set(candidates),
+        names = [...selected];
+    return {
+        kept: new Set(names.filter((name) => allowed.has(name))),
+        removed: names.filter((name) => !allowed.has(name))
+    };
 }

src/lib/beta-evidence.ts

@@ -1,23 +1,35 @@
 import type { BetaRequirement, BetaTestView } from './beta-plan.js';
 import { needsGameTelemetry } from './beta-plan-summary.js';
 
-export function betaWaitingCondition(requirement: BetaRequirement, observation?: BetaTestView['servers'][number]['observation']): string {
+export function betaWaitingCondition(
-    if (requirement.kind === 'manual') return 'Waiting for a tester to record their name and observation.';
+    requirement: BetaRequirement,
-    if (requirement.kind === 'recovery') return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
+    observation?: BetaTestView['servers'][number]['observation']
-    if (requirement.kind === 'healthy-minutes') return observation?.healthy === true
+): string {
+    if (requirement.kind === 'manual')
+        return 'Waiting for a tester to record their name and observation.';
+    if (requirement.kind === 'recovery')
+        return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
+    if (requirement.kind === 'healthy-minutes')
+        return observation?.healthy === true
             ? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.'
             : 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.';
-    if (requirement.kind === 'failure-budget') return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
+    if (requirement.kind === 'failure-budget')
+        return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
     if (needsGameTelemetry(requirement)) {
-        if (!observation?.lastGameplayAt) return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
+        if (!observation?.lastGameplayAt)
+            return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
         if (observation.gameEnded) return 'Waiting for the next game and its qualifying reports.';
-        if (requirement.kind === 'population') return `Waiting for a game report containing at least ${requirement.target} players.`;
+        if (requirement.kind === 'population')
+            return `Waiting for a game report containing at least ${requirement.target} players.`;
         const players = `at least ${requirement.minPlayers} player${requirement.minPlayers === 1 ? '' : 's'}`;
-        if ((observation.players ?? 0) < requirement.minPlayers) return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
+        if ((observation.players ?? 0) < requirement.minPlayers)
-        if (requirement.kind === 'matches') return requirement.minMinutes && !observation.matchStarted
+            return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
+        if (requirement.kind === 'matches')
+            return requirement.minMinutes && !observation.matchStarted
                 ? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.'
                 : `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`;
-        if (requirement.kind === 'round') return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
+        if (requirement.kind === 'round')
+            return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
         return `Time is credited when the next report arrives, with ${players} at both ends of the interval. It does not advance continuously between reports.`;
     }
     return 'Waiting for qualifying evidence.';
@@ -25,13 +37,26 @@ export function betaWaitingCondition(requirement: BetaRequirement, observation?:
 
 export function betaEvidenceSummary(event: BetaTestView['evidence'][number]): string {
     let payload: Record<string, unknown>;
-    try { payload = JSON.parse(event.payload); } catch { return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); }
+    try {
-    if (!payload || typeof payload !== 'object') return event.kind;
+        payload = JSON.parse(event.payload);
-    if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) {
+    } catch {
-        const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : event.kind === 'heartbeat' ? 'Heartbeat' : 'Round changed';
+        return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
-        return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`;
+    }
-    }
+    if (!payload || typeof payload !== 'object') return event.kind;
-    if (event.kind === 'manual-sign-off') return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
+    if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) {
-    if (event.kind === 'health' || event.kind === 'status-sample') return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
+        const label =
+            event.kind === 'game_start'
+                ? 'Game started'
+                : event.kind === 'game_end'
+                  ? 'Game ended'
+                  : event.kind === 'heartbeat'
+                    ? 'Heartbeat'
+                    : 'Round changed';
+        return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`;
+    }
+    if (event.kind === 'manual-sign-off')
+        return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
+    if (event.kind === 'health' || event.kind === 'status-sample')
+        return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
     return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
 }

src/lib/beta-plan-summary.ts

@@ -1,34 +1,66 @@
 import type { BetaPlan, BetaRequirement } from './beta-plan.js';
 
 const count = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`;
-const duration = (minutes: number) => minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
+const duration = (minutes: number) =>
+    minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
 export const presetDescriptions: Record<string, string> = {
     'Quick smoke test': 'A quick check that the server starts and stays healthy.',
     'Gameplay validation': 'Prove the build through real matches and populated playtime.',
     'Long soak': 'Exercise stability and participation over a longer test.'
 };
 export function requirementSummary(r: BetaRequirement): string {
-    const players = r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
+    const players =
+        r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
     switch (r.kind) {
-        case 'round': return `Reach round ${r.target}${players}`;
+        case 'round':
-        case 'matches': return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
+            return `Reach round ${r.target}${players}`;
-        case 'populated-minutes': return `Accumulate ${duration(r.target)}${players}`;
+        case 'matches':
-        case 'player-hours': return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
+            return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
-        case 'population': return `Reach ${count(r.target, 'player')} online`;
+        case 'populated-minutes':
-        case 'sustained-population': return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
+            return `Accumulate ${duration(r.target)}${players}`;
-        case 'healthy-minutes': return `Stay continuously healthy for ${duration(r.target)}`;
+        case 'player-hours':
-        case 'failure-budget': return r.target === 0 ? 'Allow no health outages' : `Allow at most ${count(r.target, 'health outage')}`;
+            return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
-        case 'recovery': return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
+        case 'population':
-        case 'manual': return r.label;
+            return `Reach ${count(r.target, 'player')} online`;
+        case 'sustained-population':
+            return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
+        case 'healthy-minutes':
+            return `Stay continuously healthy for ${duration(r.target)}`;
+        case 'failure-budget':
+            return r.target === 0
+                ? 'Allow no health outages'
+                : `Allow at most ${count(r.target, 'health outage')}`;
+        case 'recovery':
+            return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
+        case 'manual':
+            return r.label;
     }
 }
 export function coverageSummary(plan: BetaPlan): string {
-    return plan.scope === 'all' ? 'Every selected server' : plan.scope === 'quorum' ? `At least ${count(plan.quorum, 'selected server')}`
+    return plan.scope === 'all'
-        : plan.scope === 'map' ? 'One server per selected map' : plan.scope === 'game' ? 'One server per selected game' : 'One server per game / map';
+        ? 'Every selected server'
+        : plan.scope === 'quorum'
+          ? `At least ${count(plan.quorum, 'selected server')}`
+          : plan.scope === 'map'
+            ? 'One server per selected map'
+            : plan.scope === 'game'
+              ? 'One server per selected game'
+              : 'One server per game / map';
 }
 export function needsGameTelemetry(r: BetaRequirement): boolean {
-    return ['round', 'matches', 'population', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind);
+    return [
+        'round',
+        'matches',
+        'population',
+        'populated-minutes',
+        'player-hours',
+        'sustained-population'
+    ].includes(r.kind);
 }
 export function requirementCategory(r: BetaRequirement): string {
-    return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget'].includes(r.kind) ? 'Continuous check' : 'Milestone';
+    return r.kind === 'manual'
+        ? 'Tester sign-off'
+        : ['healthy-minutes', 'failure-budget'].includes(r.kind)
+          ? 'Continuous check'
+          : 'Milestone';
 }

src/lib/beta-plan.ts

@@ -1,6 +1,16 @@
-export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population',
+export const requirementKinds = [
-    'sustained-population', 'healthy-minutes', 'failure-budget', 'recovery', 'manual'] as const;
+    'round',
-export type RequirementKind = typeof requirementKinds[number];
+    'matches',
+    'populated-minutes',
+    'player-hours',
+    'population',
+    'sustained-population',
+    'healthy-minutes',
+    'failure-budget',
+    'recovery',
+    'manual'
+] as const;
+export type RequirementKind = (typeof requirementKinds)[number];
 export interface BetaRequirement {
     id: string;
     kind: RequirementKind;
@@ -21,90 +31,306 @@ export interface BetaPlan {
     waveSize: number; // 0 deploys all selected servers; otherwise pass each wave before expanding.
 }
 export const requirementLabels: Record<RequirementKind, string> = {
-    round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes',
+    round: 'Reach round',
-    'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)',
+    matches: 'Complete qualifying matches',
-    'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages',
+    'populated-minutes': 'Populated minutes',
-    recovery: 'Successful recovery cycles', manual: 'Tester checklist'
+    'player-hours': 'Player-hours',
+    population: 'Reach player count',
+    'sustained-population': 'Sustained population (minutes)',
+    'healthy-minutes': 'Continuous healthy minutes',
+    'failure-budget': 'Maximum health outages',
+    recovery: 'Successful recovery cycles',
+    manual: 'Tester checklist'
 };
 export function requirement(kind: RequirementKind, id: string): BetaRequirement {
-    return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
+    return {
-        minRound: 0, minPlayers: 1, minMinutes: 0 };
+        id,
+        kind,
+        label: requirementLabels[kind],
+        target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
+        minRound: 0,
+        minPlayers: 1,
+        minMinutes: 0
+    };
 }
 export function defaultPlan(): BetaPlan {
-    return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1,
+    return {
-        timeoutHours: 24, failureAction: 'fail', rollbackRestart: 'when-empty', waveSize: 0 };
+        version: 1,
+        requirements: [requirement('healthy-minutes', 'health')],
+        scope: 'all',
+        quorum: 1,
+        timeoutHours: 24,
+        failureAction: 'fail',
+        rollbackRestart: 'when-empty',
+        waveSize: 0
+    };
 }
 export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [
     { name: 'Quick smoke test', plan: defaultPlan() },
-    { name: 'Gameplay validation', plan: { ...defaultPlan(), requirements: [
+    {
+        name: 'Gameplay validation',
+        plan: {
+            ...defaultPlan(),
+            requirements: [
                 { ...requirement('matches', 'matches'), target: 2, minRound: 25 },
-        { ...requirement('populated-minutes', 'playtime'), target: 90 }, requirement('failure-budget', 'outages')
+                { ...requirement('populated-minutes', 'playtime'), target: 90 },
-    ] } },
+                requirement('failure-budget', 'outages')
-    { name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [
+            ]
+        }
+    },
+    {
+        name: 'Long soak',
+        plan: {
+            ...defaultPlan(),
+            timeoutHours: 72,
+            requirements: [
                 { ...requirement('healthy-minutes', 'health'), target: 360 },
-        { ...requirement('player-hours', 'participation'), target: 10 }, requirement('failure-budget', 'outages')
+                { ...requirement('player-hours', 'participation'), target: 10 },
-    ] } }
+                requirement('failure-budget', 'outages')
+            ]
+        }
+    }
 ];
 
 /** Strict allow-list validation at every boundary (including stored presets). */
 export function parseBetaPlan(value: unknown): BetaPlan {
-    if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Test plan must be an object');
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('Test plan must be an object');
     const p = value as BetaPlan;
-    if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) throw new Error('Invalid test plan version or scope');
+    if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope))
-    if (!['fail', 'pause', 'rollback'].includes(p.failureAction) || !['manual', 'when-empty'].includes(p.rollbackRestart)) throw new Error('Invalid failure policy');
+        throw new Error('Invalid test plan version or scope');
-    for (const [key, min, max] of [['quorum', 1, 1000], ['timeoutHours', 0.01, 8760], ['waveSize', 0, 1000]] as const) {
+    if (
-        if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) throw new Error(`Invalid ${key}`);
+        !['fail', 'pause', 'rollback'].includes(p.failureAction) ||
+        !['manual', 'when-empty'].includes(p.rollbackRestart)
+    )
+        throw new Error('Invalid failure policy');
+    for (const [key, min, max] of [
+        ['quorum', 1, 1000],
+        ['timeoutHours', 0.01, 8760],
+        ['waveSize', 0, 1000]
+    ] as const) {
+        if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max)
+            throw new Error(`Invalid ${key}`);
     }
-    if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) throw new Error('Quorum and wave size must be integers');
+    if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize))
-    if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) throw new Error('Choose 1–30 requirements');
+        throw new Error('Quorum and wave size must be integers');
+    if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30)
+        throw new Error('Choose 1–30 requirements');
     const ids = new Set<string>();
-    const requirements = p.requirements.map(r => {
+    const requirements = p.requirements.map((r) => {
-        if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) throw new Error('Unknown requirement');
+        if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind))
-        if (typeof r.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || r.id in Object.prototype || ids.has(r.id)) throw new Error('Requirement IDs must be unique, non-reserved identifiers');
+            throw new Error('Unknown requirement');
+        if (
+            typeof r.id !== 'string' ||
+            !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) ||
+            r.id in Object.prototype ||
+            ids.has(r.id)
+        )
+            throw new Error('Requirement IDs must be unique, non-reserved identifiers');
         ids.add(r.id);
-        if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) throw new Error('Requirement needs a label (up to 200 characters)');
+        if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200)
+            throw new Error('Requirement needs a label (up to 200 characters)');
         for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) {
-            if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`);
+            if (
+                typeof r[key] !== 'number' ||
+                !Number.isFinite(r[key]) ||
+                r[key] < 0 ||
+                r[key] > 1_000_000
+            )
+                throw new Error(`Invalid requirement ${key}`);
         }
-        if (r.kind !== 'failure-budget' && r.target <= 0) throw new Error('Requirement target must be positive');
+        if (r.kind !== 'failure-budget' && r.target <= 0)
-        if (['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers');
+            throw new Error('Requirement target must be positive');
-        if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers');
+        if (
-        if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item');
+            ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(
-        return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound,
+                r.kind
-            minPlayers: r.minPlayers, minMinutes: r.minMinutes };
+            ) &&
+            !Number.isInteger(r.target)
+        )
+            throw new Error('Count targets must be integers');
+        if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers))
+            throw new Error('Round and player qualifiers must be integers');
+        if (r.kind === 'manual' && r.target !== 1)
+            throw new Error('Add a separate requirement for each checklist item');
+        return {
+            id: r.id,
+            kind: r.kind,
+            label: r.label.trim(),
+            target: r.target,
+            minRound: r.minRound,
+            minPlayers: r.minPlayers,
+            minMinutes: r.minMinutes
+        };
     });
-    return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours,
+    return {
-        failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize };
+        version: 1,
+        requirements,
+        scope: p.scope,
+        quorum: p.quorum,
+        timeoutHours: p.timeoutHours,
+        failureAction: p.failureAction,
+        rollbackRestart: p.rollbackRestart,
+        waveSize: p.waveSize
+    };
 }
 
-export interface RequirementProgress { id: string; label: string; current: number; target: number; passed: boolean; failed: boolean; detail: string }
+export interface RequirementProgress {
+    id: string;
+    label: string;
+    current: number;
+    target: number;
+    passed: boolean;
+    failed: boolean;
+    detail: string;
+}
 export interface BetaTestView {
     plan: BetaPlan;
     outcome: string;
     reason: string | null;
-    servers: Array<{ server: string; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null;
+    servers: Array<{
-        observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>;
+        server: string;
-    evidence: Array<{ id: number; server: string; received_at: string; kind: string; payload: string }>;
+        server_deleted_at: string | null;
+        phase: number;
+        passed: boolean;
+        progress: RequirementProgress[];
+        observingAt?: string | null;
+        observation?: {
+            lastGameplayAt: string | null;
+            lastEvent: string | null;
+            players: number | null;
+            round: number | null;
+            matchStarted: boolean;
+            gameEnded: boolean;
+            healthy: boolean | null;
+            healthCheckedAt: string | null;
+        };
+    }>;
+    evidence: Array<{
+        id: number;
+        server: string;
+        server_deleted_at: string | null;
+        received_at: string;
+        kind: string;
+        payload: string;
+    }>;
 }
 
-export interface BetaPlanIssue { field: string; message: string; requirementId?: string }
+export interface BetaPlanIssue {
+    field: string;
+    message: string;
+    requirementId?: string;
+}
 export function betaPlanIssues(plan: BetaPlan, serverCount?: number): BetaPlanIssue[] {
     const issues: BetaPlanIssue[] = [];
-    const check = (field: string, value: number, min: number, max: number, integer: boolean, message: string, requirementId?: string) => {
+    const check = (
-        if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) issues.push({ field, message, requirementId });
+        field: string,
+        value: number,
+        min: number,
+        max: number,
+        integer: boolean,
+        message: string,
+        requirementId?: string
+    ) => {
+        if (
+            typeof value !== 'number' ||
+            !Number.isFinite(value) ||
+            value < min ||
+            value > max ||
+            (integer && !Number.isInteger(value))
+        )
+            issues.push({ field, message, requirementId });
     };
-    check('timeoutHours', plan.timeoutHours, .01, 8760, false, 'Enter a testing window between 0.01 and 8,760 hours.');
+    check(
-    check('waveSize', plan.waveSize, 0, 1000, true, 'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.');
+        'timeoutHours',
-    if (plan.scope === 'quorum') check('quorum', plan.quorum, 1, serverCount ?? 1000, true, `Choose between 1 and ${serverCount ?? 1000} qualifying servers.`);
+        plan.timeoutHours,
-    if (!plan.requirements.length) issues.push({ field: 'requirements', message: 'Add at least one requirement to continue.' });
+        0.01,
+        8760,
+        false,
+        'Enter a testing window between 0.01 and 8,760 hours.'
+    );
+    check(
+        'waveSize',
+        plan.waveSize,
+        0,
+        1000,
+        true,
+        'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.'
+    );
+    if (plan.scope === 'quorum')
+        check(
+            'quorum',
+            plan.quorum,
+            1,
+            serverCount ?? 1000,
+            true,
+            `Choose between 1 and ${serverCount ?? 1000} qualifying servers.`
+        );
+    if (!plan.requirements.length)
+        issues.push({
+            field: 'requirements',
+            message: 'Add at least one requirement to continue.'
+        });
     for (const r of plan.requirements) {
-        if (!r.label?.trim()) issues.push({ field: `${r.id}-label`, message: 'Describe what this requirement checks.', requirementId: r.id });
+        if (!r.label?.trim())
-        const integral = ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind);
+            issues.push({
-        check(`${r.id}-target`, r.target, r.kind === 'failure-budget' ? 0 : .000001, 1_000_000, integral, `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`, r.id);
+                field: `${r.id}-label`,
-        if (['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)) check(`${r.id}-minPlayers`, r.minPlayers, 1, 1_000_000, true, 'Enter at least one player, using a whole number.', r.id);
+                message: 'Describe what this requirement checks.',
+                requirementId: r.id
+            });
+        const integral = [
+            'round',
+            'matches',
+            'population',
+            'failure-budget',
+            'recovery',
+            'manual'
+        ].includes(r.kind);
+        check(
+            `${r.id}-target`,
+            r.target,
+            r.kind === 'failure-budget' ? 0 : 0.000001,
+            1_000_000,
+            integral,
+            `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`,
+            r.id
+        );
+        if (
+            [
+                'round',
+                'matches',
+                'populated-minutes',
+                'player-hours',
+                'sustained-population'
+            ].includes(r.kind)
+        )
+            check(
+                `${r.id}-minPlayers`,
+                r.minPlayers,
+                1,
+                1_000_000,
+                true,
+                'Enter at least one player, using a whole number.',
+                r.id
+            );
         if (r.kind === 'matches') {
-            check(`${r.id}-minRound`, r.minRound, 0, 1_000_000, true, 'Enter a whole round number; 0 allows any round.', r.id);
+            check(
-            check(`${r.id}-minMinutes`, r.minMinutes, 0, 1_000_000, false, 'Enter a duration of zero or more minutes.', r.id);
+                `${r.id}-minRound`,
+                r.minRound,
+                0,
+                1_000_000,
+                true,
+                'Enter a whole round number; 0 allows any round.',
+                r.id
+            );
+            check(
+                `${r.id}-minMinutes`,
+                r.minMinutes,
+                0,
+                1_000_000,
+                false,
+                'Enter a duration of zero or more minutes.',
+                r.id
+            );
         }
     }
     return issues;

src/lib/beta-promotion.ts

@@ -3,12 +3,24 @@ export interface BetaPromotionPreview {
     token: string;
     createdAt: string;
     files: Array<{
-        id: number; name: string; kind: string; path: string; versionId: number;
+        id: number;
-        version: number; release_version: string | null; sha256: string;
+        name: string;
-        scope: string; servers: string[];
+        kind: string;
+        path: string;
+        versionId: number;
+        version: number;
+        release_version: string | null;
+        sha256: string;
+        scope: string;
+        servers: string[];
     }>;
     servers: Array<{ name: string; tested: boolean; game: string | null; map: string | null }>;
-    configWrites: Array<{ server: string; path: string; sha256: string | null; error: string | null }>;
+    configWrites: Array<{
+        server: string;
+        path: string;
+        sha256: string | null;
+        error: string | null;
+    }>;
     blockers: string[];
     restartPolicy: 'additional-when-empty';
 }

src/lib/beta-revision.ts

@@ -0,0 +1,23 @@
+import type { BetaTestView } from './beta-plan.js';
+
+export interface BetaRevisionFile {
+    script_id: number;
+    name: string;
+    kind: string;
+    path: string;
+    version_id: number;
+    version: number;
+    release_version: string | null;
+    sha256: string;
+}
+export interface BetaRevision {
+    revision: number;
+    started_at: string;
+    notes: string | null;
+    files: BetaRevisionFile[];
+    results: {
+        state: string;
+        test: BetaTestView | null;
+        servers: Array<{ server: string; stage: string; error: string | null }>;
+    } | null;
+}

src/lib/capacity.ts

@@ -0,0 +1,82 @@
+import type { CapacityCalibration, HostReadiness } from './hosts.js';
+
+export type CapacityRange = '1h' | '24h' | '7d';
+export type CapacityGame = 'all' | 't5' | 't6';
+export interface CapacityQuery {
+    range: CapacityRange;
+    hostId: string | null;
+    game: CapacityGame;
+}
+export function parseCapacityQuery(params: URLSearchParams): CapacityQuery {
+    const range = params.get('range');
+    const game = params.get('game');
+    return {
+        range: range === '1h' || range === '7d' ? range : '24h',
+        game: game === 't5' || game === 't6' ? game : 'all',
+        hostId: params.get('host')?.trim().slice(0, 100) || null
+    };
+}
+
+export interface CapacityHistoryPoint {
+    at: number;
+    cpuCoresAvailable: number | null;
+    memoryBytesAvailable: number | null;
+    reportingHosts: number;
+    expectedHosts: number;
+}
+export interface CapacityHistory {
+    range: CapacityRange;
+    startAt: number;
+    endAt: number;
+    bucketMs: number;
+    points: CapacityHistoryPoint[];
+    temperatures: TemperatureHistorySeries[];
+}
+export interface TemperatureHistorySeries {
+    id: string;
+    hostId: string;
+    label: string;
+    kind: 'cpu' | 'nvme' | 'chipset';
+    points: { at: number; celsius: number | null }[];
+}
+export interface CapacityHost {
+    id: string;
+    name: string;
+    readiness: HostReadiness;
+    connected: boolean;
+    updatedAt: string | null;
+    eligible: boolean;
+    games: ('t5' | 't6')[];
+    reasons: string[];
+    cpuCoresAvailable: number | null;
+    memoryBytesAvailable: number | null;
+    portsAvailable: number;
+    totalPorts: number;
+    estimatedServers: number | null;
+    limits: { cpu: number | null; memory: number | null; ports: number };
+    limitingResource: 'CPU' | 'Memory' | 'Ports' | null;
+    calibration: CapacityCalibration;
+}
+export interface CapacityCurrent {
+    updatedAt: string | null;
+    summary: {
+        estimatedServers: number | null;
+        cpuCoresAvailable: number | null;
+        memoryBytesAvailable: number | null;
+        portsAvailable: number | null;
+        eligibleHosts: number;
+        totalHosts: number;
+        startupSlots: number | null;
+        startupLimit: number | null;
+        activeStartups: number | null;
+        queued: number | null;
+    };
+    hosts: CapacityHost[];
+    hostOptions: { id: string; name: string }[];
+    warnings: string[];
+}
+export interface CapacityData {
+    query: CapacityQuery;
+    current: CapacityCurrent;
+    history: CapacityHistory;
+}

src/lib/config-workbench.ts

@@ -27,33 +27,45 @@ export interface ConfigInspection {
     reason?: string;
 }
 
-export interface ConfigServer { name: string; game: string | null; map: string | null; }
+export interface ConfigServer {
+    name: string;
+    game: string | null;
+    map: string | null;
+}
 
 /** The same lowercase `set` grammar that the overlay composer accepts. */
 export function configLines(text: string) {
     return text.split(/\r?\n/).flatMap((line, index) => {
         const match = /^(\s*set\s+)(\S+)(\s+)(.*)$/.exec(line);
-        return match ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }] : [];
+        return match
+            ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }]
+            : [];
     });
 }
 
 export function changeConfigValue(text: string, index: number, value: string): string {
     // Keep every separator, comment and unrelated line byte-for-byte intact.
     const parts = text.split(/(\r?\n)/);
-    const entry = configLines(text).find(line => line.index === index);
+    const entry = configLines(text).find((line) => line.index === index);
     if (entry) parts[index * 2] = entry.prefix + value.replace(/[\r\n]/g, '');
     return parts.join('');
 }
 
 /** Add one assignment without changing existing commands or line endings. */
 export function addConfigSetting(text: string, key: string, value: string): string {
-    key = key.trim(); value = value.trim();
+    key = key.trim();
-    if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) throw new Error('Use a setting name containing letters, numbers, and underscores.');
+    value = value.trim();
-    if (!value || /[\r\n]/.test(value) || value.startsWith('//')) throw new Error('Enter a value on one line.');
+    if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key))
-    if (configLines(text).some(entry => entry.key.toLowerCase() === key.toLowerCase())) throw new Error('This setting already exists. Find it above to edit its value.');
+        throw new Error('Use a setting name containing letters, numbers, and underscores.');
+    if (!value || /[\r\n]/.test(value) || value.startsWith('//'))
+        throw new Error('Enter a value on one line.');
+    if (configLines(text).some((entry) => entry.key.toLowerCase() === key.toLowerCase()))
+        throw new Error('This setting already exists. Find it above to edit its value.');
     const newline = text.includes('\r\n') ? '\r\n' : '\n';
     const formatted = /\s/.test(value) && !value.startsWith('"') ? JSON.stringify(value) : value;
-    return text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline;
+    return (
+        text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline
+    );
 }
 
 export function configWarnings(text: string, overlay: boolean): string[] {
@@ -61,11 +73,16 @@ export function configWarnings(text: string, overlay: boolean): string[] {
     const seen = new Map<string, number>();
     for (const entry of configLines(text)) {
         const key = entry.key.toLowerCase();
-        if (seen.has(key)) warnings.push(`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`);
+        if (seen.has(key))
+            warnings.push(
+                `Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`
+            );
         seen.set(key, entry.index + 1);
-        if (!entry.value.trim() || entry.value.trim().startsWith('//')) warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
+        if (!entry.value.trim() || entry.value.trim().startsWith('//'))
+            warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
     }
-    if (overlay) text.split(/\r?\n/).forEach((line, index) => {
+    if (overlay)
+        text.split(/\r?\n/).forEach((line, index) => {
             if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) {
                 warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`);
             }

src/lib/deployment-plan.ts

@@ -6,11 +6,14 @@ export interface LaunchFileSummary {
     path: string;
     versionId: number | null;
     version: number | null;
+    /** Missing on snapshots recorded before publisher release versions were retained. */
+    releaseVersion?: string | null;
     sha256: string;
     size: number;
 }
 
 export interface LaunchRecipeSummary {
+    native?: import('./hosts.js').NativePlacement;
     game: ZombieGame;
     mapSlug: string;
     mapName: string;
@@ -34,6 +37,18 @@ export interface BlueprintSummary {
 
 export type CollectionMode = 'fill' | 'exact';
 
+export interface LaunchAvailability {
+    updatedAt: string | null;
+    requested: number;
+    estimatedAvailable: number | null;
+    portsAvailable: number | null;
+    cpuCoresAvailable: number | null;
+    memoryBytesAvailable: number | null;
+    startupSlots: number | null;
+    startupLimit: number | null;
+    queued: number | null;
+}
+
 export interface CollectionPreview {
     id: string;
     expiresAt: string;
@@ -42,10 +57,24 @@ export interface CollectionPreview {
     mode: CollectionMode;
     retainedServers: string[];
     removals: { name: string; reason: string }[];
-    entries: { count: number; existing: number; missing: number; existingServers: string[];
+    entries: {
-        coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
+        count: number;
-    servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
+        existing: number;
+        missing: number;
+        existingServers: string[];
+        coverage: { name: string; health: string; href: string }[];
+        recipe: LaunchRecipeSummary;
+    }[];
+    servers: {
+        native?: import('./hosts.js').NativePlacement;
+        entryIndex: number;
+        name: string | null;
+        port: number | null;
+        hasSavedKey: boolean;
+    }[];
     blockers: string[];
+    warnings?: string[];
+    availability?: LaunchAvailability;
     ready: boolean;
 }
 
@@ -53,9 +82,13 @@ export interface CollectionSummary {
     id: string;
     name: string;
     created_at: string;
+    cancelled_at: string | null;
     total: number;
     done: number;
     failed: number;
+    cancelled: number;
+    cleanupActive: number;
+    cleanupFailed: number;
     active: number;
     queued: number;
     provisioning: number;
@@ -65,10 +98,35 @@ export interface CollectionSummary {
 
 export interface CollectionDetail {
     collection: CollectionSummary;
+    canCancel: boolean;
     retainedServers: string[];
-    queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number };
+    retainedServerDeletedAt?: Record<string, string>;
-    servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string;
+    queue: {
-        started_at: string; finished_at: string | null; error: string | null; server_exists: number; queuePosition: number | null }[];
+        concurrency: number;
+        active: { id: string; server: string; state: string; stage: string }[];
+        queued: number;
+    };
+    servers: {
+        id: string;
+        server: string;
+        action: 'create' | 'destroy';
+        port: number | null;
+        map_slug: string | null;
+        state: string;
+        stage: string;
+        started_at: string;
+        finished_at: string | null;
+        error: string | null;
+        server_exists: number;
+        server_deleted_at?: string | null;
+        queuePosition: number | null;
+        cancellation: {
+            cleanupId: string;
+            state: string;
+            stage: string;
+            error: string | null;
+        } | null;
+    }[];
 }
 
 export interface LaunchPreview {

src/lib/discord.ts

@@ -0,0 +1,107 @@
+export type DiscordGame = 't5' | 't6';
+
+export interface DiscordSettings {
+    enabled: boolean;
+    guildId: string;
+    channels: Record<DiscordGame, string>;
+    excludedServers: string[];
+    showPlayers: boolean;
+    showAddresses: boolean;
+    leaderboardEnabled: boolean;
+}
+
+export const defaultDiscordSettings: DiscordSettings = {
+    enabled: false,
+    guildId: '',
+    channels: { t5: '', t6: '' },
+    excludedServers: [],
+    showPlayers: false,
+    showAddresses: false,
+    leaderboardEnabled: false
+};
+
+export const discordSnowflake = (value: unknown): value is string =>
+    typeof value === 'string' && /^[1-9][0-9]{16,19}$/.test(value);
+
+export function validateDiscordSettings(value: unknown): DiscordSettings {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('Provide Discord settings.');
+    const input = value as Record<string, unknown>;
+    for (const key of ['enabled', 'showPlayers', 'showAddresses', 'leaderboardEnabled']) {
+        if (typeof input[key] !== 'boolean') throw new Error(`Invalid ${key} setting.`);
+    }
+    if (typeof input.guildId !== 'string' || (input.guildId && !discordSnowflake(input.guildId)))
+        throw new Error('Provide a valid Discord server ID.');
+    if (!input.channels || typeof input.channels !== 'object' || Array.isArray(input.channels))
+        throw new Error('Provide Discord channels for each game.');
+    const channels = input.channels as Record<string, unknown>;
+    for (const game of ['t5', 't6'] as const) {
+        if (
+            typeof channels[game] !== 'string' ||
+            (channels[game] && !discordSnowflake(channels[game]))
+        )
+            throw new Error('Channel IDs must be valid Discord IDs or empty.');
+    }
+    if (
+        !Array.isArray(input.excludedServers) ||
+        input.excludedServers.length > 100 ||
+        input.excludedServers.some(
+            (name) => typeof name !== 'string' || !/^[A-Za-z0-9_.-]{1,128}$/.test(name)
+        )
+    )
+        throw new Error('Choose up to 100 servers to exclude from Discord.');
+    if (input.enabled && (!input.guildId || (!channels.t5 && !channels.t6)))
+        throw new Error('Choose a Discord server and at least one board channel before enabling.');
+    return {
+        enabled: input.enabled as boolean,
+        guildId: input.guildId,
+        channels: { t5: channels.t5 as string, t6: channels.t6 as string },
+        excludedServers: [...new Set(input.excludedServers as string[])].sort(),
+        showPlayers: input.showPlayers as boolean,
+        showAddresses: input.showAddresses as boolean,
+        leaderboardEnabled: input.leaderboardEnabled as boolean
+    };
+}
+
+export interface DiscordPublicServer {
+    name: string;
+    game: DiscordGame;
+    label: string;
+    map: string;
+    mapKey: string;
+    state: 'empty' | 'in-progress' | 'full' | 'starting' | 'offline' | 'stale' | 'unknown';
+    players: number | null;
+    capacity: number | null;
+    round: number | null;
+    observedAt: string | null;
+    playerNames: string[];
+    connection: string | null;
+}
+
+export interface DiscordMessage {
+    content?: string;
+    flags?: number;
+    components?: Record<string, unknown>[];
+    allowed_mentions: { parse: string[] };
+}
+
+export interface DiscordDeliveryStatus {
+    lastAttemptAt: string | null;
+    lastSuccessAt: string | null;
+    lastError: string | null;
+    boards: { game: DiscordGame; channelId: string; messageCount: number }[];
+    activeSubscriptions: number;
+    pendingNotifications: number;
+}
+
+export interface DiscordAdminState {
+    settings: DiscordSettings;
+    credentials: {
+        applicationId: string;
+        publicKeyConfigured: boolean;
+        botTokenConfigured: boolean;
+    };
+    status: DiscordDeliveryStatus;
+    servers: { name: string; game: DiscordGame; label: string; map: string }[];
+    preview: Record<DiscordGame, DiscordPublicServer[]>;
+}

src/lib/fleet-overview.ts

@@ -7,6 +7,12 @@ import type { Health, RosterPlayer } from './ui/stores/servers.js';
 import { mapTitle, serverTitle } from './ui/server-labels.js';
 
 export interface FleetServer {
+    plutonium?: import('./plutonium.js').PlutoniumVersion;
+    host_id?: string | null;
+    host?: Pick<
+        import('./hosts.js').Host,
+        'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat'
+    > | null;
     container_state?: string | null;
     container_checked_at?: string | null;
     last_checked_at?: string | null;
@@ -20,6 +26,8 @@ export interface FleetServer {
     map_slug?: string | null;
     game_mode?: string | null;
     last_players: number | null;
+    capacity?: number | null;
+    capacity_observed_at?: string | null;
     last_map: string | null;
     last_gametype: string | null;
     last_seen_at: string | null;
@@ -37,61 +45,132 @@ export interface FleetOperation {
     key: string;
     group: string;
     server: string;
-    kind: 'campaign' | 'deployment' | 'beta' | 'recovery';
+    kind: 'campaign' | 'deployment' | 'beta' | 'recovery' | 'maintenance';
     label: string;
     detail: string;
     href: string;
     state: 'active' | 'attention' | 'done';
     startedAt: string;
+    activityLabel?: string;
+    resultState?: string;
 }
 
 export type FleetFilter = 'all' | 'occupied' | 'empty' | 'attention' | 'testing' | 'pinned';
 export type FleetSort = 'name' | 'players' | 'round' | 'duration';
 
-export function freshServer(server: FleetServer, now: number): boolean {
+export function freshServer(
-    return server.health === 'online' && !server.last_error && freshStatusTimestamp(server.last_seen_at, now);
+    server: Pick<FleetServer, 'health' | 'last_error' | 'last_seen_at'>,
+    now: number
+): boolean {
+    return (
+        server.health === 'online' &&
+        !server.last_error &&
+        freshStatusTimestamp(server.last_seen_at, now)
+    );
 }
 
-export function fleetIdentity(server: FleetServer) {
+export function fleetIdentity(
+    server: Pick<FleetServer, 'name' | 'game' | 'map' | 'map_slug' | 'game_mode' | 'last_map'>
+) {
     const parsed = parseServerName(server.name);
     const game = server.game ?? parsed?.game ?? '';
     const mode = server.game_mode ?? parsed?.gameMode;
-    const selection = !mode || mode === 'zombie'
+    const selection =
-        ? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '') : undefined;
+        !mode || mode === 'zombie'
+            ? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '')
+            : undefined;
     const map = server.last_map ?? server.map;
     // A live map change must not keep the registered map's launch shortcut.
-    const matchingSelection = selection && (!map || selection.canonical === map) ? selection : undefined;
+    const matchingSelection =
+        selection && (!map || selection.canonical === map) ? selection : undefined;
     return {
         game,
         title: serverTitle(server.name),
         map: matchingSelection?.friendly ?? mapTitle(map),
         group: `${game}:${matchingSelection?.slug ?? map ?? 'unknown'}`,
-        launch: matchingSelection && (game === 't5' || game === 't6')
+        launch:
-            ? { game: game as ZombieGame, slug: matchingSelection.slug } : null
+            matchingSelection && (game === 't5' || game === 't6')
+                ? { game: game as ZombieGame, slug: matchingSelection.slug }
+                : null
     };
 }
 
-export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
+export function currentRound(
-    if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
+    server: Pick<
-    if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
+        FleetServer,
-        || server.game_ended_at || (server.last_players ?? 0) === 0) return null;
+        | 'telemetry'
+        | 'telemetry_after'
+        | 'health'
+        | 'last_error'
+        | 'last_seen_at'
+        | 'last_players'
+        | 'active_since'
+        | 'game_started_at'
+        | 'game_ended_at'
+    >,
+    report: GameTelemetry | undefined,
+    now: number
+): number | null {
+    if (
+        server.telemetry &&
+        (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))
+    )
+        report = server.telemetry;
+    if (
+        !report ||
+        (report.event !== 'round_change' && report.event !== 'heartbeat') ||
+        !freshServer(server, now) ||
+        !server.active_since ||
+        server.game_ended_at ||
+        (server.last_players ?? 0) === 0
+    )
+        return null;
     const received = Date.parse(report.receivedAt);
-    return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at)
+    return freshTelemetry(report, now, server.telemetry_after) &&
-        && received >= Date.parse(server.active_since) ? report.round : null;
+        (!server.game_started_at || received >= Date.parse(server.game_started_at)) &&
+        received >= Date.parse(server.active_since)
+        ? report.round
+        : null;
 }
 
-export function matchesFleet(server: FleetServer, query: string, filter: FleetFilter, game: string,
+export function matchesFleet(
-    pinned: readonly string[], operations: readonly FleetOperation[], now: number): boolean {
+    server: FleetServer,
+    query: string,
+    filter: FleetFilter,
+    game: string,
+    pinned: readonly string[],
+    operations: readonly FleetOperation[],
+    now: number
+): boolean {
     const identity = fleetIdentity(server);
     if (game && identity.game !== game) return false;
     const fresh = freshServer(server, now);
     if (filter === 'occupied' && !(fresh && (server.last_players ?? 0) > 0)) return false;
     if (filter === 'empty' && !(fresh && server.last_players === 0)) return false;
-    if (filter === 'attention' && !(server.health === 'offline' || server.health === 'stale'
+    if (
-        || (server.health === 'online' && !fresh) || operations.some(op => op.server === server.name && op.state === 'attention'))) return false;
+        filter === 'attention' &&
+        !(
+            server.health === 'offline' ||
+            server.health === 'stale' ||
+            (server.health === 'online' && !fresh) ||
+            operations.some((op) => op.server === server.name && op.state === 'attention')
+        )
+    )
+        return false;
     if (filter === 'testing' && !server.held.length) return false;
     if (filter === 'pinned' && !pinned.includes(server.name)) return false;
-    const text = [server.name, identity.title, identity.map, identity.game,
+    const text = [
-        ...(server.roster ?? []).map(player => player.name)].join(' ').toLocaleLowerCase();
+        server.name,
-    return query.trim().toLocaleLowerCase().split(/\s+/).every(term => text.includes(term));
+        identity.title,
+        identity.map,
+        identity.game,
+        ...(server.roster ?? []).map((player) => player.name)
+    ]
+        .join(' ')
+        .toLocaleLowerCase();
+    return query
+        .trim()
+        .toLocaleLowerCase()
+        .split(/\s+/)
+        .every((term) => text.includes(term));
 }

src/lib/fleet-settings.ts

@@ -0,0 +1,358 @@
+export interface FleetSettings {
+    provisioningKeepDays: number;
+    actionKeepDays: number;
+    failureKeepDays: number;
+    maintenanceKeepDays: number;
+    recoveryKeepDays: number;
+    hostResourceKeepDays: number;
+    instanceResourceKeepDays: number;
+    metricsKeepDays: number;
+    telemetryKeepDays: number;
+    telemetryMaxPerServer: number;
+    telemetryMaxTotal: number;
+    matchKeepDays: number;
+    betaKeepDays: number;
+    fileContentKeepDays: number;
+    payloadKeepDays: number;
+    versionKeepDays: number;
+    versionKeepCount: number;
+    gameFileGraceDays: number;
+    gameFileKeepCount: number;
+    uploadKeepDays: number;
+    auditKeepDays: number;
+    historyKeepCount: number;
+
+    deployKeepDays: number;
+    observationKeepDays: number;
+    restartConcurrency: number;
+    createConcurrency: number;
+    destroyConcurrency: number;
+    restartConcurrencyPerHost: number;
+    createConcurrencyPerHost: number;
+    destroyConcurrencyPerHost: number;
+    lifecycleConcurrency: number;
+    rebuildConcurrency: number;
+    preparationConcurrency: number;
+}
+
+export const defaultFleetSettings: FleetSettings = {
+    provisioningKeepDays: 0,
+    actionKeepDays: 0,
+    failureKeepDays: 0,
+    maintenanceKeepDays: 30,
+    recoveryKeepDays: 0,
+    hostResourceKeepDays: 7,
+    instanceResourceKeepDays: 1,
+    metricsKeepDays: 31,
+    telemetryKeepDays: 7,
+    telemetryMaxPerServer: 200,
+    telemetryMaxTotal: 10000,
+    matchKeepDays: 0,
+    betaKeepDays: 0,
+    fileContentKeepDays: 30,
+    payloadKeepDays: 30,
+    versionKeepDays: 0,
+    versionKeepCount: 0,
+    gameFileGraceDays: 0,
+    gameFileKeepCount: 1,
+    uploadKeepDays: 0,
+    auditKeepDays: 0,
+    historyKeepCount: 0,
+
+    deployKeepDays: 7,
+    observationKeepDays: 1,
+    restartConcurrency: 1,
+    createConcurrency: 1,
+    destroyConcurrency: 1,
+    restartConcurrencyPerHost: 0,
+    createConcurrencyPerHost: 0,
+    destroyConcurrencyPerHost: 0,
+    lifecycleConcurrency: 0,
+    rebuildConcurrency: 0,
+    preparationConcurrency: 0
+};
+
+export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
+    provisioningKeepDays: 'Provisioning history retention',
+    actionKeepDays: 'Action operation history retention',
+    failureKeepDays: 'Resolved failure history retention',
+    maintenanceKeepDays: 'Maintenance history retention',
+    recoveryKeepDays: 'Resolved recovery incident retention',
+    hostResourceKeepDays: 'Host resource history retention',
+    instanceResourceKeepDays: 'Instance resource sample retention',
+    metricsKeepDays: 'Fleet metrics history retention',
+    telemetryKeepDays: 'Telemetry diagnostic history retention',
+    telemetryMaxPerServer: 'Maximum telemetry records per server',
+    telemetryMaxTotal: 'Maximum telemetry records overall',
+    matchKeepDays: 'Match history retention',
+    betaKeepDays: 'Completed beta history retention',
+    fileContentKeepDays: 'File diffs and rollback snapshot retention',
+    payloadKeepDays: 'Launch and command file retention',
+    versionKeepDays: 'Unused artifact version retention',
+    versionKeepCount: 'Minimum artifact versions to keep',
+    gameFileGraceDays: 'Unused game-file grace period',
+    gameFileKeepCount: 'Minimum game-file versions to keep',
+    uploadKeepDays: 'Abandoned upload retention',
+    auditKeepDays: 'Authentication audit history retention',
+    historyKeepCount: 'Minimum history records to keep',
+
+    deployKeepDays: 'Campaign history retention',
+    observationKeepDays: 'Routine operation history retention',
+    restartConcurrency: 'Concurrent restarts',
+    createConcurrency: 'Concurrent server startups',
+    destroyConcurrency: 'Concurrent server removals',
+    restartConcurrencyPerHost: 'Concurrent restarts per host',
+    createConcurrencyPerHost: 'Concurrent startups per host',
+    destroyConcurrencyPerHost: 'Concurrent server removals per host',
+    lifecycleConcurrency: 'Combined lifecycle limit',
+    rebuildConcurrency: 'Concurrent rebuilds',
+    preparationConcurrency: 'Hosts preparing game files simultaneously'
+};
+
+export function validateFleetSettings(input: unknown): FleetSettings {
+    if (!input || typeof input !== 'object' || Array.isArray(input))
+        throw new Error('Provide fleet settings.');
+    const values = input as Record<string, unknown>;
+    for (const key of Object.keys(values)) {
+        if (!Object.hasOwn(defaultFleetSettings, key)) throw new Error(`Unknown setting: ${key}`);
+    }
+    for (const key of Object.keys(defaultFleetSettings) as (keyof FleetSettings)[]) {
+        const value = values[key];
+        const minimum = settingMinimum(key);
+        if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
+            throw new Error(
+                `${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`
+            );
+        }
+        if (key.endsWith('Days') && value > 36500) {
+            throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
+        }
+    }
+    return Object.fromEntries(
+        Object.keys(defaultFleetSettings).map((key) => [key, values[key]])
+    ) as unknown as FleetSettings;
+}
+
+export const additionalConcurrencyKeys: (keyof FleetSettings)[] = [
+    'restartConcurrencyPerHost',
+    'createConcurrencyPerHost',
+    'destroyConcurrencyPerHost',
+    'lifecycleConcurrency',
+    'rebuildConcurrency',
+    'preparationConcurrency'
+];
+export const concurrencyKeys: (keyof FleetSettings)[] = [
+    'restartConcurrency',
+    'createConcurrency',
+    'destroyConcurrency',
+    ...additionalConcurrencyKeys
+];
+export const retentionKeys: (keyof FleetSettings)[] = [
+    'deployKeepDays',
+    'observationKeepDays',
+    'provisioningKeepDays',
+    'actionKeepDays',
+    'failureKeepDays',
+    'maintenanceKeepDays',
+    'recoveryKeepDays',
+    'hostResourceKeepDays',
+    'instanceResourceKeepDays',
+    'metricsKeepDays',
+    'telemetryKeepDays',
+    'telemetryMaxPerServer',
+    'telemetryMaxTotal',
+    'matchKeepDays',
+    'betaKeepDays',
+    'fileContentKeepDays',
+    'payloadKeepDays',
+    'versionKeepDays',
+    'versionKeepCount',
+    'gameFileGraceDays',
+    'gameFileKeepCount',
+    'uploadKeepDays',
+    'auditKeepDays',
+    'historyKeepCount'
+];
+export function settingMinimum(key: keyof FleetSettings): number {
+    return [
+        'restartConcurrency',
+        'createConcurrency',
+        'destroyConcurrency',
+        'gameFileKeepCount'
+    ].includes(key)
+        ? 1
+        : 0;
+}
+
+export interface SettingsGroup {
+    id: string;
+    title: string;
+    description: string;
+    keys: (keyof FleetSettings)[];
+}
+
+/** Presentation groups do not change the saved schema or the section PATCH API. */
+export const settingsGroups: SettingsGroup[] = [
+    {
+        id: 'concurrency',
+        title: 'Concurrency',
+        description: 'Limit simultaneous server operations and game-file preparation.',
+        keys: concurrencyKeys
+    },
+    {
+        id: 'operation-history',
+        title: 'Operation history',
+        description: 'Campaigns, server operations, maintenance, and routine checks.',
+        keys: [
+            'deployKeepDays',
+            'provisioningKeepDays',
+            'actionKeepDays',
+            'observationKeepDays',
+            'failureKeepDays',
+            'maintenanceKeepDays',
+            'recoveryKeepDays',
+            'betaKeepDays',
+            'historyKeepCount'
+        ]
+    },
+    {
+        id: 'matches-players',
+        title: 'Matches & players',
+        description: 'Saved matches, player records, and leaderboard history.',
+        keys: ['matchKeepDays']
+    },
+    {
+        id: 'telemetry-metrics',
+        title: 'Telemetry & metrics',
+        description: 'Game diagnostics, fleet activity, and host resource samples.',
+        keys: [
+            'telemetryKeepDays',
+            'telemetryMaxPerServer',
+            'telemetryMaxTotal',
+            'metricsKeepDays',
+            'hostResourceKeepDays',
+            'instanceResourceKeepDays'
+        ]
+    },
+    {
+        id: 'files-versions',
+        title: 'Files & versions',
+        description: 'File snapshots, saved versions, game files, and abandoned uploads.',
+        keys: [
+            'fileContentKeepDays',
+            'payloadKeepDays',
+            'versionKeepDays',
+            'versionKeepCount',
+            'gameFileGraceDays',
+            'gameFileKeepCount',
+            'uploadKeepDays'
+        ]
+    },
+    {
+        id: 'accounts-audit',
+        title: 'Accounts & audit',
+        description: 'Sign-in and account administration records.',
+        keys: ['auditKeepDays']
+    }
+];
+
+/** Zero has a different meaning for age limits, count caps, and minimums. */
+export function settingZeroMode(key: keyof FleetSettings): string | null {
+    if (settingMinimum(key) > 0) return null;
+    if (
+        additionalConcurrencyKeys.includes(key) ||
+        key === 'telemetryMaxPerServer' ||
+        key === 'telemetryMaxTotal'
+    )
+        return 'No limit';
+    if (key === 'versionKeepDays') return 'No age expiry';
+    if (key === 'gameFileGraceDays') return 'No delay';
+    if (key === 'versionKeepCount' || key === 'historyKeepCount') return 'No minimum';
+    return key.endsWith('Days') ? 'Forever' : null;
+}
+
+export function settingUnit(
+    key: keyof FleetSettings
+): 'days' | 'records' | 'versions' | 'operations' | 'hosts' {
+    if (key.endsWith('Days')) return 'days';
+    if (
+        key === 'telemetryMaxPerServer' ||
+        key === 'telemetryMaxTotal' ||
+        key === 'historyKeepCount'
+    )
+        return 'records';
+    if (key === 'versionKeepCount' || key === 'gameFileKeepCount') return 'versions';
+    return key === 'preparationConcurrency' ? 'hosts' : 'operations';
+}
+
+const settingNumbers = new Intl.NumberFormat('en-US', { maximumFractionDigits: 20 });
+export function settingDisplayValue(key: keyof FleetSettings, value: number): string {
+    if (!Number.isFinite(value)) return '—';
+    const zeroMode = settingZeroMode(key);
+    if (value === 0 && zeroMode) return zeroMode;
+    const unit = settingUnit(key);
+    return `${settingNumbers.format(value)} ${value === 1 ? unit.slice(0, -1) : unit}`;
+}
+
+export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
+    provisioningKeepDays:
+        'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.',
+    actionKeepDays:
+        'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.',
+    failureKeepDays:
+        'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.',
+    maintenanceKeepDays:
+        'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
+    recoveryKeepDays:
+        'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
+    hostResourceKeepDays: 'Host CPU, memory and temperature history.',
+    instanceResourceKeepDays:
+        'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
+    metricsKeepDays: 'Historical fleet occupancy and utilization samples.',
+    telemetryKeepDays:
+        'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.',
+    telemetryMaxPerServer: 'Count cap per server, in addition to the age limit.',
+    telemetryMaxTotal:
+        'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.',
+    matchKeepDays:
+        'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
+    betaKeepDays:
+        'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
+    fileContentKeepDays:
+        'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
+    payloadKeepDays:
+        'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.',
+    versionKeepDays:
+        'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.',
+    versionKeepCount:
+        'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
+    gameFileGraceDays:
+        'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.',
+    gameFileKeepCount:
+        'Keep this many recent published versions per game, in addition to every referenced or protected version.',
+    uploadKeepDays:
+        'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.',
+    auditKeepDays: 'Recorded authentication and account-administration events.',
+    historyKeepCount:
+        'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.',
+
+    restartConcurrency:
+        'Fleet recovery slots stay occupied until container and game health are verified.',
+    createConcurrency:
+        'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
+    destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.',
+    restartConcurrencyPerHost:
+        'Additional recovery limit on each host, including starts and rebuilds.',
+    createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.',
+    destroyConcurrencyPerHost:
+        'Additional removal limit on each host, including migration source removal.',
+    lifecycleConcurrency:
+        'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.',
+    rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.',
+    preparationConcurrency:
+        'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
+    deployKeepDays:
+        'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.',
+    observationKeepDays:
+        'Successful monitoring and file-read records. Failures and actions are retained.'
+};

src/lib/game-chat.ts

@@ -4,41 +4,95 @@ export interface ChatMessage {
     slot: number;
     player: string;
     message: string;
+    /** Omitted for public chat to preserve existing archived message identities. */
+    channel?: 'team';
 }
 
 export type ChatEntry =
     | { type: 'message'; message: ChatMessage }
     | { type: 'boundary'; event: 'start' | 'end' | 'reset'; time: string | null };
 
+export interface ChatContext {
+    /** Estimated from the final log write, never from the time chat is fetched. */
+    realTime: string | null;
+    matchElapsedMs: number | null;
+    round: number | null;
+}
+
+export type ContextualChatEntry = ChatEntry & {
+    context: ChatContext;
+    matchId?: number;
+    /** Stable archive identity, independent of estimated wall-clock timestamps. */
+    messageId?: string;
+};
+
+export interface ChatMatch {
+    id?: number;
+    startedAt: string;
+    endedAt: string | null;
+    lastReceivedAt: string;
+    status: 'active' | 'completed' | 'interrupted' | 'end-report-missing';
+    startObserved: boolean;
+}
+
+export interface ChatRoundReport {
+    receivedAt: string;
+    event: 'game_start' | 'round_change' | 'heartbeat' | 'game_end';
+    round: number | null;
+    playerCount: number;
+}
+
+function logTimestamp(line: string) {
+    return /^\s*(\d+:[0-5]\d)\s+/.exec(line);
+}
+
+function logSeconds(time: string): number {
+    const [minutes, seconds] = time.split(':').map(Number);
+    return minutes * 60 + seconds;
+}
+
 function plainText(value: string): string {
-    return value.replace(/\^[0-9]/g, '').replace(/[\x00-\x1f\x7f]/g, '').trim();
+    return (
+        value
+            .replace(/\^[0-9]/g, '')
+            // eslint-disable-next-line no-control-regex -- Strip control characters from displayed chat text.
+            .replace(/[\x00-\x1f\x7f]/g, '')
+            .trim()
+    );
 }
 
 /** Preserve log order: elapsed timestamps can reset between matches. */
 export function parseGameChat(lines: readonly string[]): ChatMessage[] {
-    return parseGameChatTimeline(lines).flatMap(entry => entry.type === 'message' ? [entry.message] : []);
+    return parseGameChatTimeline(lines).flatMap((entry) =>
+        entry.type === 'message' ? [entry.message] : []
+    );
 }
 
 /** Keep match boundaries alongside messages, including when the log clock resets. */
 export function parseGameChatTimeline(lines: readonly string[]): ChatEntry[] {
-    const entries: ChatEntry[] = [];
+    return indexedTimeline(lines).map(({ entry }) => entry);
+}
+
+function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineIndex: number }[] {
+    const entries: { entry: ChatEntry; lineIndex: number }[] = [];
     let previousSeconds: number | null = null;
     let ended = false;
 
-    for (const line of lines) {
+    for (const [lineIndex, line] of lines.entries()) {
-        const timestamp = /^\s*(\d+:[0-5]\d)\s+/.exec(line);
+        const push = (entry: ChatEntry) => entries.push({ entry, lineIndex });
+        const timestamp = logTimestamp(line);
         const time = timestamp?.[1] ?? null;
-        const seconds = time === null ? null : Number(time.split(':')[0]) * 60 + Number(time.split(':')[1]);
+        const seconds = time === null ? null : logSeconds(time);
         const event = line.slice(timestamp?.[0].length ?? 0).trimStart();
         if (/^InitGame(?:\s|:|$)/.test(event)) {
-            entries.push({ type: 'boundary', event: 'start', time });
+            push({ type: 'boundary', event: 'start', time });
             ended = false;
         } else if (/^(?:ExitLevel|ShutdownGame)(?:\s|:|$)/.test(event)) {
             // Engines may emit both end events for the same match.
-            if (!ended) entries.push({ type: 'boundary', event: 'end', time });
+            if (!ended) push({ type: 'boundary', event: 'end', time });
             ended = true;
         } else if (seconds !== null && previousSeconds !== null && seconds < previousSeconds) {
-            entries.push({ type: 'boundary', event: 'reset', time });
+            push({ type: 'boundary', event: 'reset', time });
             ended = false;
         }
         // Untimed lifecycle events also clear the previous match's clock.
@@ -46,12 +100,147 @@ export function parseGameChatTimeline(lines: readonly string[]): ChatEntry[] {
             previousSeconds = seconds;
         }
 
-        const match = /^\s*(\d+:[0-5]\d)\s+say;([^;]+);(\d+);([^;]*);(.*)$/.exec(line);
+        const match = /^\s*(\d+:[0-5]\d)\s+(say|sayteam);([^;]+);(\d+);([^;]*);(.*)$/.exec(line);
         if (!match) continue;
-        const message = plainText(match[5]);
+        const message = plainText(match[6]);
         if (!message) continue;
-        entries.push({ type: 'message', message: { time: match[1], guid: match[2], slot: Number(match[3]),
+        push({
-            player: plainText(match[4]) || 'Unknown player', message } });
+            type: 'message',
+            message: {
+                time: match[1],
+                guid: match[3],
+                slot: Number(match[4]),
+                player: plainText(match[5]) || 'Unknown player',
+                message,
+                ...(match[2] === 'sayteam' ? { channel: 'team' as const } : {})
+            }
+        });
     }
     return entries;
 }
+
+/**
+ * The engine clock is not the match clock. Anchor only its final uninterrupted
+ * segment to the file's last write, then join observed gameplay at that time.
+ * Resets have unknown downtime, so earlier segments cannot share this anchor.
+ */
+export function enrichGameChatTimeline(
+    lines: readonly string[],
+    options: {
+        mtime: string | null;
+        matches: readonly ChatMatch[];
+        reports: readonly ChatRoundReport[];
+    }
+): ContextualChatEntry[] {
+    let segmentStart = 0;
+    let previousSeconds: number | null = null;
+    let lastLine = -1;
+    for (const [index, line] of lines.entries()) {
+        if (!line.trim()) continue;
+        lastLine = index;
+        const timestamp = logTimestamp(line);
+        const event = line.slice(timestamp?.[0].length ?? 0).trimStart();
+        const lifecycle = /^(?:InitGame|ExitLevel|ShutdownGame)(?:\s|:|$)/.test(event);
+        if (!timestamp) {
+            if (lifecycle) {
+                segmentStart = index + 1;
+                previousSeconds = null;
+            }
+            continue;
+        }
+        const seconds = logSeconds(timestamp[1]);
+        if (
+            previousSeconds !== null &&
+            (seconds < previousSeconds || (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))
+        )
+            segmentStart = index;
+        previousSeconds = seconds;
+    }
+    // An untimed final write cannot locate the preceding timed event in real time.
+    const finalTimestamp = lastLine < 0 ? null : logTimestamp(lines[lastLine]);
+    const writtenAt = options.mtime ? Date.parse(options.mtime) : NaN;
+    const offset = finalTimestamp ? writtenAt - logSeconds(finalTimestamp[1]) * 1000 : NaN;
+    const reports = options.reports
+        .map((report) => ({ ...report, at: Date.parse(report.receivedAt) }))
+        .sort((a, b) => b.at - a.at);
+    const matches = options.matches
+        .map((match) => ({ ...match, start: Date.parse(match.startedAt) }))
+        .sort((a, b) => b.start - a.start)
+        .map((match, index, sorted) => {
+            const nextStart = sorted[index - 1]?.start ?? Infinity;
+            // Identical heartbeats are deduplicated by the leaderboard, but still
+            // establish that this match was running when the message was sent.
+            const latest = reports.find(
+                (report) => report.at >= match.start && report.at < nextStart
+            );
+            const lastSeen = Math.max(Date.parse(match.lastReceivedAt), latest?.at ?? -Infinity);
+            const end = match.endedAt
+                ? Date.parse(match.endedAt)
+                : match.status === 'active'
+                  ? lastSeen + 120_000
+                  : Date.parse(match.lastReceivedAt);
+            return { ...match, end: Math.min(end, nextStart) };
+        });
+    let logStartedAt: number | null = null;
+    let logEnded = false;
+
+    return indexedTimeline(lines).map(({ entry, lineIndex }) => {
+        const context: ChatContext = { realTime: null, matchElapsedMs: null, round: null };
+        const time = entry.type === 'message' ? entry.message.time : entry.time;
+        const at =
+            time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN;
+        if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime()))
+            context.realTime = new Date(at).toISOString();
+        if (entry.type === 'boundary' && entry.event !== 'end') {
+            // The first event after an implicit reset can be several seconds into
+            // the new process. Its clock origin is the lower bound, not that event.
+            // Untimed starts provide no safe bound for attaching an older match.
+            logStartedAt = context.realTime ? (entry.event === 'reset' ? offset : at) : Infinity;
+            logEnded = false;
+        }
+        const match =
+            context.realTime && !logEnded
+                ? matches.find(
+                      (match) =>
+                          at >= match.start &&
+                          at <= match.end &&
+                          // Allow the log's one-second precision at a start, but never cross a later InitGame.
+                          (logStartedAt === null ||
+                              match.start + 1000 >= logStartedAt ||
+                              reports.some(
+                                  (report) =>
+                                      report.event === 'game_start' &&
+                                      report.playerCount > 0 &&
+                                      report.at >= match.start &&
+                                      report.at <= at &&
+                                      report.at + 1000 >= logStartedAt!
+                              ))
+                  )
+                : undefined;
+        if (match) {
+            const report = reports.find((report) => report.at <= at && report.at >= match.start);
+            if (!report || (report.event !== 'game_end' && report.playerCount > 0)) {
+                // A repeated start updates the fleet timer even if the leaderboard
+                // deduplicates it. Use that same observed start for later messages.
+                const start =
+                    reports.find(
+                        (report) =>
+                            report.event === 'game_start' &&
+                            report.playerCount > 0 &&
+                            report.at >= match.start &&
+                            report.at <= at
+                    )?.at ?? (match.startObserved ? match.start : null);
+                if (start !== null) context.matchElapsedMs = at - start;
+                if (
+                    report &&
+                    at - report.at <= 120_000 &&
+                    (report.event === 'round_change' || report.event === 'heartbeat')
+                ) {
+                    context.round = report.round;
+                }
+            }
+        }
+        if (entry.type === 'boundary' && entry.event === 'end') logEnded = true;
+        return { ...entry, context, ...(match?.id === undefined ? {} : { matchId: match.id }) };
+    });
+}

src/lib/game-files.ts

@@ -0,0 +1,72 @@
+export type GameLibraryGame = 't5' | 't6';
+export const GAME_FILE_GAMES = [
+    { id: 't5', name: 'Black Ops I' },
+    { id: 't6', name: 'Black Ops II' }
+] as const;
+export type GameLibraryState = 'importing' | 'ready' | 'failed';
+export type LibraryCacheState =
+    'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
+
+export interface GameFileEntry {
+    path: string;
+    size: number;
+    sha256: string;
+}
+export interface LibraryTransferProgress {
+    phase: 'downloading' | 'verifying';
+    startedAt: string;
+    elapsedSeconds: number;
+    phaseElapsedSeconds: number;
+    bytesPerSecond: number | null;
+    etaSeconds: number | null;
+}
+export interface HostLibraryCache {
+    hostId: string;
+    libraryVersionId: string;
+    status: LibraryCacheState;
+    bytesDownloaded: number;
+    bytesTotal: number;
+    manifestSha256: string | null;
+    error: string | null;
+    updatedAt: string;
+    operationId: string | null;
+    bytesVerified?: number;
+    transfer?: LibraryTransferProgress;
+}
+export interface GameLibraryVersion {
+    id: string;
+    libraryId: string;
+    game: GameLibraryGame;
+    version: string;
+    description: string;
+    state: GameLibraryState;
+    sizeBytes: number;
+    fileCount: number;
+    manifestSha256: string | null;
+    files: GameFileEntry[];
+    createdAt: string;
+    publishedAt: string | null;
+    storageLocation: string;
+    sourceHostId: string | null;
+    sourceVolume: string | null;
+    error: string | null;
+    operationId: string | null;
+    caches: HostLibraryCache[];
+    filesRemoved?: boolean;
+    filesRemoving?: boolean;
+}
+export interface GameLibrary {
+    id: string;
+    name: string;
+    game: GameLibraryGame;
+    description: string;
+    createdAt: string;
+    latestVersionId: string | null;
+    versions: GameLibraryVersion[];
+}
+
+export function formatGameBytes(bytes: number): string {
+    if (!bytes) return '0 B';
+    const unit = Math.min(4, Math.floor(Math.log(bytes) / Math.log(1024)));
+    return `${(bytes / 1024 ** unit).toFixed(unit ? 1 : 0)} ${['B', 'KiB', 'MiB', 'GiB', 'TiB'][unit]}`;
+}

src/lib/host-operation-labels.ts

@@ -0,0 +1,30 @@
+export const hostOperationTitles: Record<string, string> = {
+    'instance.create': 'Create server container',
+    'instance.rebuild': 'Rebuild server container',
+    'instance.start': 'Start server container',
+    'instance.restart': 'Restart server container',
+    'instance.stop': 'Stop server container',
+    'instance.remove': 'Destroy server container',
+    'instance.cancel-create': 'Cancel server creation',
+    'instance.inspect': 'Check server container',
+    'host.inspect': 'Check host',
+    'host.decommission': 'Remove host',
+    'runtime.pull': 'Prepare container runtime',
+    'library.download': 'Download game files',
+    'library.import': 'Import game files',
+    'plutonium.prepare': 'Prepare game update',
+    'files.stage': 'Stage server files',
+    'files.apply': 'Apply server files',
+    'files.read': 'Read server files',
+    'files.list': 'List server files',
+    'files.stat': 'Check server files',
+    'files.manifest': 'Check file versions',
+    'files.remove': 'Remove server files',
+    'rcon.status-batch': 'Check game status',
+    'rcon.execute': 'Game command',
+    'logs.read': 'Read server logs'
+};
+
+export function hostOperationTitle(kind: string): string {
+    return hostOperationTitles[kind] ?? 'Host operation';
+}

src/lib/host-temperatures.ts

@@ -0,0 +1,92 @@
+export const temperatureKinds = ['cpu-package', 'cpu-die', 'cpu-core', 'nvme', 'chipset'] as const;
+export type TemperatureKind = (typeof temperatureKinds)[number];
+export interface HostTemperatureSensor {
+    id: string;
+    device: string;
+    label: string;
+    kind: TemperatureKind;
+    celsius: number;
+}
+
+/** Optional telemetry must never make an otherwise valid heartbeat fail. */
+export function validateTemperatures(value: unknown): HostTemperatureSensor[] {
+    if (!Array.isArray(value)) return [];
+    const seen = new Set<string>();
+    return value.slice(0, 128).flatMap((sensor) => {
+        if (
+            !sensor ||
+            typeof sensor !== 'object' ||
+            Array.isArray(sensor) ||
+            typeof sensor.id !== 'string' ||
+            !/^[a-zA-Z0-9_-]{1,100}$/.test(sensor.id) ||
+            seen.has(sensor.id) ||
+            !['device', 'label'].every(
+                (key) =>
+                    typeof sensor[key] === 'string' &&
+                    sensor[key].length > 0 &&
+                    sensor[key].length <= 120
+            ) ||
+            !temperatureKinds.includes(sensor.kind) ||
+            typeof sensor.celsius !== 'number' ||
+            !Number.isFinite(sensor.celsius) ||
+            sensor.celsius < -100 ||
+            sensor.celsius > 200
+        )
+            return [];
+        seen.add(sensor.id);
+        return [
+            {
+                id: sensor.id,
+                device: sensor.device,
+                label: sensor.label,
+                kind: sensor.kind,
+                celsius: sensor.celsius
+            }
+        ];
+    });
+}
+
+export function cpuTemperature(
+    sensors: HostTemperatureSensor[] = []
+): HostTemperatureSensor | null {
+    // Prefer physical package/die readings. Cores are a fallback, never mixed
+    // into an average that might hide a hot package on a multi-socket host.
+    for (const kind of ['cpu-package', 'cpu-die', 'cpu-core'] as const) {
+        const candidates = sensors.filter((sensor) => sensor.kind === kind);
+        if (candidates.length)
+            return candidates.reduce((hot, sensor) =>
+                sensor.celsius > hot.celsius ? sensor : hot
+            );
+    }
+    return null;
+}
+
+export function temperatureLabel(sensor: HostTemperatureSensor): string {
+    if (sensor.kind === 'cpu-package')
+        return `CPU package ${sensor.label.replace(/^Package id /, '')}`;
+    if (sensor.kind === 'cpu-core') return `CPU ${sensor.label.toLowerCase()}`;
+    if (sensor.kind === 'cpu-die') return `CPU die · ${sensor.label}`;
+    if (sensor.kind === 'chipset') return 'Chipset';
+    return `NVMe · ${sensor.label}`;
+}
+
+export function formatTemperature(celsius: number | null | undefined): string {
+    return typeof celsius === 'number' && Number.isFinite(celsius)
+        ? `${Number(celsius.toFixed(1))} °C`
+        : 'Unavailable';
+}
+
+/** One CPU series per host, plus bounded device series for the host drill-down. */
+export function historyTemperatures(
+    sensors: HostTemperatureSensor[] = []
+): HostTemperatureSensor[] {
+    const cpu = cpuTemperature(sensors);
+    const devices = sensors
+        .filter(
+            (sensor) =>
+                sensor.kind === 'chipset' ||
+                (sensor.kind === 'nvme' && sensor.label.toLowerCase() === 'composite')
+        )
+        .slice(0, 16);
+    return [...(cpu ? [{ ...cpu, id: 'cpu', label: 'CPU temperature' }] : []), ...devices];
+}

src/lib/hosts.ts

@@ -0,0 +1,321 @@
+import type { HostLibraryCache } from './game-files.js';
+import type { HostTemperatureSensor } from './host-temperatures.js';
+
+export type HostMode = 'active' | 'draining' | 'disabled';
+export type HostReadiness =
+    | 'preparing'
+    | 'needs-runtime'
+    | 'needs-game-files'
+    | 'ready'
+    | 'degraded'
+    | 'draining'
+    | 'disabled'
+    | 'deleting';
+export type HostOperationState =
+    'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
+
+export function hostReadinessLabel(readiness: HostReadiness): string {
+    return {
+        preparing: 'Preparing',
+        'needs-runtime': 'Needs runtime',
+        'needs-game-files': 'Needs game files',
+        ready: 'Ready',
+        degraded: 'Degraded',
+        draining: 'Draining',
+        disabled: 'Disabled',
+        deleting: 'Deleting'
+    }[readiness];
+}
+
+export interface HostDeletion {
+    allowed: boolean;
+    blockers: string[];
+}
+export interface HostDecommission {
+    operationId: string;
+    stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
+    requestedAt: string;
+    updatedAt: string;
+    error: string | null;
+    retryable: boolean;
+}
+export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
+    if (!host.decommission) return '';
+    switch (host.decommission.stage) {
+        case 'queued':
+            return host.connected
+                ? 'Host cleanup is queued.'
+                : 'Waiting for the host to reconnect before removing its files and agent.';
+        case 'running':
+            return 'Removing host files, cached game libraries, and the agent installation.';
+        case 'cleaned':
+            return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
+        case 'failed':
+            return 'Cleanup needs attention. Complete removal has not been confirmed.';
+        case 'completed':
+            return 'Host files and agent removal confirmed.';
+    }
+}
+
+export interface HostLibrary {
+    id: string;
+    state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
+    progress?: number;
+    error?: string;
+}
+export interface HostAgentUpdate {
+    enabled: boolean;
+    state: 'current' | 'checking' | 'deferred' | 'updating' | 'updated' | 'failed';
+    currentVersion: string;
+    targetVersion?: string;
+    error?: string;
+    checkedAt?: string;
+}
+
+export interface HostAgentLane {
+    name: 'health-1' | 'health-2' | 'read' | 'write';
+    alive: boolean;
+    busy: boolean;
+    lastPollAt: string | null;
+    pollAgeSeconds: number;
+}
+export function hostAgentLaneLabel(name: HostAgentLane['name']): string {
+    return {
+        'health-1': 'Health check 1',
+        'health-2': 'Health check 2',
+        read: 'File reads',
+        write: 'Actions'
+    }[name];
+}
+export function hostAgentLaneState(
+    lane: HostAgentLane
+): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
+    if (!lane.alive) return 'stopped';
+    // Reads and mutations can legitimately take minutes. Health work has a
+    // bounded budget; an old poll there indicates a blocked worker even if busy.
+    if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-')))
+        return 'stalled';
+    return lane.busy ? 'busy' : lane.lastPollAt === null ? 'starting' : 'polling';
+}
+
+export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean {
+    return (
+        inventory.agentUpdate?.enabled === true &&
+        inventory.capabilities?.includes('agent.update') === true
+    );
+}
+
+export function hostAgentUpdateLabel(inventory: HostInventory): string {
+    if (!hostAgentUpdatesEnabled(inventory)) return 'Setup required';
+    const update = inventory.agentUpdate!;
+    if (update.state === 'current' && !update.checkedAt) return 'Awaiting first check';
+    return {
+        current: 'Current',
+        checking: 'Checking',
+        deferred: 'Deferred',
+        updating: 'Updating',
+        updated: 'Updated',
+        failed: 'Needs attention'
+    }[update.state];
+}
+
+export function hostAgentUpdateMessage(inventory: HostInventory): string {
+    if (!hostAgentUpdatesEnabled(inventory))
+        return 'Run the current installer once on this host to enable automatic agent updates.';
+    const update = inventory.agentUpdate!;
+    switch (update.state) {
+        case 'current':
+            return update.checkedAt
+                ? 'The last check found no newer compatible agent release.'
+                : 'The host timer will check for a compatible agent release automatically.';
+        case 'checking':
+            return 'The host is checking the controller release and verifying its download.';
+        case 'deferred':
+            return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
+        case 'updating':
+            return 'The host is replacing agent code and checking that the new process connects successfully.';
+        case 'updated':
+            return 'The new agent connected successfully. Automatic checks continue.';
+        case 'failed':
+            return 'The update did not complete successfully. Review the reported error before retrying setup.';
+    }
+}
+
+export interface HostInventory {
+    hostname?: string;
+    agentVersion?: string;
+    protocolVersion?: number;
+    os?: string;
+    architecture?: string;
+    cpu?: string;
+    cpuCount?: number;
+    cpuUsagePercent?: number;
+    temperatures?: HostTemperatureSensor[];
+    memoryTotal?: number;
+    memoryAvailable?: number;
+    storageTotal?: number;
+    storageAvailable?: number;
+    dockerVersion?: string;
+    dockerRunning?: boolean;
+    directoriesWritable?: boolean;
+    /** Agent-reported Docker cache; removed from presented host inventory. */
+    runtimeImages?: string[];
+    /** Prepared, locally available image references eligible for launches. */
+    preparedRuntimeImages?: string[];
+    libraries?: HostLibrary[];
+    currentOperation?: string | null;
+    [key: string]: unknown;
+    capabilities?: string[];
+    plutoniumCache?: { revision: number; bytes: number };
+    agentUpdate?: HostAgentUpdate;
+    agentLanes?: HostAgentLane[];
+    instanceResources?: InstanceResources[];
+    instanceResourcesError?: string;
+    instanceStates?: InstanceState[];
+}
+export interface InstanceState {
+    instanceId: string;
+    generation: number;
+    state: string;
+    healthy: boolean;
+    observedAt: string;
+}
+export interface InstanceResources {
+    instanceId: string;
+    generation: number;
+    uptimeSeconds: number;
+    cpuCores: number;
+    memoryBytes: number;
+}
+export interface ResourceAllowance {
+    cpuCores: number;
+    memoryBytes: number;
+    source: 'host' | 'fleet' | 'bootstrap';
+    samples: number;
+    observedAt: string | null;
+}
+export interface CapacityCalibration {
+    running: ResourceAllowance;
+    startup: ResourceAllowance;
+}
+export interface CapacitySample {
+    hostId: string;
+    hostName: string | null;
+    instanceId: string;
+    server: string | null;
+    generation: number;
+    phase: 'startup' | 'running';
+    timestamp: string;
+    cpuCores: number;
+    memoryBytes: number;
+}
+export interface CapacitySamplePage {
+    samples: CapacitySample[];
+    total: number;
+    page: number;
+    pageSize: number;
+}
+export interface Host {
+    id: string;
+    name: string;
+    hostname: string;
+    mode: HostMode;
+    portMin: number;
+    portMax: number;
+    bindAddress: string;
+    publicAddress: string;
+    minimumFreeBytes: number;
+    createdAt: string;
+    lastHeartbeat: string | null;
+    connected: boolean;
+    connectionState: 'online' | 'offline';
+    readiness: HostReadiness;
+    readinessReasons: string[];
+    inventory: HostInventory;
+    enrolled: boolean;
+    credentialActive: boolean;
+    decommission: HostDecommission | null;
+    preparation?: HostPreparation;
+    enablingSince?: string | null;
+}
+export interface HostOperation {
+    id: string;
+    hostId: string;
+    kind: string;
+    instanceId: string | null;
+    generation: number | null;
+    payload: Record<string, unknown>;
+    state: HostOperationState;
+    result: unknown;
+    error: string | null;
+    createdAt: string;
+    updatedAt: string;
+}
+export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'result'> {
+    hostName: string | null;
+    serverName: string | null;
+    targetPath: string | null;
+    source: string | null;
+    routine: boolean;
+}
+export type HostOperationFilter = 'important' | 'routine' | 'all';
+export interface HostOperationScope {
+    host?: string;
+    server?: string;
+    operation?: string;
+}
+export function hostOperationScope(params: URLSearchParams): HostOperationScope {
+    const scope: HostOperationScope = {};
+    for (const key of ['host', 'server', 'operation'] as const) {
+        const value = params.get(key);
+        // eslint-disable-next-line no-control-regex -- Reject control characters in operation scope filters.
+        if (value && value.length <= 512 && !/[\x00-\x1f\x7f]/.test(value)) scope[key] = value;
+    }
+    return scope;
+}
+export function isHostOperationFilter(value: unknown): value is HostOperationFilter {
+    return typeof value === 'string' && ['important', 'routine', 'all'].includes(value);
+}
+export interface HostOperationHistoryPage {
+    operations: HostOperationSummary[];
+    page: number;
+    pageSize: 25;
+    total: number;
+    totalPages: number;
+    filter: HostOperationFilter;
+}
+
+export interface HostPreparationItem {
+    kind: 'runtime' | 'library' | 'plutonium';
+    label: string;
+    state: 'waiting' | 'preparing' | 'ready' | 'blocked';
+    detail: string;
+    operationId?: string;
+    bytesDownloaded?: number;
+    bytesTotal?: number;
+    cache?: HostLibraryCache;
+}
+export interface HostPreparation {
+    state: HostPreparationItem['state'];
+    items: HostPreparationItem[];
+}
+
+export interface NativeDeploymentSelection {
+    libraryIds: Partial<Record<'t5' | 't6', string>>;
+    image: string;
+    bindAddress?: string;
+    publicAddress?: string;
+    publicPort?: number;
+}
+
+/** Queued automatic launches have no assigned host or port until provisioning. */
+export interface NativePlacement {
+    hostId?: string;
+    libraryId: string;
+    image?: string;
+    bindAddress?: string;
+    publicAddress?: string;
+    publicPort?: number;
+    automatic?: boolean;
+    network?: { bindAddress?: string; publicAddress?: string; publicPort?: number };
+}