From e5c0fda1ed4446e802cc72594dd170d326f04fea Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sat, 3 Oct 2026 11:42:36 -0400 Subject: [PATCH] Add native host management and role-based operator authentication Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun. --- .env.example | 68 +- .gitignore | 2 + Dockerfile | 3 + README.md | 570 ++------ agent/ec_agent.py | 1231 +++++++++++++++++ agent/install.sh | 109 ++ agent/package_release.py | 13 + agent/tests/test_agent.py | 547 ++++++++ docker-compose.yml | 44 +- docs/authentication.md | 125 ++ docs/beta-test-plans.md | 32 +- docs/native-hosts.md | 174 +++ package.json | 7 +- scripts/reset-admin.mjs | 55 + src/app.d.ts | 7 +- src/hooks.server.ts | 5 +- src/lib/auth.ts | 17 + src/lib/deployment-plan.ts | 1 + src/lib/game-files.ts | 28 + src/lib/hosts.ts | 29 + src/lib/provisioning-detail.ts | 2 +- src/lib/server/auth/access.ts | 51 + src/lib/server/auth/guard.ts | 111 ++ src/lib/server/auth/http.ts | 49 + src/lib/server/auth/index.ts | 184 +++ src/lib/server/auth/passwords.js | 45 + src/lib/server/auth/schema.ts | 27 + src/lib/server/auth/telemetry.ts | 34 + src/lib/server/backends/files.ts | 50 + src/lib/server/backends/native.ts | 41 + src/lib/server/backends/schema.ts | 20 + src/lib/server/db/index.ts | 2 + src/lib/server/fleet/beta.ts | 26 +- src/lib/server/fleet/collections.ts | 32 +- src/lib/server/fleet/drift.ts | 2 +- src/lib/server/fleet/files.ts | 2 +- src/lib/server/fleet/launch-recipes.ts | 1 + src/lib/server/fleet/logs.ts | 3 + src/lib/server/fleet/servers.ts | 16 +- src/lib/server/game-files/http.ts | 44 + src/lib/server/game-files/index.ts | 256 ++++ src/lib/server/game-files/schema.ts | 39 + src/lib/server/game-files/storage.ts | 103 ++ src/lib/server/hosts/http.ts | 50 + src/lib/server/hosts/index.ts | 248 ++++ src/lib/server/hosts/releases.ts | 11 + src/lib/server/hosts/schema.ts | 47 + src/lib/server/integrations/komodo.ts | 2 + src/lib/server/integrations/rcon.ts | 29 + src/lib/server/maintenance/recovery.ts | 3 +- src/lib/server/provisioning/history.ts | 6 +- src/lib/server/provisioning/instances.ts | 20 +- src/lib/server/provisioning/native.ts | 404 ++++++ src/lib/server/provisioning/requests.ts | 8 +- src/lib/server/provisioning/restarts.ts | 2 + src/lib/server/provisioning/worker.ts | 16 + .../components/DeployCollectionDialog.svelte | 37 +- .../ui/components/DeployServerDialog.svelte | 24 +- .../ui/components/DestroyServerModal.svelte | 2 +- src/lib/ui/components/LifecycleDialog.svelte | 2 +- .../ui/components/NativeHostSelection.svelte | 41 + .../ui/components/ProvisioningProgress.svelte | 2 +- src/lib/ui/stores/fleet.ts | 2 +- src/routes/+layout.server.ts | 4 + src/routes/+layout.svelte | 100 +- src/routes/+page.svelte | 13 +- src/routes/account/+page.svelte | 57 + src/routes/api/agent/credentials/+server.ts | 12 + src/routes/api/agent/enroll/+server.ts | 8 + src/routes/api/agent/heartbeat/+server.ts | 16 + src/routes/api/agent/imports/[id]/+server.ts | 15 + .../agent/imports/[id]/complete/+server.ts | 15 + .../imports/[id]/files/[...path]/+server.ts | 11 + src/routes/api/agent/install.sh/+server.ts | 2 + .../libraries/[id]/files/[...path]/+server.ts | 7 + .../agent/libraries/[id]/manifest/+server.ts | 12 + .../agent/libraries/[id]/status/+server.ts | 11 + src/routes/api/agent/operations/+server.ts | 7 + .../api/agent/operations/[id]/+server.ts | 8 + .../api/agent/release/[file]/+server.ts | 6 + src/routes/api/auth/login/+server.ts | 10 + src/routes/api/auth/logout/+server.ts | 9 + src/routes/api/auth/me/+server.ts | 2 + src/routes/api/auth/password/+server.ts | 11 + src/routes/api/auth/setup/+server.ts | 7 + .../deployment-collections/preview/+server.ts | 2 +- src/routes/api/game-files/+server.ts | 10 + src/routes/api/game-files/[id]/+server.ts | 6 + .../api/game-files/[id]/download/+server.ts | 12 + src/routes/api/hosts/+server.ts | 13 + src/routes/api/hosts/[id]/+server.ts | 14 + .../api/hosts/[id]/credentials/+server.ts | 12 + .../api/hosts/[id]/enrollment/+server.ts | 8 + .../api/hosts/[id]/operations/+server.ts | 12 + src/routes/api/servers/+server.ts | 15 +- src/routes/api/system-status/+server.ts | 19 + src/routes/api/telemetry/+server.ts | 10 + src/routes/api/users/+server.ts | 8 + src/routes/api/users/[id]/+server.ts | 8 + .../api/users/[id]/revoke-sessions/+server.ts | 8 + src/routes/api/users/audit/+server.ts | 3 + src/routes/collections/+page.svelte | 13 +- src/routes/game-files/+page.svelte | 120 ++ src/routes/hosts/+page.svelte | 52 + src/routes/hosts/[id]/+page.svelte | 55 + src/routes/login/+page.svelte | 46 + src/routes/maintenance/[id]/+page.svelte | 2 +- src/routes/servers/[name]/+page.svelte | 32 +- src/routes/settings/users/+page.svelte | 154 +++ src/routes/setup/+page.svelte | 44 + static/agent/ec-agent.py | 1231 +++++++++++++++++ static/agent/install.sh | 109 ++ static/agent/manifest.json | 6 + svelte.config.js | 13 +- tests/helpers/telemetry.ts | 8 +- tests/unit/auth-access.test.ts | 123 ++ tests/unit/auth-guard.test.ts | 268 ++++ tests/unit/auth-users.test.ts | 369 +++++ tests/unit/game-file-library.test.ts | 104 ++ tests/unit/game-file-storage.test.ts | 61 + tests/unit/native-hosts.test.ts | 132 ++ tests/unit/native-provisioning.test.ts | 284 ++++ tests/unit/native-safety.test.ts | 121 ++ tests/unit/system-status.test.ts | 7 +- tests/unit/telemetry-auth.test.ts | 112 ++ tests/unit/telemetry-diagnostics.test.ts | 6 +- tests/unit/telemetry.test.ts | 4 +- 127 files changed, 8485 insertions(+), 657 deletions(-) create mode 100644 agent/ec_agent.py create mode 100644 agent/install.sh create mode 100644 agent/package_release.py create mode 100644 agent/tests/test_agent.py create mode 100644 docs/authentication.md create mode 100644 docs/native-hosts.md create mode 100644 scripts/reset-admin.mjs create mode 100644 src/lib/auth.ts create mode 100644 src/lib/game-files.ts create mode 100644 src/lib/hosts.ts create mode 100644 src/lib/server/auth/access.ts create mode 100644 src/lib/server/auth/guard.ts create mode 100644 src/lib/server/auth/http.ts create mode 100644 src/lib/server/auth/index.ts create mode 100644 src/lib/server/auth/passwords.js create mode 100644 src/lib/server/auth/schema.ts create mode 100644 src/lib/server/auth/telemetry.ts create mode 100644 src/lib/server/backends/files.ts create mode 100644 src/lib/server/backends/native.ts create mode 100644 src/lib/server/backends/schema.ts create mode 100644 src/lib/server/game-files/http.ts create mode 100644 src/lib/server/game-files/index.ts create mode 100644 src/lib/server/game-files/schema.ts create mode 100644 src/lib/server/game-files/storage.ts create mode 100644 src/lib/server/hosts/http.ts create mode 100644 src/lib/server/hosts/index.ts create mode 100644 src/lib/server/hosts/releases.ts create mode 100644 src/lib/server/hosts/schema.ts create mode 100644 src/lib/server/provisioning/native.ts create mode 100644 src/lib/ui/components/NativeHostSelection.svelte create mode 100644 src/routes/+layout.server.ts create mode 100644 src/routes/account/+page.svelte create mode 100644 src/routes/api/agent/credentials/+server.ts create mode 100644 src/routes/api/agent/enroll/+server.ts create mode 100644 src/routes/api/agent/heartbeat/+server.ts create mode 100644 src/routes/api/agent/imports/[id]/+server.ts create mode 100644 src/routes/api/agent/imports/[id]/complete/+server.ts create mode 100644 src/routes/api/agent/imports/[id]/files/[...path]/+server.ts create mode 100644 src/routes/api/agent/install.sh/+server.ts create mode 100644 src/routes/api/agent/libraries/[id]/files/[...path]/+server.ts create mode 100644 src/routes/api/agent/libraries/[id]/manifest/+server.ts create mode 100644 src/routes/api/agent/libraries/[id]/status/+server.ts create mode 100644 src/routes/api/agent/operations/+server.ts create mode 100644 src/routes/api/agent/operations/[id]/+server.ts create mode 100644 src/routes/api/agent/release/[file]/+server.ts create mode 100644 src/routes/api/auth/login/+server.ts create mode 100644 src/routes/api/auth/logout/+server.ts create mode 100644 src/routes/api/auth/me/+server.ts create mode 100644 src/routes/api/auth/password/+server.ts create mode 100644 src/routes/api/auth/setup/+server.ts create mode 100644 src/routes/api/game-files/+server.ts create mode 100644 src/routes/api/game-files/[id]/+server.ts create mode 100644 src/routes/api/game-files/[id]/download/+server.ts create mode 100644 src/routes/api/hosts/+server.ts create mode 100644 src/routes/api/hosts/[id]/+server.ts create mode 100644 src/routes/api/hosts/[id]/credentials/+server.ts create mode 100644 src/routes/api/hosts/[id]/enrollment/+server.ts create mode 100644 src/routes/api/hosts/[id]/operations/+server.ts create mode 100644 src/routes/api/users/+server.ts create mode 100644 src/routes/api/users/[id]/+server.ts create mode 100644 src/routes/api/users/[id]/revoke-sessions/+server.ts create mode 100644 src/routes/api/users/audit/+server.ts create mode 100644 src/routes/game-files/+page.svelte create mode 100644 src/routes/hosts/+page.svelte create mode 100644 src/routes/hosts/[id]/+page.svelte create mode 100644 src/routes/login/+page.svelte create mode 100644 src/routes/settings/users/+page.svelte create mode 100644 src/routes/setup/+page.svelte create mode 100644 static/agent/ec-agent.py create mode 100644 static/agent/install.sh create mode 100644 static/agent/manifest.json create mode 100644 tests/unit/auth-access.test.ts create mode 100644 tests/unit/auth-guard.test.ts create mode 100644 tests/unit/auth-users.test.ts create mode 100644 tests/unit/game-file-library.test.ts create mode 100644 tests/unit/game-file-storage.test.ts create mode 100644 tests/unit/native-hosts.test.ts create mode 100644 tests/unit/native-provisioning.test.ts create mode 100644 tests/unit/native-safety.test.ts create mode 100644 tests/unit/telemetry-auth.test.ts diff --git a/.env.example b/.env.example index b2829c3..80f811a 100644 --- a/.env.example +++ b/.env.example @@ -1,43 +1,33 @@ -# Filesystem paths — inside the Komodo container, these mirror the host layout -SERVERS_ROOT=/root/zombies/t5 -# T6 is separate; defaults to a sibling t6 directory beside SERVERS_ROOT. -T6_SERVERS_ROOT=/root/zombies/t6 -REL_DEST=runtime/plutonium/storage/t5/raw/scripts/sp -# On-disk archive of active beta files (cleared per-beta on promote/rollback) -BETA_ARCHIVE_DIR=/root/zombies/beta-archive +# Controller URL: use HTTPS with a trusted certificate for enrolled hosts. +EC_PUBLIC_URL=https://electric-cherry.example.com +# Generate separate secrets with: openssl rand -hex 32 +# Setup token authorizes creation of the first administrator only. Remove it +# after setup. Never reuse the telemetry token as a setup or host credential. +EC_SETUP_TOKEN= +# Game reporters send Authorization: Bearer or X-EC-Telemetry-Token. +# Telemetry ingestion is disabled until a token of at least 32 chars is set. +EC_TELEMETRY_TOKEN= +# Persistent SQLite metadata, game-file storage, and beta archives. +# For an existing deployment, retain its original data directory. +EC_DATA_DIR=/root/zombies/.electric-cherry +DB_PATH=./data/cherry.db +GAME_FILES_ROOT=./data/game-files +BETA_ARCHIVE_DIR=./data/beta-archive +BODY_SIZE_LIMIT=Infinity -# SQLite location -DB_PATH=/data/cherry.db +# Prepare this runtime from Hosts before deploying. Image and game-file +# library versions are selected independently in the launch preview. +PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest -# Configure campaign retention and restart/startup concurrency in Settings → Fleet settings. +# Optional shared RCON password. Native instances use independently generated +# secrets when omitted; RCON packets are sent locally by each host agent. +# RCON_PASSWORD= -# Komodo -KOMODO_ADDRESS=https://komodo.example.internal -KOMODO_API_KEY= -KOMODO_API_SECRET= -# Komodo server (host) that new deployments land on. Must match the exact -# name shown on the Servers page in the Komodo UI. Required to deploy new -# servers from the Fleet page. -KOMODO_TARGET_SERVER= -# Optional overrides for the plutainer defaults used when EC creates a new -# server. Change these if you fork the image or use a different docker network. -# PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest -# PLUTAINER_NETWORK=komodo_default -# PLUTAINER_GAMEFILES_VOLUME=t5_game_files -# PLUTAINER_T6_GAMEFILES_VOLUME=t6_game_files -# T6_REL_DEST_SCRIPT=runtime/plutonium/storage/t6/scripts/zm -# T6_CONFIG_TARGET_FILE=dedicated_zm.cfg -# Port range EC picks from when auto-assigning a game_port to a new server. -# Must correspond to what's actually forwarded on the Komodo host firewall. -# SERVER_PORT_MIN=28960 -# SERVER_PORT_MAX=28980 +# Development tests only. Production host transport requires HTTPS. +# EC_AGENT_ALLOW_HTTP=true +# Browser login on localhost only (never for an exposed controller): +# EC_AUTH_ALLOW_HTTP=true -# RCON — shared across the fleet (every server uses the same rcon_password). -# Host defaults to host.docker.internal so the container can reach game -# servers on the same Docker host; override if RCON lives elsewhere. -RCON_PASSWORD= -RCON_HOST=host.docker.internal -RCON_TIMEOUT_MS=3000 - -# HTTP (Komodo API) -HTTP_TIMEOUT_MS=10000 +# Existing legacy registrations can still use their previous KOMODO_*, +# SERVERS_ROOT, T6_SERVERS_ROOT, and RCON_HOST settings. Native hosts need none. +# Fleet concurrency and campaign retention are saved in Settings, not env vars. diff --git a/.gitignore b/.gitignore index f0394b9..691ede3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ build data/ test-fixtures/ .gstack/ +__pycache__/ +*.pyc diff --git a/Dockerfile b/Dockerfile index b73d1b8..769bbb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,12 +23,15 @@ COPY --from=build /app/build ./build COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/package.json ./package.json COPY --from=build /app/src/lib/server/db/schema.sql ./build/schema.sql +COPY scripts/reset-admin.mjs ./scripts/reset-admin.mjs +COPY src/lib/server/auth/passwords.js ./src/lib/server/auth/passwords.js COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENV NODE_ENV=production ENV PORT=3000 ENV HOST=0.0.0.0 +ENV BODY_SIZE_LIMIT=Infinity EXPOSE 3000 ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] diff --git a/README.md b/README.md index f89434a..08139c2 100644 --- a/README.md +++ b/README.md @@ -1,497 +1,137 @@ # Electric Cherry -Web app that manages a fleet of T5 and T6 Plutonium Zombies servers running under Komodo. +Electric Cherry manages T5 and T6 Plutonium Zombies servers through its own host +agents. Docker and Plutainer run the games; EC handles hosts, game libraries, +collections, configuration, campaigns, maintenance and operation history. +Komodo is not required for native hosts. -Fleet membership lives in SQLite. EC registers servers during creation and removes -them during destruction; startup does not discover or import server folders. -Existing database registrations are preserved, and missing mounts do not remove -servers from the fleet. The legacy `seedServerRegistry` helper is an explicit -import utility only. +## Getting started -Status refreshes check Komodo container readiness, including nested Docker health -status. A running container with `health: starting` is still **Starting**. Starting -or restarting containers hide old game data and RCON errors; stopped, unhealthy, -or unverifiable containers do not trigger routine RCON polling. Containers without -a Docker health check use their running state as the readiness signal. +Run the controller with Docker Compose behind HTTPS: -Status polling runs every 30 seconds with at most four server checks in flight. -Each completed check publishes a current fleet snapshot; a slow check does not -hold back updates or the next poll for servers that have finished. RCON status -queries retry timeouts and send errors twice, with 250ms then 500ms backoff. -Overlapping status and occupancy probes share the in-flight query for the same -server process and endpoint. Configuration and invalid-reply errors are reported -immediately; commands that change server state are never retried by this logic. +```sh +cp .env.example .env +# Set EC_PUBLIC_URL, generate EC_SETUP_TOKEN, and retain EC_DATA_DIR. +docker compose up -d --build +``` -With fresh container health, validated telemetry received in the last 45 seconds -supplies the current player count, roster, map, and game status. RCON is the fallback -when telemetry is unavailable or stale. Fleet, server details, metrics, collection -coverage, setup readiness, and beta health checks share this policy. Reports from -before the latest start, restart, or rebuild cannot verify the new process. -Historical scoreboards remain visible and are labeled separately from live counts; -RCON diagnostics retain their own results. Telemetry does not report ping or player -capacity, so those fields remain unknown in telemetry-only observations. +Open EC and create the first administrator using `EC_SETUP_TOKEN` (at least 32 +characters; generate it with `openssl rand -hex 32`). After setup, remove that +token from the environment. Administrators can add administrators, operators and +viewers from **Settings → Users**. See [Users and access](docs/authentication.md) +for permissions, sessions, password recovery and upgrade instructions. -**Active** measures time since the lobby was first observed with players; -**Match** measures time since the script's `game_start` was received. Telemetry -records occupancy changes immediately, so a later heartbeat or status poll does -not move the active timer forward. A new match keeps the active timer if the -lobby stays occupied. An empty report clears the match timer and begins the -inactive period; a later join begins a new active period. These are observed -times, not exact player connection timestamps. +Operator pages and management APIs require login. Enrolled agents authenticate +independently and connect outbound over HTTPS; the +controller needs no Docker socket, game-server mounts or inbound RCON access. +The default Compose data mount preserves the previous +`/root/zombies/.electric-cherry` database location. For a new installation, set +`EC_DATA_DIR` to your preferred persistent directory. Back up that directory, +including SQLite metadata and published game-file storage. -Server chat shows real time in your selected timezone, elapsed match time using -the same observed start as Fleet, and the round last reported when each message -was sent. Real time is approximate: the latest log write anchors the engine clock. -Earlier clock segments, missing match starts, and unavailable or stale round -reports show `—` instead of borrowing context from another match. Round history -uses retained accepted telemetry (up to 200 requests per server or seven days). +1. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its + generated enrollment command. +2. Prepare a Plutainer runtime from the host page. +3. **Library → Game Files**: import existing T5/T6 files once, then cache a + published version on the destination host. +4. Add the exact Plutonium server keys in **Keys**, or supply missing keys when + launching. +5. Launch a server or collection with an explicit host, game-file version and + runtime image. EC reserves host-specific ports and verifies startup. -## Fleet overview +See [Native host management](docs/native-hosts.md) for installation, credentials, +storage, failure recovery, API contracts and the live-host acceptance checklist. -Fleet searches server names, map names, and the full player roster. Filter occupied, -empty, unhealthy, testing, or pinned lobbies; switch between cards and a compact -table; and group by game or map. Pins and display preferences are saved in this -browser. Map groups preserve T6 variants and can preselect the exact map when -launching another lobby. +## Fleet and operations -Health summaries exclude stale player counts. The match spotlight uses fresh round -telemetry from an occupied, ongoing match. Persisted campaign, beta, creation, -removal, and restart-recovery state appears alongside affected servers. Fleet -membership and operation progress refresh every ten seconds, with live server -status between refreshes. Failed refreshes keep the last snapshot visible. +The fleet registry lives in SQLite, independently of host filesystems. Status +combines agent container health, fresh game telemetry and agent-local RCON. +Unreachable hosts report unknown instance state; they do not trigger duplicate +replacements. Running containers continue through controller or agent outages. -**Metrics** explores recorded server use by time range, game, and map. It includes -occupied/idle/unknown timelines, sortable rankings, previous-period comparisons, -map capacity pressure, and an activity heatmap in the selected timezone. Hosting -experiments suggest changes from explicit rules, including tentative advice from -sparse observations; they never change the fleet automatically. Background refresh -keeps the last report visible if it fails. See [Metrics](docs/metrics.md) for the -definitions, evidence limits, and experimental rules. +Provisioning owns Create, Start, Stop, Restart, Rebuild and Destroy. Commands and +results have durable operation IDs and immutable instance generations. Queues, +capacity reservations and history survive controller restarts. **Rebuild removes +per-instance writable files** and creates a new generation. Shared immutable game +libraries survive rebuilds and destruction. -## Stack +Disruptive actions wait for a fresh empty lobby or verified stopped container. +The agent rechecks occupancy locally immediately before activation and disruption. +Long player sessions can continue indefinitely. **Drain** prevents new launches +and safely retires existing servers as they become eligible. -- SvelteKit (adapter-node) + TypeScript -- SQLite via better-sqlite3 (replaces `script_metadata.json` as the source of - truth) -- Server-sent events for live pipeline / server-status streams -- Game telemetry for current player counts and rosters, with direct UDP RCON as fallback -- Komodo API for restarts and per-server game-port discovery -- Runs as a single Komodo-managed container on the same host as the game - servers, with `/root/zombies` bind-mounted in +Collections save target maps and lobby counts. Fill missing servers adds only the +shortfall; Match collection exactly also queues safe removal of excess servers. +Launch previews select a host, pinned base-game versions and an independent +runtime image. Every new launch uses the latest promoted configuration, scripts +and plugins when execution starts. Saved Plutonium keys are reused by exact +server ID and retained after removal. -## Layout +## Libraries and campaigns -- `src/lib/server/db/` — schema, migrations, better-sqlite3 handle -- `src/lib/server/fleet/` — file walking, push/beta engines -- `src/lib/server/integrations/` — Komodo, RCON -- `src/lib/server/pipeline/` — sweep loop, event bus, worker singleton -- `src/lib/server/provisioning/` — lifecycle requests, queues, execution, recovery, and history -- `src/routes/api/` — REST + SSE endpoints -- `src/routes/` — UI pages (dashboard, campaigns, server deployments, betas, per-server) -- `Dockerfile`, `docker-compose.yml`, `entrypoint.sh` — container build -- `komodo/` — Komodo stack definition -- `scripts/` — legacy JSON importer, fixture seeder -- `tests/` — vitest unit tests +Game-file versions include game, version, DLC metadata, manifests, checksums and +size. Actual game files use dedicated filesystem storage, never SQLite blobs. +Downloads resume, report progress and verify checksums before becoming usable. +Multiple instances on one host share read-only base files. -## Plugin releases +Scripts, plugins, configuration baselines and overlays retain their versioning, +beta, promotion and rollback workflows. Campaigns stage bytes remotely without +stopping games; activation uses the same occupancy and concurrency gates as +lifecycle operations. Agent before images preserve exact rollback bytes and +operation history. -Plugins support named release versions, GitHub DLL imports and update discovery. -See [Plugin releases](docs/plugin-releases.md) for the import and beta workflow. +T5 Zombies uses `t5sp` and `dedicated_sp.cfg`; T6 uses `t6zm` and +`dedicated_zm.cfg`. Map selection generates `electric_cherry_map.cfg`, loaded +after the main config. T6 variants retain their distinct catalog selections. +Game-file libraries and Plutonium runtime revisions are tracked independently. -## Maintenance +Game telemetry uses its own `EC_TELEMETRY_TOKEN`; configure reporters to send it +in an authorization header. Anonymous reports are rejected before they can +affect occupancy or game history. Existing JSON payloads are unchanged. -Open **Operations → Maintenance** for health recovery, configuration drift, -Plutonium updates, and scheduled restarts. Overview shows active work, attention, -version coverage, and upcoming restarts. Policies & schedules controls each -capability independently; Activity links findings and results to Provisioning. -Fleet and server pages show runtime revision indicators and maintenance links. +## Documentation -Maintenance decides when work is needed and verifies the result. Provisioning -executes every lifecycle operation using the shared safety checks and queues. -Health recovery uses one restart followed by one rebuild if needed; drift repair -requests a rebuild with a linked Campaign. **Rebuild deletes all instance files.** -Maintenance skips observations and new automatic work on servers with active -Provisioning operations or retained recovery capacity. Other servers continue -normally, and existing maintenance still tracks its own operation's outcome. +- [Native hosts, storage and operation recovery](docs/native-hosts.md) +- [Users, roles and authentication](docs/authentication.md) +- [Provisioning and safety queues](docs/provisioning.md) +- [Collections and promoted launch recipes](docs/deployment-blueprints.md) +- [Maintenance](docs/maintenance.md) +- [Beta test plans and game telemetry](docs/beta-test-plans.md) +- [Plugin releases](docs/plugin-releases.md) +- [Metrics](docs/metrics.md) +- [Leaderboard and match history](docs/leaderboard.md) -Plutonium updates are applied by Plutainer's startup updater -(`PLUTAINER_AUTO_UPDATE` must be enabled). Check versions and choose **Update on -restart**, or opt into automatic updates for verified outdated servers. Revision -checks run hourly, and Create and Rebuild verify and record the running revision -before completing. Unknown or stale revisions remain unverified. A healthy restart -can still fail update verification; Maintenance reports that outcome without -rebuilding or repeatedly restarting the server. Electric Cherry does not download -binaries or pin a runtime revision. +Older workflow documents may describe the legacy Komodo backend. Native hosts +use the agent boundary described in the native-host document. -Optional daily restarts are independent of updates. Schedules use the saved -timezone, follow daylight saving changes, and coalesce missed days into one run. -Intentionally stopped servers are excluded. Existing schedules, pending jobs, -operation IDs, and history survive the move from Settings and Watchdog. -Servers owned by Provisioning are skipped for that scheduled occurrence. +## Development -**Pause all maintenance** holds new and queued maintenance while checks and -submitted recovery continue. Each capability retains its own policy. Automatic -updates remain disabled until explicitly enabled. See [Maintenance](docs/maintenance.md) -for verification, retry, scheduling, and compatibility details. +The controller is SvelteKit/TypeScript with SQLite via better-sqlite3, the Node +adapter and server-sent events. The host agent is Python 3 standard-library code +installed as a systemd service. Hosts do not install Node.js or build EC. -## Fleet settings - -**Settings → Fleet settings** controls campaign history retention (default 7 days), -concurrent restarts (default 1), concurrent server startups (default 1), and -concurrent server destructions (default 1). -Values are positive whole numbers; retention supports up to 36500 days. Settings -are saved in SQLite for all users and apply at the next queue or cleanup check -without restarting the app. Lowering a limit leaves active operations running -and holds queued work until capacity is available. Collection previews and -progress pages display the current startup limit. - -Retention removes eligible old campaign records and file diffs. Pending, failed, -held, and health-recovery-linked campaigns remain protected; Provisioning history -is retained. Shortening retention can remove older history on the next sweep. - -On the first startup after upgrading, existing `DEPLOY_KEEP_DAYS`, -`DEPLOYMENT_RESTART_CONCURRENCY`, and `DEPLOYMENT_CREATE_CONCURRENCY` environment -values initialize the saved settings. Keep custom values for that first startup; -afterward, the database values take precedence and those variables can be removed. -The supplied Compose and Komodo examples use the built-in defaults. -Existing saved settings retain their values when upgraded; the destruction -limit starts at 1 and can be changed in Settings. - -`GET /api/settings` reads the four values (`deployKeepDays`, `restartConcurrency`, -`createConcurrency`, `destroyConcurrency`); `PUT /api/settings` saves all four together as JSON. -Invalid or incomplete requests return HTTP 400 without changing saved settings. - -## How restarts stay safe - -Betas support configurable gameplay, participation, stability and manual-review -requirements, saved presets, coverage rules and staged waves. See -[Beta test plans](docs/beta-test-plans.md) for usage and the four-event game telemetry contract. - -File changes are staged sequentially. Their activation uses rolling restarts: -by default only **one server across the fleet** can restart or recover at a -time. Promotions, rollbacks, drift reconciliation, and beta restarts share this -limit. Adjust **Concurrent restarts** in **Settings → Fleet settings** to raise it. - -The Provisioning worker runs every 15s. Campaigns, betas, and maintenance submit -durable restart requests and read their results. Provisioning checks recovering -servers first, then admits queued operations when there is capacity. Requests -normally run in submission order; on a shared server, beta activation precedes -campaign activation, and maintenance waits for both. Busy -or unreachable servers stay queued without taking a slot, so other empty -servers can proceed. A successful Komodo restart request retains its slot -until the container is healthy and RCON or fresh telemetry verifies game status. Health checks start no sooner -than 15 seconds after the request returns, avoiding an immediate check of the -old process. An unhealthy server keeps its slot rather than letting failures -spread across the fleet. A timed-out restart also keeps its slot because Komodo -may have accepted the request. - -Slots persist in SQLite across app restarts, and overlapping timer/API triggers -share the active run. Beta options to skip health or match gates still work; -skipping a beta gate does not release the fleet's recovery slot early. New-server -creation uses a separate shared startup queue, holding its slot through container, -game-status, selected-map, and Plutonium revision verification. - -No token, no session, no auth freeze. If Komodo or RCON aren't configured, -the sweep marks pending rows `manual` so nothing hangs forever. - -## Collections, provisioning, and launch previews - -Provisioning also offers **House Party Protocol**: review the entire fleet and queue -destruction in one action. This includes queued creations and reuses pending -destruction requests. Each server waits for a fresh zero-player reading; occupied -or unreachable servers remain queued. Containers, server files, and registrations -are removed, while saved keys and collections are retained. The preview must be -reviewed again if fleet membership changes before confirmation. Servers created -after confirmation are not included. Progress appears in deployment history. -If any target belongs to an active beta, no removals are queued. Promote or roll -back that beta before retrying fleet removal or an exact collection launch. - -If deletion fails after Komodo removal, the operation detail page offers -**Resume cleanup** to retry local file and database removal without RCON or -repeating Komodo deletion. Requesting destruction again also resumes that cleanup. -Recovery is blocked if a newer operation exists for the same server name. - -**Operations → Collections** includes a builder for whole server lineups: choose -multiple T5/T6 maps and set the desired total lobbies for each. Collections also -has **Active launches** and **Launch history** tabs, with progress at -`/collections/launches/:id`. -Each server operation remains in Provisioning and links back to its collection -launch. Existing collection launch bookmarks redirect to the new location. -**Conform fleet** -counts existing and queued servers and creates only the shortfall. The preview -shows target/existing/missing counts and IDs, ports, and key requirements for new -servers only. Already-satisfied targets create nothing; excess servers are kept. -Choose **Match collection exactly** in the preview to also remove excess lobbies -and maps outside the collection. Review the keep/create/remove lists and confirm -removals. Matching servers stay untouched; unstarted excess creations are cancelled, -and running servers wait for a fresh zero-player check before removal. New lobbies -can reserve ports from these removals and wait until they are released. The collection -page tracks both actions. **Fill missing servers** remains the default. -All jobs and capacity reservations are committed together before provisioning; -one collection page tracks each server's independent progress and failures. -New servers start one at a time by default across all collections and individual -launches. A startup slot stays occupied until Komodo is healthy, RCON or fresh -telemetry reports the selected map, and RCON verifies the current Plutonium -production revision or newer. Unknown release or runtime revisions hold the queue -while verification retries; a confirmed older runtime fails provisioning. The -queue survives application restarts. The collection preview -shows the startup limit, and its progress page shows active startups and each -server's place in the shared queue. Adjust **Concurrent server startups** in -**Settings → Fleet settings** to increase the limit. Each worker pass starts at most the -available number of slots; immediate failures do not drain the remaining queue. -Existing single-server collections can be expanded with **Edit**. Single-server -launches and launching again from recorded recipes remain available. -Collections save maps and target lobby counts. Every new server uses the latest -promoted scripts, plugins, and configuration when provisioning begins, including -promotions made after preview or while queued. Existing collections automatically -follow this policy. Deployment history records the files selected for that launch; -**Launch again** uses the current promoted setup for the same game and map. -The progress page explains waiting conditions, shows elapsed time, and can resume -health verification of an interrupted creation when its original server remains. -See [Deployment collections](docs/deployment-blueprints.md) for the workflow, -promotion policy, and API details. - -## Saved server keys - -Use **Keys** to add, replace, or remove a Plutonium key for an exact -server ID such as `t5_zombie_theater_2`. IDs use a supported map slug and a -lobby number from 1 to 20; the server does not need to be deployed first. -During deployment, choose the map. The lowest available lobby is assigned -automatically, with the exact server ID shown before deployment. Its saved key is reused automatically, -and any newly entered key is saved for that ID, including if deployment fails. -Destroying a server preserves its saved key. Updating or removing a saved key -does not change an existing server; changes apply when that server ID is next -created or rebuilt. Electric Cherry does not check whether Plutonium accepts a -saved key. **Last saved** shows when the key was saved, not a validation result. - -Keys are stored in the application's SQLite database at `DB_PATH`, independently -of server records. Preserve that database across container replacements. Key -values are not returned by the management API, but are stored unencrypted in -SQLite, so database files and backups contain credentials. Existing deployment -keys are not automatically imported from Komodo; add them once to retain them. - -## Local dev - -Windows or macOS — no `/root/zombies` on disk, so seed a fixture tree first. - -```bash +```sh npm install -npm run seed:fixture +npm run agent:package +# Export EC_SETUP_TOKEN in your shell using the secret saved for /setup. +EC_AUTH_ALLOW_HTTP=true npm run dev -# Point env at the fixture (PowerShell) -$env:SHARED_SCRIPTS = "$PWD/test-fixtures/zombies/shared/scripts" -$env:SERVERS_ROOT = "$PWD/test-fixtures/zombies/t5" -$env:DB_PATH = "$PWD/data/cherry.db" - -npm run dev -# open http://localhost:3000 -``` - -Without RCON_PASSWORD / KOMODO_* set, the pipeline marks pending rows -`manual` after a sweep — the UI still works, you just don't get real -restarts. Enough to develop the push/beta flow. - -## Map selection and configs - -The map selected when deploying a T5 zombies server controls its startup -rotation automatically. No map-choice config overlay is required. New servers -use the promoted game baseline and matching overlays when available; otherwise -Plutainer seeds its usual defaults and Electric Cherry applies the selected map -through a generated `configs/electric_cherry_map.cfg`, loaded after the main -config and before map rotation. This file is tracked and restored by drift -reconciliation; operators do not need to create or promote it. The launch order -uses Plutainer's [extra arguments and map-rotation controls](https://github.com/Ayymoss/Plutainer/blob/main/docs/configuration.md). -Plutainer's -default main config is preserved until a game baseline is promoted. - -Beta previews, promotion, and config drift repair all apply the server's stored -catalog map after composing the baseline and overlays. An old `sv_maprotation` -overlay cannot override that selection, but its other settings still apply. -Existing servers receive this behavior on their next config deployment or drift -reconciliation; this change does not immediately rewrite or restart them. -Servers with an unknown map retain their existing config behavior. Rollback -restores the exact prior config bytes. - -## T6 Zombies (Black Ops II) - -Choose **Black Ops II (T6)** in Deploy new server. T6 uses `t6_zombie__` -IDs, its own saved Plutonium keys, `PLUTAINER_GAME=t6zm`, and -`configs/dedicated_zm.cfg`. Existing T5 deployments and environment settings -keep their current paths. API clients can send `game: "t6"`; omitting it selects T5. - -Before the first deployment, populate the Docker volume `t6_game_files` on the -Komodo host with your Black Ops II game files, including the DLC maps you intend -to host. Set `PLUTAINER_T6_GAMEFILES_VOLUME` to use another volume or an absolute -host directory. It is mounted read-only at `/home/plutainer/gamefiles`. -Plutainer supplies its runtime and config seeds, not the base game files. -Use a Plutonium server key created for **T6 Zombies**. - -Server state defaults to `/root/zombies/t6`, or the sibling `t6` directory beside -`SERVERS_ROOT`; override with `T6_SERVERS_ROOT`. Electric Cherry and the Komodo -host must see this directory at the same absolute path. The supplied Compose -mount already covers both games. GSC files go to -`runtime/plutonium/storage/t6/scripts/zm` (no `raw/`), and game logs use -`logs/games_zm.log`. Optional overrides are `T6_REL_DEST_SCRIPT` and -`T6_CONFIG_TARGET_FILE`; a custom config filename needs a promoted baseline or -a file supplied at that name, since Plutainer seeds `dedicated_zm.cfg`. - -The catalog includes the classic maps, Survival, Grief, and Turned selections. -Rotations use `execgts .cfg map `; Plutainer seeds the required -`storage/t6/gamesettings` files. Keep seeding enabled, including with a promoted -baseline. Town, Farm, and Bus Depot share `zm_transit`, so EC keeps their selection -in `map_slug` through creation, beta, promotion, and drift repair. Map-scoped -artifacts target the engine map and therefore apply to all its variants. -Creation health checks verify the engine map reported by RCON or fresh telemetry; this does not -independently verify the start location within that map. - -Both games share the fleet's port allocator (`SERVER_PORT_MIN`/`SERVER_PORT_MAX`) -to prevent collisions. EC explicitly sets `PLUTAINER_PORT`, so T6 need not use -its upstream default of 4976. Forward the allocated UDP ports. Use a current -Plutainer v2 image: EC relies on `PLUTAINER_RCON_PASSWORD`, -`PLUTAINER_MAP_ROTATE=false`, and extra arguments loaded after the main config. -Current Plutainer automatically whitelists the Docker gateway for T5/T6 RCON; -an admin client on another host needs its address allowed in the server config. - -Implementation references (checked September 2026): - -- [Plutainer game requirements](https://github.com/Ayymoss/Plutainer/blob/main/docs/games.md) -- [Plutainer launch, seeding, and RCON implementation](https://github.com/Ayymoss/Plutainer/blob/main/scripts/lib/cod.sh) -- [Plutainer configuration](https://github.com/Ayymoss/Plutainer/blob/main/docs/configuration.md) -- [T6 map/ruleset combinations](https://github.com/xerxes-at/T6ServerConfigs/blob/master/localappdata/Plutonium/storage/t6/dedicated_zm.cfg) -- [Plutonium GSC paths](https://plutonium.pw/docs/modding/loading-mods/) - -## Game telemetry - -The server script sends JSON to `POST /api/telemetry`: - -- `game_start`: server, map, and all players (name and GUID). -- `round_change`: server, map, round, and all players with kills, headshots, downs, revives, and score. -- `heartbeat`: the same fields as round change, sent periodically during a round to refresh player stats. -- `game_end`: the same fields as round change, with final stats. - -These reports work with or without an active beta. Electric Cherry records -the full player data, updates Fleet, and applies gameplay progress to active -betas automatically. No discovery request or beta-specific fields are needed. - -See [Game reporter protocol](docs/beta-test-plans.md#game-reporter-protocol) for -complete payload examples. Fresh telemetry can verify player count and game -activity alongside RCON during provisioning and Watchdog recovery. Occupied -reports prevent interruptions; stale reports cannot authorize them. Komodo still -verifies container readiness. - -For a missing scoreboard, open the server's **Diagnostics → Telemetry API requests**. -The log shows received requests for that exact server name, including accepted -reports, rejected payloads, HTTP status, errors, and expandable submitted JSON. -It refreshes every five seconds while visible and survives app restarts. Logging -starts with this version; older requests cannot be reconstructed. See -[Telemetry diagnostics](docs/beta-test-plans.md#telemetry-diagnostics) for retention -and how to interpret missing or rejected reports. - -The **Leaderboard** records persistent player totals and map records from these -same reports, including highest round, cumulative kills, rounds survived, -headshots, revives, downs, and games played. See [Leaderboard](docs/leaderboard.md) -for counting rules, history import, and the read API. Only completed games count; -in-progress and interrupted games remain in Match History without contributing -to player rankings or profile totals. - -In **Match History**, select **View match** on a completed match to open its -shareable results page. It shows the map, server, final round, timing, and every -recorded player's stats, including players absent from the final report. Completed -matches also link from server history and player profiles. Results survive server -deletion; missing starts and unrecorded stats remain explicitly unknown. - -## Running tests - -```bash +npm run check npm test +npm run test:agent +npm run build ``` -Covers push (add/update/skip), beta lifecycle (open → rollback, open → -promote, conflict blocks), sweep (integrations-off manual branch), and -RCON parsing (fake UDP socket). +Builds regenerate the downloadable agent and checksum manifest. Local controller +metadata defaults to `./data/cherry.db`; use a separate test database when +exercising enrollment. Production agent transport requires HTTPS. -## Deploy to the Komodo host +`src/lib/server/hosts` owns enrollment, readiness and command transport; +`src/lib/server/backends` provides instance identity and remote file access; +`src/lib/server/provisioning/native.ts` connects existing lifecycle and campaign +queues to agents; `src/lib/server/game-files` owns immutable libraries and +streaming distribution; `agent/` contains the service, installer and agent tests. -1. Build the image once locally or on the host: - - ```bash - docker compose build - ``` - -2. Import `komodo/stack.toml` into Komodo (or point Komodo at the - `docker-compose.yml` at repo root). - -3. Set the env vars in Komodo: - - ``` - KOMODO_ADDRESS, KOMODO_API_KEY, KOMODO_API_SECRET - RCON_PASSWORD # single shared password across every T5/T6 server - RCON_HOST # optional; defaults to host.docker.internal - ``` - -4. First boot triggers the legacy import: if - `/root/zombies/script_metadata.json` exists and `/data/cherry.db` is - empty, `entrypoint.sh` runs `scripts/import-legacy.ts` before starting - the server. The source JSON gets renamed to `.migrated`. - -5. Load the app on the LAN or over tailscale (no web-app auth by design). - -## Campaigns and provisioning - -Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility. - -Beta deployments, promotions, drift repairs, and rollbacks stage their file changes -in SQLite. Live server files and inventory remain unchanged while waiting for an -empty or stopped server, other operations, and rolling restart capacity. Provisioning -captures rollback snapshots and applies the staged changes immediately before -requesting the restart. Failed file activation blocks that restart. Queued changes -survive application restarts; cancelling an unapplied beta discards its pending -changes. Manual beta rollback holds both file restoration and restart until -**Resume when empty** is selected on its rollback campaign. -Finish pending file campaigns before opening a beta on their servers; this keeps -an older queued change from overwriting the version tested by the new beta. - -Deleting a server preserves its name, recorded beta progress and test evidence, -campaign results and file diffs, provisioning and maintenance history, and match -records. Historical entries show **Deleted** and omit live server links and -actions. Reusing the same server name starts a new instance; old records remain -marked deleted, and old work cannot target the replacement. Servers linked to an -active beta cannot be destroyed or rebuilt until that beta is promoted or rolled -back. A participant removed outside provisioning still pauses its open beta; roll -it back and open a new beta to test the replacement. -Normal history retention still applies. Records erased by earlier versions cannot -be restored by this upgrade. Existing history is marked deleted where the server -is already absent; instances that reused a name before upgrading cannot always be -distinguished. - -Provisioning (`/provisioning`, `/api/provisioning`) owns Create, Start, Restart, Rebuild, Stop, and Destroy: request persistence, eligibility and occupancy checks, execution, recovery, and history. Campaigns, betas, and scheduled maintenance request lifecycle work through Provisioning and link to the resulting operation. They retain their file, test-evidence, and version-verification workflows. Pending automated restarts appear in Provisioning before execution. History survives server removal. New creation requests return HTTP 202 with an operation ID and open the provisioning detail page. Create and Rebuild complete only when Komodo reports healthy, RCON or fresh telemetry verifies the selected engine map and game status, and RCON reports a Plutonium runtime revision at least as new as the current production release. Provisioning records the release and runtime evidence so new servers have verified version status immediately. Unknown release or runtime revisions are retried; a confirmed older runtime fails with a clear error and no automatic restart. Queued requests reserve their server name and port; queued creation and pending verification survive application restarts. The detail page records startup progress and explains any health, map, or revision issue blocking completion. - -Each server deployment opens a detail page with its outcome, blocker, permitted -recovery actions, grouped execution phases, and timestamped verification evidence. -New jobs record command certainty and the source of game-health checks; older -records retain their known outcomes without invented evidence. A failed restart's -ongoing recovery and held capacity appear separately from its recorded failure. -Recent activity is visible immediately, with searchable history loaded in pages. -Launch files distinguish the setup captured while queued from the files selected -when creation began. The page retains its last snapshot if a refresh fails. - -Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh zero-player RCON or telemetry report, or a verified stopped container, before teardown. Occupied reports from either source block teardown; servers without current verification remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page. - -**Settings → Concurrent server destructions** limits simultaneous individual and -fleet removals (default 1). Each destruction holds its slot through teardown and -local cleanup. Lowering the limit lets active removals finish before more begin. - - -Use **Provisioning → Manage server**, or a server’s **Server actions**, to start, -restart, stop, or rebuild an existing instance. **Rebuild destroys and creates in -place**: the old container, Komodo deployment, server registration, and all server -files are deleted. A fresh instance uses the same server ID, map, and port with -the latest promoted setup. No old instance files are carried forward. The key -stored separately in **Keys** must be available before teardown. Stop retains the -instance so Start can bring it back. Disruptive operations wait for a fresh empty -reading or a confirmed stopped container, and completion requires verification. -See [Provisioning](docs/provisioning.md) for lifecycle behavior and recovery. - -Existing `/deployments` page links redirect to `/provisioning`, preserving filters -and operation IDs. Legacy `/api/deployments` endpoints and stored history remain -compatible. +Automated tests use simulated Docker/game hosts. Real Debian installer and +T5/T6 game-server validation remain deployment acceptance work; see the checklist +in the native-host guide. diff --git a/agent/ec_agent.py b/agent/ec_agent.py new file mode 100644 index 0000000..db8ab0c --- /dev/null +++ b/agent/ec_agent.py @@ -0,0 +1,1231 @@ +#!/usr/bin/env python3 +"""Electric Cherry host agent. Python 3 standard library; no remote shell API. + +Each operation is recorded before execution. An interrupted operation becomes +unknown and is reported for reconciliation, never automatically executed twice. +""" + +import argparse +import base64 +import contextlib +import hashlib +import fcntl +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import shutil +import signal +import socket +import sqlite3 +import stat as stat_module +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +VERSION = "1.0.0" +LABEL = "io.electric-cherry" +MAX_FILE = 32 * 1024 * 1024 +ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") + + +class AgentError(Exception): + pass + + +class Held(AgentError): + """A reversible prerequisite prevented any disruptive action.""" + + +class Rejected(AgentError): + """A definitive precondition failure, before any destructive action.""" + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def file_digest(path): + h = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def status_players(text): + lines = text.splitlines() + separator = next((index for index, line in enumerate(lines) if re.match(r"^\s*-+\s+-+", line)), None) + if separator is not None and any(re.search(r"\bnum\s+score\s+ping\b", line, re.I) for line in lines[:separator]): + rows = [line.strip() for line in lines[separator + 1:] if line.strip()] + if any(not re.match(r"^\d+\s+", row) for row in rows): + return None + return len(rows) + info = next((index for index, line in enumerate(lines) if line.startswith("\\") and "\\mapname\\" in line), None) + if info is not None: + return sum(bool(line.strip()) for line in lines[info + 1:]) + # A truncated header or an unfamiliar dialect is never proof of emptiness. + return None + + +def identifier(value): + if not isinstance(value, str) or not ID_RE.fullmatch(value): + raise AgentError("Invalid identifier") + return value + + +def instance_identity(instance_id, generation): + try: + if str(uuid.UUID(instance_id)) != instance_id.lower(): + raise ValueError() + except (ValueError, AttributeError, TypeError): + raise AgentError("Instance ID must be a UUID") from None + if isinstance(generation, bool) or not isinstance(generation, int) or generation < 1: + raise AgentError("Invalid instance generation") + return instance_id.lower(), generation + + +def scoped(root, value, allow_root=False): + """Reject traversal AND existing symlinks, including dangling symlinks.""" + if not isinstance(value, str) or "\\" in value or "\x00" in value: + raise AgentError("Invalid managed path") + rel = PurePosixPath(value) + if rel.is_absolute() or any(p in ("..", "") for p in rel.parts): + raise AgentError("Path is outside the managed directory") + if value not in ("", ".") and str(rel) != value: + raise AgentError("Managed paths must use their canonical relative form") + if (not rel.parts or str(rel) == ".") and not allow_root: + raise AgentError("A file path is required") + root = Path(root) + current = root + if current.is_symlink(): + raise AgentError("Managed root must not be a symlink") + for part in rel.parts: + current = current / part + if current.is_symlink(): + raise AgentError("Symlinks are not allowed in managed file operations") + return current + + +def atomic_bytes(path, data, mode=0o600): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + try: + with open(tmp, "xb") as stream: + os.chmod(tmp, mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + finally: + with contextlib.suppress(FileNotFoundError): + tmp.unlink() + + +def atomic_json(path, value): + atomic_bytes(path, json.dumps(value, sort_keys=True).encode()) + + +@contextlib.contextmanager +def managed_parent(root, value, create=False, uid=1000, gid=1000): + """Use dirfds so an instance cannot race a symlink into a host-root write.""" + scoped(root, value) + parts = PurePosixPath(value).parts + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + fd = os.open(root, flags) + try: + for part in parts[:-1]: + if create: + with contextlib.suppress(FileExistsError): + os.mkdir(part, 0o755, dir_fd=fd) + if os.geteuid() == 0: + os.chown(part, uid, gid, dir_fd=fd, follow_symlinks=False) + child = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = child + yield fd, parts[-1] + finally: + os.close(fd) + + +@contextlib.contextmanager +def managed_open(root, value): + with managed_parent(root, value) as (parent, name): + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) + try: + if not stat_module.S_ISREG(os.fstat(fd).st_mode): + raise AgentError("Requested managed path is not a regular file") + with os.fdopen(fd, "rb", closefd=False) as stream: + yield stream + finally: + os.close(fd) + + +def managed_read(root, value): + try: + with managed_open(root, value) as stream: + if os.fstat(stream.fileno()).st_size > MAX_FILE: + raise Rejected("Managed file exceeds the 32 MiB read limit") + content = stream.read(MAX_FILE + 1) + if len(content) > MAX_FILE: + raise Rejected("Managed file exceeds the 32 MiB read limit") + return content + except FileNotFoundError: + return None + + +def managed_write(root, value, content, uid=1000, gid=1000): + if content is None: + try: + with managed_parent(root, value) as (parent, name): + with contextlib.suppress(FileNotFoundError): + os.unlink(name, dir_fd=parent) + os.fsync(parent) + except FileNotFoundError: + pass + return + with managed_parent(root, value, create=True, uid=uid, gid=gid) as (parent, name): + prior = None + try: + prior = os.stat(name, dir_fd=parent, follow_symlinks=False) + if not stat_module.S_ISREG(prior.st_mode): + raise Rejected("Managed target is not a regular file") + except FileNotFoundError: + pass + temp_name = ".ec-" + uuid.uuid4().hex + fd = os.open(temp_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent) + try: + os.fchmod(fd, prior.st_mode & 0o777 if prior else 0o644) + if os.geteuid() == 0: + os.fchown(fd, prior.st_uid if prior else uid, prior.st_gid if prior else gid) + with os.fdopen(fd, "wb", closefd=False) as stream: + stream.write(content) + stream.flush() + os.fsync(fd) + os.replace(temp_name, name, src_dir_fd=parent, dst_dir_fd=parent) + os.fsync(parent) + finally: + os.close(fd) + with contextlib.suppress(FileNotFoundError): + os.unlink(temp_name, dir_fd=parent) + + +class Journal: + def __init__(self, path): + self.db = sqlite3.connect(path) + self.db.row_factory = sqlite3.Row + self.db.execute("PRAGMA journal_mode=WAL") + self.db.execute("PRAGMA synchronous=FULL") + self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)") + if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}: + self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT") + self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),)) + self.db.commit() + + def accept(self, request): + op_id = identifier(request.get("id")) + immutable = {key: request.get(key) for key in ("id", "kind", "instanceId", "generation", "payload")} + canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":")) + fingerprint = digest(canonical.encode()) + row = self.db.execute("SELECT * FROM operations WHERE id=?", (op_id,)).fetchone() + if row: + if row["fingerprint"] != fingerprint: + raise AgentError("Operation ID was reused with different content") + return dict(row), False + self.db.execute("INSERT INTO operations(id,fingerprint,request,state,updated) VALUES(?,?,?,'accepted',?)", (op_id, fingerprint, canonical, time.time())) + self.db.commit() + return self.get(op_id), True + + def get(self, op_id): + row = self.db.execute("SELECT * FROM operations WHERE id=?", (op_id,)).fetchone() + if row is None: + raise AgentError("Unknown operation") + return dict(row) + + def update(self, op_id, state, result=None, error=None): + self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id)) + self.db.commit() + return self.get(op_id) + + def close(self): + self.db.close() + + def checkpoint(self, op_id, value): + self.db.execute("UPDATE operations SET checkpoint=?, updated=? WHERE id=?", (json.dumps(value), time.time(), op_id)) + self.db.commit() + + +class Docker: + def call(self, *args, timeout=120, tolerate_missing=False, merge_stderr=False): + try: + result = subprocess.run(["docker", *args], stdout=subprocess.PIPE, stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE, timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired): + raise AgentError("Docker command unavailable or timed out") from None + if result.returncode: + # Never include arbitrary Docker stderr: create errors may echo secrets. + if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")): + return None + raise AgentError("Docker rejected the requested operation") + return result.stdout.decode("utf-8", "replace") + + def inspect(self, name): + raw = self.call("container", "inspect", name, tolerate_missing=True) + return json.loads(raw)[0] if raw is not None else None + + +class Agent: + def __init__(self, config, docker=None): + self.config = config + self.root = Path(config.get("root", "/var/lib/electric-cherry")).resolve() + for directory in ("agent", "instances", "libraries"): + (self.root / directory).mkdir(parents=True, exist_ok=True) + os.chmod(self.root / "agent", 0o700) + self.journal = Journal(self.root / "agent" / "operations.sqlite3") + self.docker = docker or Docker() + self.stopping = threading.Event() + self.current_operation = None + self.library_progress = {} + self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") + self.executing_id = None + + def instance_dir(self, instance_id, generation): + instance_id, generation = instance_identity(instance_id, generation) + return self.root / "instances" / instance_id / str(generation) + + def container_name(self, instance_id, generation): + instance_id, generation = instance_identity(instance_id, generation) + return f"ec-{instance_id}-g{generation}" + + def inspect(self, instance_id, generation): + name = self.container_name(instance_id, generation) + info = self.docker.inspect(name) + if info is None: + return {"exists": False, "running": False, "healthy": False, "state": "missing", "instanceId": instance_id, "generation": generation} + labels = info.get("Config", {}).get("Labels", {}) or {} + if labels.get(LABEL + ".managed") != "true" or labels.get(LABEL + ".instance") != instance_id or labels.get(LABEL + ".generation") != str(generation): + raise AgentError("Container ownership or generation does not match") + state = info.get("State", {}) + status = state.get("Status", "unknown") + healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + if status == "running" and not healthy: + status = state.get("Health", {}).get("Status", "unknown") + return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")} + + def checkpoint(self, value): + if self.executing_id: + self.journal.checkpoint(self.executing_id, value) + + def mark_removed(self, instance_id, generation): + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if not marker.exists() or json.loads(marker.read_text())["generation"] <= generation: + atomic_json(marker, {"generation": generation, "removed": True}) + + def make_writable(self, path, directory=False): + if os.geteuid() == 0: + os.chown(path, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + os.chmod(path, 0o755 if directory else 0o644) + elif directory: + os.chmod(path, 0o777) + + def spec(self, instance_id, generation): + path = self.instance_dir(instance_id, generation) / "spec.json" + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + raise AgentError("Managed instance metadata is unavailable") from None + + def request(self, method, path, payload=None, credential=True): + url = self.config["controller"].rstrip("/") + path + if urllib.parse.urlsplit(url).scheme != "https" and not self.config.get("allowInsecureLocalhost", False): + raise AgentError("Controller URL must use HTTPS") + if urllib.parse.urlsplit(url).scheme != "https" and urllib.parse.urlsplit(url).hostname not in ("127.0.0.1", "localhost", "::1"): + raise AgentError("Insecure development transport is limited to loopback") + headers = {"Content-Type": "application/json", "User-Agent": "electric-cherry-agent/" + VERSION} + if credential: + headers["Authorization"] = "Bearer " + self.config["credential"] + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + raise AgentError("Controller redirects are not allowed") + with urllib.request.build_opener(NoRedirect()).open(req, timeout=35) as response: + return json.load(response) + except (OSError, ValueError): + raise AgentError("Controller request failed") from None + + def inventory(self): + usage = shutil.disk_usage(self.root) + memory = {} + try: + for line in Path("/proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + memory[key] = int(value.strip().split()[0]) * 1024 + except (OSError, ValueError): + pass + docker_version, docker_running = "", False + try: + info = json.loads(self.docker.call("info", "--format", "{{json .}}", timeout=15)) + docker_version, docker_running = info.get("ServerVersion", ""), True + except (AgentError, ValueError): + pass + libraries, caches = [], [] + for marker in (self.root / "libraries").glob("*/*/manifest.json"): + try: + value = json.loads(marker.read_text()) + libraries.append({"id": value["libraryId"], "state": "ready"}) + caches.append({"libraryVersionId": value["libraryId"], "status": "ready", "bytesDownloaded": value["size"], "bytesTotal": value["size"], "manifestSha256": value.get("manifestSha256")}) + except (OSError, ValueError, KeyError): + continue + progress = list(self.library_progress.values()) + libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress) + caches.extend(progress) + images = [] + if docker_running: + with contextlib.suppress(AgentError): + lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines() + image_ids = set() + for line in lines: + parts = line.split("|") + if len(parts) != 4: + continue + repository, tag, image_digest, image_id = parts + image_ids.add(image_id) + if repository != "" and tag != "": + images.append(repository + ":" + tag) + if repository != "" and image_digest != "": + images.append(repository + "@" + image_digest) + aliases = self.root / "agent" / "runtime-images.json" + if aliases.exists(): + with contextlib.suppress(OSError, ValueError): + images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) + os_name = platform.system() + with contextlib.suppress(OSError): + for line in Path("/etc/os-release").read_text().splitlines(): + if line.startswith("PRETTY_NAME="): + os_name = line.split("=", 1)[1].strip('"') + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + + def safe_to_disrupt(self, instance_id, generation): + health = self.inspect(instance_id, generation) + if health["state"] in ("missing", "stopped", "exited", "created", "dead"): + return + if not health["running"]: + raise Held("Container state is unknown; disruption held") + result = self.rcon(instance_id, generation, "status") + if not result.get("ok") or result.get("players") is None: + raise Held("Fresh local player occupancy is unavailable; disruption held") + if result["players"] != 0: + raise Held("Players are present; disruption held") + + def rcon(self, instance_id, generation, command): + if not isinstance(command, str) or not command or len(command) > 2048 or any(ch in command for ch in ("\x00", "\r", "\n")): + raise AgentError("Invalid RCON command") + saved = self.spec(instance_id, generation) + password = saved.get("rconPassword") or saved.get("env", {}).get("PLUTAINER_RCON_PASSWORD") + if not password: + return {"ok": False, "reason": "unconfigured", "players": None, "text": ""} + bind = saved.get("bindAddress", "0.0.0.0") + host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind + family = socket.AF_INET6 if ":" in host else socket.AF_INET + chunks = [] + with socket.socket(family, socket.SOCK_DGRAM) as sock: + sock.settimeout(3) + try: + sock.connect((host, int(saved["port"]))) + sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n") + while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024: + chunks.append(sock.recv(65535)) + sock.settimeout(0.15) + except socket.timeout: + pass + except OSError: + return {"ok": False, "reason": "unreachable", "players": None, "text": ""} + if any(not chunk.startswith(b"\xff\xff\xff\xff") for chunk in chunks): + return {"ok": False, "reason": "invalid-reply", "players": None, "text": ""} + text = "\n".join(chunk[4:].decode("utf-8", "replace").removeprefix("print\n") for chunk in chunks) + if not text or re.search(r"(bad|invalid|wrong).{0,20}(password|rcon)", text, re.I): + return {"ok": False, "reason": "timeout" if not text else "authentication", "players": None, "text": text} + players = status_players(text) if command == "status" else None + return {"ok": True, "text": text, "players": players} + + def validate_spec(self, payload): + game = payload.get("game") + if game not in ("t5", "t6"): + raise AgentError("Only T5 and T6 are supported") + library = self.library_dir(game, payload.get("libraryId")) + self.verify_library(library) + image = payload.get("image", "") + if not isinstance(image, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", image): + raise AgentError("Invalid runtime image") + port = payload.get("port") + allowed = self.config.get("portRange", {"min": 28960, "max": 29060}) + if isinstance(port, bool) or not isinstance(port, int) or not allowed["min"] <= port <= allowed["max"]: + raise AgentError("Port is outside the configured host range") + bind = payload.get("bindAddress", "0.0.0.0") + try: + import ipaddress + ipaddress.ip_address(bind) + except (ValueError, TypeError): + raise AgentError("Invalid local bind address") from None + env = dict(payload.get("env", {})) + if payload.get("rconPassword"): + env["PLUTAINER_RCON_PASSWORD"] = payload["rconPassword"] + if not isinstance(env, dict) or any(not re.fullmatch(r"[A-Z][A-Z0-9_]*", key) or not isinstance(value, str) or "\x00" in value or "\n" in value for key, value in env.items()): + raise AgentError("Invalid runtime environment") + # Runtime download is a separate preparation operation. A deployment + # uses the already prepared image even during a registry outage. + self.docker.call("image", "inspect", image) + files = payload.get("files", []) + if not isinstance(files, list) or len(files) > 10000: + raise Rejected("Invalid initial file list") + total, paths = 0, set() + for item in files: + scoped(self.root / "agent" / "validation", item["path"]) + if item["path"] in paths: + raise Rejected("Duplicate initial file path") + paths.add(item["path"]) + total += len(self.decode_file(item)) + if total > 64 * 1024 * 1024: + raise Rejected("Initial files exceed the 64 MiB operation limit") + return game, library, image, port, bind, env + + def create(self, instance_id, generation, payload): + directory = self.instance_dir(instance_id, generation) + current_marker = directory.parent / "current.json" + if current_marker.exists(): + current = json.loads(current_marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed"): + raise AgentError("Instance generation has been superseded or removed") + existing = self.inspect(instance_id, generation) + if existing["exists"]: + if self.spec(instance_id, generation) != payload: + raise AgentError("Instance already exists with a different specification") + return existing + game, library, image, port, bind, env = self.validate_spec(payload) + app = directory / "app" + app.mkdir(parents=True, exist_ok=True) + self.make_writable(app, directory=True) + self.write_initial_files(app, payload.get("files", [])) + atomic_json(directory / "spec.json", payload) + env_path = directory / "environment" + atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) + name = self.container_name(instance_id, generation) + spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) + args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] + bind_arg = "[" + bind + "]" if ":" in bind else bind + for protocol in ("udp", "tcp"): + args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) + args.append(image) + self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) + self.docker.call(*args, timeout=120) + self.docker.call("container", "start", name) + atomic_json(current_marker, {"generation": generation}) + return self.inspect(instance_id, generation) + + def write_initial_files(self, root, files): + if not isinstance(files, list) or len(files) > 10000: + raise AgentError("Invalid file list") + for item in files: + content = self.decode_file(item) + managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + + def decode_file(self, item): + try: + content = base64.b64decode(item["contentBase64"], validate=True) + except (ValueError, KeyError, TypeError): + raise AgentError("Invalid encoded file content") from None + if len(content) > MAX_FILE: + raise AgentError("File exceeds staging size limit") + if item.get("sha256") and item["sha256"] != digest(content): + raise AgentError("Staged file checksum mismatch") + return content + + def lifecycle(self, kind, instance_id, generation, payload): + if kind == "instance.create": + return self.create(instance_id, generation, payload) + if kind == "instance.inspect": + return self.inspect(instance_id, generation) + health = self.inspect(instance_id, generation) + name = self.container_name(instance_id, generation) + before = [] + if kind == "instance.start": + if not health["exists"]: + raise AgentError("Instance does not exist") + if payload.get("stageId"): + self.safe_to_disrupt(instance_id, generation) + self.checkpoint({"phase": "activating", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), "restartRequired": health["running"]}) + before = self.activate(instance_id, generation, payload["stageId"]) + health = self.inspect(instance_id, generation) + self.safe_to_disrupt(instance_id, generation) + restart_required = bool(payload.get("stageId") and health["running"]) + self.checkpoint({"phase": "starting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), "restartRequired": restart_required}) + if restart_required: + self.docker.call("container", "restart", "--time", "30", name) + else: + self.docker.call("container", "start", name) + elif kind in ("instance.stop", "instance.restart", "instance.remove", "instance.rebuild"): + if kind == "instance.rebuild": + next_generation = payload.get("nextGeneration", payload.get("newGeneration")) + if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation: + raise AgentError("Rebuild must use a newer immutable generation") + spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")} + self.validate_spec(spec) + self.safe_to_disrupt(instance_id, generation) + self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")}) + if payload.get("stageId") and kind == "instance.restart": + before = self.activate(instance_id, generation, payload["stageId"]) + if kind in ("instance.remove", "instance.rebuild"): + if health["exists"]: + self.docker.call("container", "stop", "--time", "30", name) + self.docker.call("container", "rm", name) + if kind == "instance.rebuild": + result = self.create(instance_id, next_generation, spec) + shutil.rmtree(self.instance_dir(instance_id, generation), ignore_errors=False) + return result + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + self.mark_removed(instance_id, generation) + elif health["exists"]: + if kind == "instance.restart" and payload.get("stageId"): + self.safe_to_disrupt(instance_id, generation) + self.docker.call("container", "stop" if kind.endswith("stop") else "restart", "--time", "30", name) + else: + raise AgentError("Unsupported instance operation") + return {**self.inspect(instance_id, generation), "before": before} + + def stage(self, instance_id, generation, payload, operation_id): + directory = self.instance_dir(instance_id, generation) + self.spec(instance_id, generation) + stage_id = identifier(payload.get("stageId", operation_id)) + stage = directory / "stages" / stage_id + files = payload.get("files") + if not isinstance(files, list) or not files or len(files) > 10000: + raise AgentError("Invalid staged file list") + planned = [] + paths = set() + total_size = 0 + for item in files: + scoped(directory / "app", item["path"]) + if item["path"] in paths: + raise AgentError("Duplicate staged file path") + paths.add(item["path"]) + deleting = item.get("delete") is True or item.get("contentBase64") is None + content = None if deleting else self.decode_file(item) + total_size += len(content) if content is not None else 0 + if total_size > 64 * 1024 * 1024: + raise Rejected("Staged file content exceeds the 64 MiB operation limit") + planned.append({"id": item.get("id"), "path": item["path"], "delete": deleting, "sha256": None if deleting else digest(content), "checkBefore": item.get("checkBefore", "expectedSha256" in item), "expectedBeforeSha": item.get("expectedBeforeSha", item.get("expectedSha256")), "contentBase64": None if deleting else base64.b64encode(content).decode()}) + manifest = {"stageId": stage_id, "files": planned, "state": "staged"} + marker = stage / "manifest.json" + if marker.exists(): + existing = json.loads(marker.read_text()) + if existing["files"] != planned: + raise AgentError("Stage ID already contains different files") + return {"stageId": stage_id, "files": len(planned), "state": existing["state"]} + atomic_json(marker, manifest) + return {"stageId": stage_id, "files": len(planned), "state": "staged"} + + def activate(self, instance_id, generation, stage_id): + directory = self.instance_dir(instance_id, generation) + stage = directory / "stages" / identifier(stage_id) + marker = stage / "manifest.json" + saved = json.loads(marker.read_text()) + if saved["state"] == "activated": + return saved["before"] + if saved["state"] != "staged": + raise AgentError("Interrupted activation requires explicit rollback") + before = [] + for item in saved["files"]: + content = managed_read(directory / "app", item["path"]) + actual_sha = None if content is None else digest(content) + if item.get("checkBefore") and actual_sha != item.get("expectedBeforeSha"): + raise Rejected("Target changed after campaign review") + before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()}) + # Snapshot preparation may take time. Occupancy is rechecked at the + # local activation boundary, after all file I/O and integrity checks. + self.safe_to_disrupt(instance_id, generation) + saved.update(state="activating", before=before) + atomic_json(marker, saved) + try: + for item in saved["files"]: + managed_write(directory / "app", item["path"], None if item["delete"] else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + saved["state"] = "activated" + atomic_json(marker, saved) + except Exception: + self.restore_files(directory / "app", before) + saved["state"] = "rolled-back" + atomic_json(marker, saved) + raise + return before + + def restore_files(self, app, before): + for item in before: + managed_write(app, item["path"], None if item["contentBase64"] is None else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + + def file_operation(self, kind, instance_id, generation, payload, operation_id): + directory = self.instance_dir(instance_id, generation) + app = directory / "app" + if kind == "files.stage": + return self.stage(instance_id, generation, payload, operation_id) + if kind in ("files.activate", "files.rollback"): + self.safe_to_disrupt(instance_id, generation) + stage_id = identifier(payload["stageId"]) + health = self.inspect(instance_id, generation) + self.checkpoint({"phase": "activating" if kind == "files.activate" else "rolling-back", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")}) + if kind == "files.activate": + before = self.activate(instance_id, generation, stage_id) + result = {"activated": True, "before": before} + else: + marker = directory / "stages" / stage_id / "manifest.json" + saved = json.loads(marker.read_text()) + if saved["state"] not in ("activated", "activating", "rolled-back"): + raise AgentError("Stage has no activation to roll back") + if saved["state"] != "rolled-back": + self.restore_files(app, saved["before"]) + saved["state"] = "rolled-back" + atomic_json(marker, saved) + result = {"rolledBack": True} + if payload.get("restart", True) and self.inspect(instance_id, generation)["exists"]: + self.safe_to_disrupt(instance_id, generation) + self.docker.call("container", "restart", "--time", "30", self.container_name(instance_id, generation)) + return result + if kind == "logs.read": + return self.read_log(instance_id, generation, payload) + path = scoped(app, payload.get("path", ""), allow_root=kind in ("files.list", "files.stat")) + if kind == "files.read": + content = managed_read(app, payload["path"]) + if content is None: + return {"contentBase64": None} + return {"contentBase64": base64.b64encode(content).decode(), "sha256": digest(content), "size": len(content)} + if kind == "files.stat": + if not path.exists(): + return None + stat = path.stat() + return {"size": stat.st_size, "mtime": stat.st_mtime, "isDirectory": path.is_dir(), "isFile": path.is_file()} + if kind == "files.list": + if not path.exists(): + return {"entries": []} + return {"entries": [{"name": child.name, "path": child.relative_to(app).as_posix(), "isDirectory": child.is_dir(), "size": child.stat().st_size} for child in sorted(path.iterdir()) if not child.is_symlink()][:10000]} + raise AgentError("Unsupported file operation") + + def read_log(self, instance_id, generation, payload): + app = self.instance_dir(instance_id, generation) / "app" + value = payload["path"] + count = min(2000, max(1, int(payload.get("lines", 200)))) + # Validate the original relative spelling independently of symlinks. + scoped(self.root / "agent" / "log-validation", value) + try: + resolved = (app / value).resolve(strict=True) + relative = resolved.relative_to(app.resolve(strict=True)).as_posix() + except FileNotFoundError: + if value != "logs/console.log" or not self.inspect(instance_id, generation)["exists"]: + return None + text = self.docker.call("container", "logs", "--tail", str(count), "--timestamps", self.container_name(instance_id, generation), timeout=30, merge_stderr=True) + lines = text.splitlines() + return {"lines": lines[-count:], "size": len(text.encode()), "mtime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "truncated": len(lines) >= count, "source": "container"} + except ValueError: + raise AgentError("Log symlink resolves outside the managed instance") from None + # Reopen the resolved relative target through no-follow directory FDs; + # swapping either the original link or a target component cannot escape. + with managed_open(app, relative) as stream: + stat = os.fstat(stream.fileno()) + start = max(0, stat.st_size - 256 * 1024) + stream.seek(start) + lines = stream.read(256 * 1024).decode("utf-8", "replace").splitlines() + if start: + lines = lines[1:] + return {"lines": lines[-count:], "size": stat.st_size, "mtime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(stat.st_mtime)), "truncated": start > 0} + + def library_dir(self, game, library_id): + if game not in ("t5", "t6"): + raise AgentError("Unsupported library game") + return self.root / "libraries" / game / identifier(library_id) + + def verify_library(self, directory): + marker = directory / "manifest.json" + if not marker.exists(): + raise AgentError("Required game library is not ready") + manifest = json.loads(marker.read_text()) + for item in manifest["files"]: + path = scoped(directory / "content", item["path"]) + if not path.is_file() or path.stat().st_size != item["size"] or file_digest(path) != item["sha256"]: + raise AgentError("Game library integrity verification failed") + return manifest + + def artifact_url(self, value): + controller = self.config["controller"].rstrip("/") + url = urllib.parse.urljoin(controller + "/", value) + parsed = urllib.parse.urlsplit(url) + origin = urllib.parse.urlsplit(controller) + if (parsed.scheme, parsed.netloc) != (origin.scheme, origin.netloc) or parsed.username or parsed.password: + raise AgentError("Library transport must remain on the authenticated controller origin") + if parsed.scheme != "https" and not (self.config.get("allowInsecureLocalhost") and parsed.hostname in ("127.0.0.1", "localhost", "::1")): + raise AgentError("Library transport must use HTTPS") + return url + + def artifact_open(self, method, path, data=None, extra_headers=None): + headers = {"Authorization": "Bearer " + self.config["credential"], **(extra_headers or {})} + req = urllib.request.Request(self.artifact_url(path), data=data, method=method, headers=headers) + # Redirects cannot forward host credentials to another origin. + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + raise AgentError("Artifact redirects are not allowed") + return urllib.request.build_opener(NoRedirect()).open(req, timeout=60) + + def library_download(self, payload): + library_id = identifier(payload.get("libraryVersionId", payload.get("libraryId"))) + with self.artifact_open("GET", payload["manifestUrl"]) as response: + manifest = json.load(response) + if manifest.get("id") != library_id or not isinstance(manifest.get("files"), list) or not manifest["files"]: + raise AgentError("Library manifest identity mismatch") + files = manifest["files"] + canonical = json.dumps([{ "path": item["path"], "size": item["size"], "sha256": item["sha256"] } for item in files], separators=(",", ":"), ensure_ascii=False) + manifest_sha = manifest.get("manifestSha256") + if not isinstance(manifest_sha, str) or not re.fullmatch("[0-9a-f]{64}", manifest_sha): + raise AgentError("Library manifest checksum is missing") + if digest(canonical.encode("utf-8")) != manifest_sha: + raise AgentError("Library manifest checksum mismatch") + if payload.get("manifestSha256") and payload["manifestSha256"] != manifest_sha: + raise AgentError("Library manifest differs from the requested immutable version") + directory = self.library_dir(manifest["game"], library_id) + if directory.exists(): + saved = self.verify_library(directory) + if saved.get("manifestSha256") != manifest_sha: + raise AgentError("Published library changed unexpectedly") + return {"libraryId": library_id, "state": "ready", "size": saved["size"]} + staging = directory.with_name("." + library_id + ".partial") + staging.mkdir(parents=True, exist_ok=True) + source_marker = staging / "source.json" + if source_marker.exists() and json.loads(source_marker.read_text()).get("manifestSha256") != manifest_sha: + raise Rejected("Partial library download belongs to a different manifest") + atomic_json(source_marker, {"libraryId": library_id, "manifestSha256": manifest_sha}) + size = sum(item["size"] for item in files) + downloaded = 0 + paths = set() + progress = {"libraryVersionId": library_id, "status": "downloading", "bytesDownloaded": 0, "bytesTotal": size, "manifestSha256": manifest_sha} + self.library_progress[library_id] = progress + try: + if shutil.disk_usage(staging).free < max(0, size - sum(path.stat().st_size for path in staging.rglob("*") if path.is_file())): + raise AgentError("Insufficient library storage") + for item in files: + if item["path"] in paths or not isinstance(item["size"], int) or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]): + raise AgentError("Invalid library file manifest") + paths.add(item["path"]) + target = scoped(staging / "content", item["path"]) + target.parent.mkdir(parents=True, exist_ok=True) + offset = target.stat().st_size if target.exists() else 0 + if offset > item["size"]: + target.unlink() + offset = 0 + if offset == item["size"] and target.exists() and file_digest(target) != item["sha256"]: + target.unlink() + offset = 0 + if offset < item["size"] or not target.exists(): + url = payload["filesBaseUrl"].rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/") + with self.artifact_open("GET", url, extra_headers={"Range": f"bytes={offset}-"} if offset else {}) as response: + if offset and response.status == 206: + content_range = response.headers.get("Content-Range", "") + if not content_range.startswith(f"bytes {offset}-"): + raise AgentError("Invalid resumed download response") + elif offset: + offset = 0 + with target.open("ab" if offset else "wb") as output: + received = offset + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + received += len(chunk) + if received > item["size"]: + raise AgentError("Library download exceeded its manifest size") + output.write(chunk) + progress["bytesDownloaded"] = downloaded + received + output.flush() + os.fsync(output.fileno()) + progress["status"] = "verifying" + if target.stat().st_size != item["size"] or file_digest(target) != item["sha256"]: + raise AgentError("Downloaded library file failed integrity verification") + downloaded += item["size"] + progress.update(status="downloading", bytesDownloaded=downloaded) + saved = {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha} + atomic_json(staging / "manifest.json", saved) + for path in (staging / "content").rglob("*"): + os.chmod(path, 0o555 if path.is_dir() else 0o444) + os.chmod(staging / "content", 0o555) + os.replace(staging, directory) + self.library_progress.pop(library_id, None) + return {"libraryId": library_id, "state": "ready", "size": size, "manifestSha256": manifest_sha} + except Exception: + progress.update(status="failed", error="Library download or integrity verification failed") + raise + + def library_import(self, payload): + game = payload.get("game") + volume = payload.get("sourceVolume") + if game not in ("t5", "t6") or volume != game + "_game_files": + raise AgentError("Only the explicitly approved game-file Docker volumes can be imported") + raw = self.docker.call("volume", "inspect", volume) + source = Path(json.loads(raw)[0]["Mountpoint"]) + if not source.is_dir() or source.is_symlink(): + raise AgentError("Import volume is unavailable") + files = [] + for path in sorted(source.rglob("*")): + if path.is_symlink(): + raise AgentError("Import source contains a symlink") + if path.is_file(): + files.append({"path": path.relative_to(source).as_posix(), "size": path.stat().st_size, "sha256": file_digest(path)}) + elif not path.is_dir(): + raise AgentError("Import source contains a special file") + if not files: + raise AgentError("Import volume is empty") + with self.artifact_open("PUT", payload["manifestUrl"], json.dumps({"files": files}).encode(), {"Content-Type": "application/json"}) as response: + response.read() + for item in files: + path = scoped(source, item["path"]) + if path.stat().st_size != item["size"] or file_digest(path) != item["sha256"]: + raise AgentError("Import source changed while publishing") + url = payload["uploadBaseUrl"].rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/") + with path.open("rb") as stream, self.artifact_open("PUT", url, stream, {"Content-Type": "application/octet-stream", "Content-Length": str(item["size"])}) as response: + response.read() + self.checkpoint({"phase": "publishing-import"}) + return self.complete_import(payload, start=True) + + def complete_import(self, payload, start=False): + method = "POST" if start else "GET" + while not self.stopping.is_set(): + with self.artifact_open(method, payload["completeUrl"], b"{}" if method == "POST" else None, {"Content-Type": "application/json"}) as response: + result = json.load(response) + if result.get("status") == "published": + return result + if result.get("status") == "failed": + raise Rejected("Controller rejected the game-file import after integrity verification") + if result.get("status") not in ("pending", "verifying"): + raise AgentError("Controller returned an invalid import publication state") + method = "POST" if result["status"] == "pending" else "GET" + self.stopping.wait(3) + raise AgentError("Agent stopped while the library import was being published") + + def dispatch(self, operation): + kind, payload = operation.get("kind"), operation.get("payload", {}) + if not isinstance(kind, str) or not isinstance(payload, dict): + raise AgentError("Invalid operation") + if kind == "credentials.rotate": + pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode() + self.config["pendingCredential"] = pending + atomic_json(self.config_path, self.config) + self.checkpoint({"phase": "rotating"}) + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + if not self.recover_credential(): + raise + self.config["credential"] = pending + self.config.pop("pendingCredential", None) + atomic_json(self.config_path, self.config) + return {"rotated": True} + if kind == "runtime.pull": + image = payload.get("image") + if not isinstance(image, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", image): + raise AgentError("Invalid runtime image") + self.docker.call("image", "pull", image, timeout=3600) + image_id = self.docker.call("image", "inspect", "--format", "{{.Id}}", image).strip() + aliases_file = self.root / "agent" / "runtime-images.json" + aliases = json.loads(aliases_file.read_text()) if aliases_file.exists() else {} + aliases[image] = image_id + atomic_json(aliases_file, aliases) + return {"image": image, "ready": True} + if kind == "host.inspect": + return self.inventory() + if kind == "library.download": + return self.library_download(payload) + if kind == "library.import": + return self.library_import(payload) + if kind == "library.inspect": + return self.verify_library(self.library_dir(payload["game"], payload["libraryId"])) + instance_id, generation = instance_identity(operation.get("instanceId"), operation.get("generation")) + current_marker = self.instance_dir(instance_id, generation).parent / "current.json" + if current_marker.exists(): + current = json.loads(current_marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"): + raise AgentError("Instance generation has been superseded or removed") + if kind.startswith("instance."): + return self.lifecycle(kind, instance_id, generation, payload) + if kind.startswith("files.") or kind == "logs.read": + return self.file_operation(kind, instance_id, generation, payload, operation["id"]) + if kind == "rcon.execute": + if payload["command"] not in ("status", "getstatus", "version"): + self.checkpoint({"phase": "rcon-command"}) + return self.rcon(instance_id, generation, payload["command"]) + raise AgentError("Unsupported operation kind") + + def execute(self, operation, report_running=False): + row, _ = self.journal.accept(operation) + if row["state"] == "unknown": + return self.reconcile(operation, row) + if row["state"] != "accepted": + return row + running = self.journal.update(operation["id"], "running") + if report_running: + try: + self.report(running) + except AgentError: + return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"}) + self.executing_id = operation["id"] + try: + result = self.dispatch(operation) + except Held as err: + return self.journal.update(operation["id"], "accepted", {"blocked": "occupied" if "Players" in str(err) else "unknown", "message": str(err)}) + except Exception as err: + # The command may already have changed Docker or files. Never infer + # failure means nothing happened, and never retry without inspection. + message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state" + checkpoint = self.journal.get(operation["id"]).get("checkpoint") + return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message) + finally: + self.executing_id = None + return self.journal.update(operation["id"], "completed", result) + + def recover_credential(self): + pending = self.config.get("pendingCredential") + if not pending: + return False + previous = self.config["credential"] + self.config["credential"] = pending + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + self.config["credential"] = previous + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + return False + self.config["credential"] = pending + self.config.pop("pendingCredential", None) + atomic_json(self.config_path, self.config) + return True + + def reconcile(self, operation, row): + """Prove desired state with immutable ownership; never resubmit a command.""" + kind, payload = operation["kind"], operation.get("payload", {}) + instance_id, generation = operation.get("instanceId"), operation.get("generation") + checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {} + try: + if kind == "credentials.rotate": + if not self.config.get("pendingCredential") or self.recover_credential(): + return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True}) + return row + if kind == "library.import": + try: + with self.artifact_open("GET", payload["completeUrl"]) as response: + status = json.load(response) + if status.get("status") == "published": + return self.journal.update(operation["id"], "completed", status) + if checkpoint.get("phase") == "publishing-import" or status.get("status") == "verifying": + return self.journal.update(operation["id"], "completed", self.complete_import(payload)) + # Import uploads use an immutable import ID and per-file + # checksums. Reconcile their staged files after interruption. + self.journal.update(operation["id"], "running") + self.executing_id = operation["id"] + return self.journal.update(operation["id"], "completed", self.library_import(payload)) + except Rejected as err: + return self.journal.update(operation["id"], "failed", error=str(err)) + except Exception: + return self.journal.update(operation["id"], "unknown", error="Import reconciliation is waiting for controller contact") + finally: + self.executing_id = None + if kind == "library.download": + library_id = payload.get("libraryVersionId", payload.get("libraryId")) + for game in ("t5", "t6"): + directory = self.library_dir(game, library_id) + if directory.exists(): + saved = self.verify_library(directory) + return self.journal.update(operation["id"], "completed", {"libraryId": library_id, "state": "ready", "size": saved["size"], "manifestSha256": saved.get("manifestSha256"), "reconciled": True}) + # This is a resumable immutable transfer, not a repeated + # container command: verified completed files remain untouched. + self.journal.update(operation["id"], "running") + try: + return self.journal.update(operation["id"], "completed", self.library_download(payload)) + except Exception: + return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed") + if kind == "files.stage": + return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"])) + if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"): + return self.journal.update(operation["id"], "completed", self.dispatch(operation)) + if kind in ("files.activate", "files.rollback") and checkpoint: + stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json" + saved = json.loads(stage.read_text()) + expected = "activated" if kind == "files.activate" else "rolled-back" + if saved["state"] == expected: + health = self.inspect(instance_id, generation) + restarted = health["running"] and health.get("containerId") == checkpoint.get("containerId") and bool(checkpoint.get("startedAt")) and health.get("startedAt") != checkpoint["startedAt"] + if not payload.get("restart", True) or restarted: + return self.journal.update(operation["id"], "completed", {"activated": expected == "activated", "rolledBack": expected == "rolled-back", "before": saved["before"], "reconciled": True}) + return row + if not kind.startswith("instance.") or not checkpoint: + return row + observed_generation = payload.get("nextGeneration", payload.get("newGeneration")) if kind == "instance.rebuild" else generation + health = self.inspect(instance_id, observed_generation) + before = [] + if payload.get("stageId"): + stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json" + saved = json.loads(stage.read_text()) + if saved["state"] != "activated": + return row + before = saved["before"] + proven = False + if kind in ("instance.create", "instance.rebuild"): + proven = health["running"] and health.get("creationOperation") == operation["id"] and health.get("specSha256") == checkpoint.get("specSha256") + if proven: + directory = self.instance_dir(instance_id, observed_generation) + atomic_json(directory.parent / "current.json", {"generation": observed_generation}) + if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: + old = self.instance_dir(instance_id, generation) + if old.exists(): + shutil.rmtree(old) + elif kind in ("instance.start", "instance.stop"): + proven = health.get("containerId") == checkpoint.get("containerId") and (health["running"] if kind == "instance.start" else health["state"] in ("stopped", "exited", "created", "dead")) + if kind == "instance.start": + proven = proven and checkpoint.get("phase") == "starting" + if checkpoint.get("restartRequired"): + proven = proven and bool(checkpoint.get("startedAt")) and bool(health.get("startedAt")) and health["startedAt"] != checkpoint["startedAt"] + elif kind == "instance.restart": + proven = health["running"] and health.get("containerId") == checkpoint.get("containerId") and bool(checkpoint.get("startedAt")) and bool(health.get("startedAt")) and health["startedAt"] != checkpoint["startedAt"] + elif kind == "instance.remove": + proven = not health["exists"] + if proven: + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + self.mark_removed(instance_id, generation) + if proven: + return self.journal.update(operation["id"], "completed", {**health, "before": before, "reconciled": True}) + except (AgentError, OSError, ValueError, KeyError): + pass + return row + + def report(self, row): + payload = {"state": row["state"]} + if row.get("result"): + payload["result"] = json.loads(row["result"]) + if row["state"] == "accepted" and isinstance(payload["result"], dict) and payload["result"].get("blocked"): + # The controller sees a durable running/waiting operation; + # locally accepted means its mutation gate can safely retry. + payload["state"] = "running" + if row.get("error"): + payload["error"] = row["error"] + return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload) + + def heartbeat(self): + response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation}) + host = response.get("host", {}) + if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int): + self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]} + + def heartbeat_loop(self): + while not self.stopping.wait(15): + try: + self.heartbeat() + except AgentError: + pass + + def run(self): + if self.config.get("pendingCredential"): + self.recover_credential() + # Obtain the configured host port range before accepting any deployment. + # Enrollment's default values must never race the first heartbeat. + while not self.stopping.is_set(): + try: + self.heartbeat() + break + except AgentError: + self.stopping.wait(3) + heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True) + heartbeats.start() + while not self.stopping.is_set(): + try: + if self.config.get("pendingCredential"): + self.recover_credential() + response = self.request("GET", "/api/agent/operations") + for operation in response.get("operations", []): + if self.stopping.is_set(): + break + self.current_operation = operation["id"] + accepted, _ = self.journal.accept(operation) + self.report(accepted) + self.report(self.execute(operation, report_running=True)) + self.current_operation = None + except (AgentError, OSError, ValueError, KeyError): + # Logs deliberately omit URLs, tokens, payloads and raw errors. + print("Host controller contact unavailable; existing containers continue running", file=sys.stderr) + self.stopping.wait(3) + heartbeats.join(timeout=2) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", default="/etc/electric-cherry/agent.json") + parser.add_argument("--enroll", action="store_true") + parser.add_argument("--controller") + parser.add_argument("--token-file") + parser.add_argument("--root", default="/var/lib/electric-cherry") + parser.add_argument("--force-enroll", action="store_true") + parser.add_argument("--version", action="version", version=VERSION) + args = parser.parse_args() + if args.enroll: + if not args.controller or not args.token_file: + parser.error("Enrollment needs --controller and --token-file") + if Path(args.config).exists() and not args.force_enroll: + existing = json.loads(Path(args.config).read_text()) + if existing.get("credential"): + if existing.get("controller", "").rstrip("/") != args.controller.rstrip("/") or Path(existing.get("root", "")).resolve() != Path(args.root).resolve(): + raise AgentError("Existing enrollment uses a different controller or data directory") + print("Host is already enrolled; retaining existing credentials") + return + agent = Agent({"controller": args.controller.rstrip("/"), "root": args.root}) + token = Path(args.token_file).read_text().strip() + enrolled = agent.request("POST", "/api/agent/enroll", {"token": token, "inventory": agent.inventory()}, credential=False) + if not enrolled.get("hostId") or not enrolled.get("credential"): + raise AgentError("Controller returned invalid enrollment credentials") + atomic_json(args.config, {"controller": args.controller.rstrip("/"), "root": args.root, "hostId": enrolled["hostId"], "credential": enrolled["credential"], "portRange": enrolled.get("portRange", {"min": 28960, "max": 29060})}) + agent.journal.close() + print("Host enrollment completed") + return + config = json.loads(Path(args.config).read_text()) + config["configPath"] = args.config + lock_root = Path(config.get("root", "/var/lib/electric-cherry")) / "agent" + lock_root.mkdir(parents=True, exist_ok=True) + lock = open(lock_root / "process.lock", "a") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise AgentError("Another host agent already owns this data directory") from None + agent = Agent(config) + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: agent.stopping.set()) + try: + agent.run() + finally: + agent.journal.close() + lock.close() + + +if __name__ == "__main__": + try: + main() + except (AgentError, OSError, ValueError, KeyError): + print("Electric Cherry host agent failed; check configuration and controller availability", file=sys.stderr) + sys.exit(1) diff --git a/agent/install.sh b/agent/install.sh new file mode 100644 index 0000000..d96601d --- /dev/null +++ b/agent/install.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Electric Cherry host installer. Downloaded release integrity is checked before +# execution. Existing Docker data and unrelated containers are never changed. +set -euo pipefail +umask 077 + +controller='' +enrollment_token='' +data_dir='/var/lib/electric-cherry' +reenroll='' +while [ "$#" -gt 0 ]; do + case "$1" in + --controller) controller="${2:?Missing controller URL}"; shift 2 ;; + --token) enrollment_token="${2:?Missing enrollment token}"; shift 2 ;; + --data-dir) data_dir="${2:?Missing data directory}"; shift 2 ;; + --reenroll) reenroll='--force-enroll'; shift ;; + *) echo "Unknown installer option" >&2; exit 2 ;; + esac +done +[ "$(id -u)" = 0 ] || { echo 'Run the installer as root.' >&2; exit 1; } +case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac +[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; } +case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac +case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac +[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; } +. /etc/os-release +[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; } +case "${VERSION_ID:-}" in 12|13) ;; *) echo 'Supported platform: Debian 12 or 13.' >&2; exit 1 ;; esac +[ "$(dpkg --print-architecture)" = amd64 ] || { echo 'Supported architecture: amd64.' >&2; exit 1; } +command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; } + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends ca-certificates curl python3 +python3 - "$controller" "$data_dir" <<'PY' +import json, pathlib, sys +config = pathlib.Path('/etc/electric-cherry/agent.json') +if config.exists(): + saved = json.loads(config.read_text()) + if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve(): + raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.') +PY +if ! command -v docker >/dev/null; then + if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then + echo 'Existing Docker data/configuration detected without a Docker CLI. Repair Docker, then rerun.' >&2 + exit 1 + fi + apt-get install -y --no-install-recommends docker.io + systemctl enable --now docker +fi +docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; } +install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances" +install -d -m 0700 "$data_dir/agent" /etc/electric-cherry +available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}') +[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; } +temporary_dir=$(mktemp -d) +trap 'rm -rf "$temporary_dir"' EXIT +controller="${controller%/}" +curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json" +curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py" +python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY' +import hashlib, json, pathlib, sys +manifest = json.loads(pathlib.Path(sys.argv[1]).read_text()) +actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest() +if actual != manifest.get('sha256'): + raise SystemExit('Agent release checksum verification failed.') +if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1: + raise SystemExit('Unsupported agent release manifest.') +PY +# Stop only the agent while replacing its own executable/journal owner. Existing +# game containers keep running under Docker throughout installation and updates. +if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then + systemctl stop electric-cherry-agent.service +fi +install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || { + install -d -m 0755 /usr/local/libexec + install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py +} +printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token" +unset enrollment_token +python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"} +cat > /etc/systemd/system/electric-cherry-agent.service <&2; exit 1; } +echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.' diff --git a/agent/package_release.py b/agent/package_release.py new file mode 100644 index 0000000..64ec0af --- /dev/null +++ b/agent/package_release.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +"""Publish the reviewed local agent source into the controller static release.""" +import hashlib +import json +from pathlib import Path +import shutil + +source = Path(__file__).resolve().parent +target = source.parent / "static" / "agent" +target.mkdir(parents=True, exist_ok=True) +shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py") +shutil.copyfile(source / "install.sh", target / "install.sh") +(target / "manifest.json").write_text(json.dumps({"version": "1.0.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n") diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py new file mode 100644 index 0000000..2739da8 --- /dev/null +++ b/agent/tests/test_agent.py @@ -0,0 +1,547 @@ +import base64 +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("ec_agent", Path(__file__).resolve().parents[1] / "ec_agent.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +Agent, AgentError, Held, Journal = module.Agent, module.AgentError, module.Held, module.Journal +INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542" + + +class FakeDocker: + def __init__(self): + self.containers = {} + self.calls = [] + self.fail_restart = False + self.lose_reply_after = None + self.started = 0 + + def inspect(self, name): + return self.containers.get(name) + + def call(self, *args, **kwargs): + self.calls.append(args) + if args[0] == "info": + return json.dumps({"ServerVersion": "27.5.1"}) + if args[:2] == ("image", "ls"): + return "ghcr.io/ayymoss/plutainer:latest\n" + if args[:2] == ("container", "create"): + labels = {args[index + 1].split("=", 1)[0]: args[index + 1].split("=", 1)[1] for index, arg in enumerate(args) if arg == "--label"} + name = args[args.index("--name") + 1] + self.containers[name] = {"Id": "container-123", "Config": {"Labels": labels}, "State": {"Status": "created", "Running": False}} + if args[:2] in (("container", "start"), ("container", "restart")): + if self.fail_restart and args[1] == "restart": + raise AgentError("Docker command unavailable or timed out") + self.started += 1 + self.containers[args[-1]]["State"] = {"Status": "running", "Running": True, "StartedAt": f"started-{self.started}"} + if args[:2] == ("container", "stop"): + self.containers[args[-1]]["State"] = {"Status": "exited", "Running": False} + if args[:2] == ("container", "rm"): + self.containers.pop(args[-1]) + if args[0] == "container" and args[1] == self.lose_reply_after: + raise AgentError("Docker command unavailable or timed out") + return "" + + +class Response(io.BytesIO): + def __init__(self, content, status=200, headers=None): + super().__init__(content) + self.status = status + self.headers = headers or {} + + +class AgentTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.config = {"root": self.temp.name, "controller": "https://controller.example", "credential": "secret", "portRange": {"min": 28960, "max": 28980}} + self.docker = FakeDocker() + self.agent = Agent(self.config, self.docker) + self.library_id = "library-one" + self.library = self.agent.library_dir("t5", self.library_id) + (self.library / "content").mkdir(parents=True) + (self.library / "content" / "game.ff").write_bytes(b"game-data") + files = [{"path": "game.ff", "size": 9, "sha256": module.digest(b"game-data")}] + module.atomic_json(self.library / "manifest.json", {"libraryId": self.library_id, "game": "t5", "size": 9, "files": files, "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())}) + + def tearDown(self): + self.agent.journal.close() + self.temp.cleanup() + + def operation(self, kind, payload=None, op_id="operation-1", generation=1): + return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}} + + def create(self): + payload = {"name": "t5_zombie_test_1", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {"PLUTAINER_PORT": "28960"}, "rconPassword": "rcon-secret", "files": [{"path": "configs/server.cfg", "contentBase64": base64.b64encode(b"original").decode()}]} + result = self.agent.execute(self.operation("instance.create", payload, "create")) + self.assertEqual(result["state"], "completed", result) + return payload + + def test_same_id_reports_saved_result_without_executing_again(self): + self.create() + operation = self.operation("instance.restart") + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + first = self.agent.execute(operation) + self.assertEqual(first["state"], "completed") + second = self.agent.execute({**operation, "state": "running", "updatedAt": "later", "result": None}) + self.assertEqual(second["result"], first["result"]) + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1) + + def test_reusing_id_with_different_payload_is_rejected(self): + self.agent.execute(self.operation("host.inspect")) + with self.assertRaisesRegex(AgentError, "different content"): + self.agent.execute(self.operation("host.inspect", {"changed": True})) + + def test_agent_restart_marks_running_unknown_and_never_replays(self): + operation = self.operation("instance.restart") + self.agent.journal.accept(operation) + self.agent.journal.update(operation["id"], "running") + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + result = self.agent.execute(operation) + self.assertEqual(result["state"], "unknown") + self.assertEqual(self.docker.calls, []) + + def test_occupied_and_unknown_wait_then_revalidate_same_operation(self): + self.create() + operation = self.operation("instance.restart") + for players in (2, None, 7): + self.agent.rcon = lambda *_, count=players: {"ok": True, "players": count} + result = self.agent.execute(operation) + self.assertEqual(result["state"], "accepted") + self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls)) + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.assertEqual(self.agent.execute(operation)["state"], "completed") + + def test_unknown_container_health_cannot_disrupt(self): + self.create() + self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"] = {"Status": "unknown", "Running": False} + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.assertEqual(self.agent.execute(self.operation("instance.stop"))["state"], "accepted") + self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) + + def test_files_stage_is_nondisruptive_activation_snapshots_and_rollback(self): + self.create() + target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg" + staged = {"stageId": "campaign", "files": [{"id": 9, "path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode(), "sha256": module.digest(b"new"), "checkBefore": True, "expectedBeforeSha": module.digest(b"original")}]} + self.agent.execute(self.operation("files.stage", staged, "stage")) + self.assertEqual(target.read_bytes(), b"original") + self.agent.rcon = lambda *_: {"ok": True, "players": 3} + operation = self.operation("instance.restart", {"stageId": "campaign"}) + self.assertEqual(self.agent.execute(operation)["state"], "accepted") + self.assertEqual(target.read_bytes(), b"original") + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + result = json.loads(self.agent.execute(operation)["result"]) + self.assertEqual(base64.b64decode(result["before"][0]["contentBase64"]), b"original") + self.assertEqual(target.read_bytes(), b"new") + rollback = self.agent.execute(self.operation("files.rollback", {"stageId": "campaign", "restart": False}, "rollback")) + self.assertEqual(rollback["state"], "completed") + self.assertEqual(target.read_bytes(), b"original") + + def test_changed_target_fails_before_activation_or_restart(self): + self.create() + target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg" + self.agent.execute(self.operation("files.stage", {"stageId": "campaign", "files": [{"path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode(), "checkBefore": True, "expectedBeforeSha": "0" * 64}]}, "stage")) + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + result = self.agent.execute(self.operation("instance.restart", {"stageId": "campaign"})) + self.assertEqual(result["state"], "failed") + self.assertEqual(target.read_bytes(), b"original") + self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls)) + + def test_ambiguous_restart_keeps_before_images_on_disk_and_is_not_replayed(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.agent.execute(self.operation("files.stage", {"stageId": "campaign", "files": [{"path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode()}]}, "stage")) + self.docker.fail_restart = True + operation = self.operation("instance.restart", {"stageId": "campaign"}) + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.agent.execute(operation) + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1) + saved = json.loads((self.agent.instance_dir(INSTANCE, 1) / "stages/campaign/manifest.json").read_text()) + self.assertEqual(saved["state"], "activated") + self.assertEqual(base64.b64decode(saved["before"][0]["contentBase64"]), b"original") + + def test_remove_only_own_instance_and_preserves_shared_library(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + result = self.agent.execute(self.operation("instance.remove")) + self.assertEqual(result["state"], "completed") + self.assertTrue((self.library / "content/game.ff").exists()) + self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + + def test_unowned_container_never_mutated(self): + self.create() + self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {} + self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed") + self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) + + def test_rebuild_moves_generation_and_rejects_old_commands(self): + payload = self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + result = self.agent.execute(self.operation("instance.rebuild", {**payload, "nextGeneration": 2})) + self.assertEqual(result["state"], "completed", result) + self.assertEqual(json.loads(result["result"])["generation"], 2) + self.assertEqual(self.agent.execute(self.operation("instance.stop", op_id="old-stop"))["state"], "failed") + self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"]) + + def test_traversal_and_symlinks_are_rejected(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + (app / "escape").symlink_to(self.temp.name) + for path in ("../spec.json", "/etc/passwd", "escape/secret", "configs/../../spec.json"): + with self.assertRaises(AgentError): + module.scoped(app, path) + + def manifest(self, content=b"complete-game-file"): + files = [{"path": "zone/test.ff", "size": len(content), "sha256": module.digest(content)}] + return {"id": "downloaded", "game": "t6", "files": files, "sizeBytes": len(content), "manifestSha256": module.digest(json.dumps(files, separators=(",", ":")).encode())} + + def test_resume_verifies_and_atomically_publishes_library(self): + content = b"complete-game-file" + manifest = self.manifest(content) + staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone" + staging.mkdir(parents=True) + (staging / "test.ff").write_bytes(content[:8]) + calls = [] + def fetch(method, path, data=None, extra_headers=None): + calls.append((path, extra_headers)) + if path == "/manifest": + return Response(json.dumps(manifest).encode()) + return Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"}) + self.agent.artifact_open = fetch + result = self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}) + self.assertEqual(result["state"], "ready") + self.assertEqual(calls[1][1], {"Range": "bytes=8-"}) + self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content/zone/test.ff").read_bytes(), content) + self.assertEqual(self.agent.inventory()["libraryCaches"][-1]["status"], "ready") + + def test_corrupt_or_incomplete_library_never_promoted(self): + manifest = self.manifest() + def fetch(method, path, data=None, extra_headers=None): + return Response(json.dumps(manifest).encode() if path == "/manifest" else b"corrupt") + self.agent.artifact_open = fetch + with self.assertRaisesRegex(AgentError, "integrity"): + self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}) + self.assertFalse(self.agent.library_dir("t6", "downloaded").exists()) + self.assertEqual(self.agent.library_progress["downloaded"]["status"], "failed") + + def test_manifest_tampering_rejected_before_download(self): + manifest = self.manifest() + manifest["files"][0]["size"] = 999 + self.agent.artifact_open = lambda *_: Response(json.dumps(manifest).encode()) + with self.assertRaisesRegex(AgentError, "manifest checksum"): + self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}) + + def test_artifact_urls_never_leak_credentials_to_other_origins(self): + for value in ("https://attacker.example/files", "//attacker.example/files", "http://controller.example/files", "https://user:pass@controller.example/files"): + with self.assertRaises(AgentError): + self.agent.artifact_url(value) + + def test_unsupported_operation_does_not_spawn_shell(self): + result = self.agent.execute(self.operation("shell.execute", {"command": "touch /tmp/unsafe"})) + self.assertEqual(result["state"], "failed") + self.assertEqual(self.docker.calls, []) + + def test_restart_lost_acknowledgement_recovers_from_changed_start_timestamp(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.docker.lose_reply_after = "restart" + operation = self.operation("instance.restart") + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + recovered = self.agent.execute(operation) + self.assertEqual(recovered["state"], "completed") + self.assertTrue(json.loads(recovered["result"])["reconciled"]) + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1) + + def test_created_container_lost_start_reply_reconciles_immutable_operation_label(self): + payload = {"name": "t5_zombie_test_1", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {}, "rconPassword": "secret", "files": []} + self.docker.lose_reply_after = "start" + operation = self.operation("instance.create", payload) + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + result = self.agent.execute(operation) + self.assertEqual(result["state"], "completed") + self.assertEqual(sum(call[:2] == ("container", "create") for call in self.docker.calls), 1) + + def test_remove_lost_reply_reconciles_absence_without_duplicate_delete(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.docker.lose_reply_after = "rm" + operation = self.operation("instance.remove") + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + self.assertEqual(sum(call[:2] == ("container", "rm") for call in self.docker.calls), 1) + + def test_rotation_recovers_lost_reply_using_durably_saved_pending_credential(self): + self.agent.config_path = str(Path(self.temp.name) / "agent.json") + accepted = {} + def request(method, path, payload): + if not accepted: + accepted["credential"] = payload["credential"] + persisted = json.loads(Path(self.agent.config_path).read_text()) + self.assertEqual(persisted["pendingCredential"], payload["credential"]) + raise AgentError("Controller request failed") + self.assertEqual(self.agent.config["credential"], accepted["credential"]) + return {"ok": True} + self.agent.request = request + result = self.agent.execute(self.operation("credentials.rotate")) + self.assertEqual(result["state"], "completed") + saved = json.loads(Path(self.agent.config_path).read_text()) + self.assertNotIn("pendingCredential", saved) + self.assertEqual(saved["credential"], accepted["credential"]) + + def test_status_parser_never_treats_partial_or_unfamiliar_rows_as_empty(self): + header = "map: zombie_cosmodrome\nnum score ping guid name\n" + self.assertIsNone(module.status_players(header)) + self.assertEqual(module.status_players(header + "--- ----- ---- ---- ----\n"), 0) + for row in ("0 0 CNCT abc player", "1 90 ZMBI abc player", "2 4 ???? abc player"): + self.assertEqual(module.status_players(header + "--- ----- ---- ---- ----\n" + row), 1) + self.assertIsNone(module.status_players(header + "--- ----- ---- ---- ----\nconnection in progress")) + self.assertEqual(module.status_players('statusResponse\n\\mapname\\mp_kino\\sv_maxclients\\4\n10 45 "joe"'), 1) + + def test_udp_rcon_uses_host_local_endpoint_and_returns_structured_occupancy(self): + self.create() + class FakeSocket: + def __enter__(self): return self + def __exit__(self, *_): pass + def settimeout(self, value): pass + def connect(self, endpoint): self.endpoint = endpoint + def send(self, packet): self.packet = packet + def recv(self, _size): + if getattr(self, "done", False): raise module.socket.timeout() + self.done = True + return b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n0 42 CNCT abc player\n" + sock = FakeSocket() + with patch.object(module.socket, "socket", return_value=sock): + result = self.agent.rcon(INSTANCE, 1, "status") + self.assertEqual(sock.endpoint, ("127.0.0.1", 28960)) + self.assertTrue(sock.packet.startswith(b"\xff\xff\xff\xffrcon rcon-secret status")) + self.assertEqual(result["players"], 1) + self.assertTrue(result["ok"]) + + def test_invalid_rebuild_preflight_preserves_running_original(self): + payload = self.create() + result = self.agent.execute(self.operation("instance.rebuild", {**payload, "image": "--privileged", "nextGeneration": 2})) + self.assertEqual(result["state"], "failed") + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) + + def test_file_operation_resists_parent_symlink_swap_after_validation(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + outside = Path(self.temp.name) / "outside" + outside.mkdir() + (outside / "server.cfg").write_bytes(b"host-data") + original = module.scoped + def swap(root, path, **kwargs): + result = original(root, path, **kwargs) + (app / "configs").rename(app / "saved-configs") + (app / "configs").symlink_to(outside) + return result + with patch.object(module, "scoped", side_effect=swap): + with self.assertRaises(OSError): + module.managed_write(app, "configs/server.cfg", b"escape-attempt") + self.assertEqual((outside / "server.cfg").read_bytes(), b"host-data") + + def test_deleting_absent_file_with_absent_parent_is_idempotent(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + module.managed_write(app, "missing/subdirectory/file.gsc", None) + self.assertFalse((app / "missing").exists()) + + def test_controller_contact_failure_holds_before_running_any_docker_command(self): + self.create() + count = len(self.docker.calls) + self.agent.report = lambda *_: (_ for _ in ()).throw(AgentError("Controller request failed")) + result = self.agent.execute(self.operation("instance.stop"), report_running=True) + self.assertEqual(result["state"], "accepted") + self.assertEqual(len(self.docker.calls), count) + + def test_async_import_publication_resumes_after_controller_restart(self): + responses = iter([{"status": "verifying"}, {"status": "pending"}, {"status": "verifying"}, {"status": "published", "version": {"id": "library-one"}}]) + methods = [] + def request(method, path, *args): + methods.append(method) + return Response(json.dumps(next(responses)).encode()) + self.agent.artifact_open = request + with patch.object(self.agent.stopping, "wait", return_value=False): + result = self.agent.complete_import({"completeUrl": "/complete"}, start=True) + self.assertEqual(result["status"], "published") + self.assertEqual(methods, ["POST", "GET", "POST", "GET"]) + + def test_digest_pinned_runtime_inventory_and_exact_aliases(self): + image_digest = "sha256:" + "1" * 64 + image_id = "sha256:" + "2" * 64 + original = self.docker.call + def call(*args, **kwargs): + if args[:2] == ("image", "ls"): + return f"ghcr.io/ayymoss/plutainer|latest|{image_digest}|{image_id}\n" + return original(*args, **kwargs) + self.docker.call = call + module.atomic_json(self.agent.root / "agent/runtime-images.json", {"ghcr.io/ayymoss/plutainer": image_id, "removed-image": "missing-id"}) + images = self.agent.inventory()["runtimeImages"] + self.assertIn("ghcr.io/ayymoss/plutainer@" + image_digest, images) + self.assertIn("ghcr.io/ayymoss/plutainer:latest", images) + self.assertIn("ghcr.io/ayymoss/plutainer", images) + self.assertNotIn("removed-image", images) + + def test_heartbeat_applies_configured_host_port_range_before_deployment(self): + self.agent.request = lambda *_: {"host": {"portMin": 30000, "portMax": 30100}} + self.agent.heartbeat() + self.assertEqual(self.agent.config["portRange"], {"min": 30000, "max": 30100}) + + def test_download_content_permissions_allow_container_read_access_with_private_umask(self): + content = b"complete-game-file" + manifest = self.manifest(content) + self.agent.artifact_open = lambda method, path, **kwargs: Response(json.dumps(manifest).encode() if path == "/manifest" else content) + previous = module.os.umask(0o077) + try: + self.agent.library_download({"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files"}) + finally: + module.os.umask(previous) + content_root = self.agent.library_dir("t6", "downloaded") / "content" + self.assertEqual(content_root.stat().st_mode & 0o777, 0o555) + self.assertEqual((content_root / "zone").stat().st_mode & 0o777, 0o555) + self.assertEqual((content_root / "zone/test.ff").stat().st_mode & 0o777, 0o444) + + def test_agent_restart_resumes_same_immutable_library_operation(self): + content = b"complete-game-file" + manifest = self.manifest(content) + payload = {"libraryVersionId": "downloaded", "manifestUrl": "/manifest", "filesBaseUrl": "/files", "manifestSha256": manifest["manifestSha256"]} + operation = self.operation("library.download", payload) + self.agent.journal.accept(operation) + self.agent.journal.update(operation["id"], "running") + staging = self.agent.root / "libraries/t6/.downloaded.partial/content/zone" + staging.mkdir(parents=True) + (staging / "test.ff").write_bytes(content[:8]) + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + def fetch(method, path, data=None, extra_headers=None): + if path == "/manifest": + return Response(json.dumps(manifest).encode()) + self.assertEqual(extra_headers, {"Range": "bytes=8-"}) + return Response(content[8:], 206, {"Content-Range": f"bytes 8-{len(content)-1}/{len(content)}"}) + self.agent.artifact_open = fetch + result = self.agent.execute(operation) + self.assertEqual(result["state"], "completed") + self.assertEqual((self.agent.library_dir("t6", "downloaded") / "content/zone/test.ff").read_bytes(), content) + + def test_player_join_during_activation_holds_restart_without_reapplying_files(self): + self.create() + target = self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg" + original_write = module.managed_write + for index, kind in enumerate(("instance.restart", "files.activate", "instance.start")): + with self.subTest(kind=kind): + before = target.read_bytes() + stage_id = "late-join-" + str(index) + content = ("new-" + str(index)).encode() + self.agent.execute(self.operation("files.stage", {"stageId": stage_id, "files": [{"id": index, "path": "configs/server.cfg", "contentBase64": base64.b64encode(content).decode()}]}, "stage-late-" + str(index))) + joined, writes = [False], [] + self.agent.rcon = lambda *_: {"ok": True, "players": 1 if joined[0] else 0} + def write_then_join(*args, **kwargs): + original_write(*args, **kwargs) + writes.append(args) + joined[0] = True + operation = self.operation(kind, {"stageId": stage_id}, "action-late-" + str(index)) + restarts_before = sum(call[:2] == ("container", "restart") for call in self.docker.calls) + with patch.object(module, "managed_write", side_effect=write_then_join): + held = self.agent.execute(operation) + self.assertEqual(held["state"], "accepted") + self.assertEqual(json.loads(held["result"])["blocked"], "occupied") + self.assertEqual(target.read_bytes(), content) + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), restarts_before) + joined[0] = False + completed = self.agent.execute(operation) + self.assertEqual(completed["state"], "completed") + self.assertEqual(len(writes), 1) + self.assertEqual(base64.b64decode(json.loads(completed["result"])["before"][0]["contentBase64"]), before) + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), restarts_before + 1) + + def test_player_join_during_rollback_holds_restart_without_repeating_restore(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.agent.execute(self.operation("files.stage", {"stageId": "rollback-join", "files": [{"path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode()}]}, "stage-rollback-join")) + self.agent.execute(self.operation("files.activate", {"stageId": "rollback-join", "restart": False}, "activate-rollback-join")) + joined, writes = [False], [] + original_write = module.managed_write + self.agent.rcon = lambda *_: {"ok": True, "players": 1 if joined[0] else 0} + def restore_then_join(*args, **kwargs): + original_write(*args, **kwargs) + writes.append(args) + joined[0] = True + operation = self.operation("files.rollback", {"stageId": "rollback-join"}, "rollback-join") + with patch.object(module, "managed_write", side_effect=restore_then_join): + self.assertEqual(self.agent.execute(operation)["state"], "accepted") + self.assertFalse(any(call[:2] == ("container", "restart") for call in self.docker.calls)) + joined[0] = False + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertEqual(len(writes), 1) + self.assertEqual((self.agent.instance_dir(INSTANCE, 1) / "app/configs/server.cfg").read_bytes(), b"original") + + def test_start_with_stage_does_not_reconcile_success_before_required_restart(self): + self.create() + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.agent.execute(self.operation("files.stage", {"stageId": "start-stage", "files": [{"path": "configs/server.cfg", "contentBase64": base64.b64encode(b"new").decode()}]}, "stage-start")) + self.docker.fail_restart = True + operation = self.operation("instance.start", {"stageId": "start-stage"}) + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1) + + def test_running_unhealthy_container_uses_fresh_occupancy_before_recovery(self): + self.create() + self.docker.containers[self.agent.container_name(INSTANCE, 1)]["State"]["Health"] = {"Status": "unhealthy"} + operation = self.operation("instance.restart") + self.agent.rcon = lambda *_: {"ok": True, "players": 2} + self.assertEqual(self.agent.execute(operation)["state"], "accepted") + self.agent.rcon = lambda *_: {"ok": True, "players": 0} + self.assertEqual(self.agent.execute(operation)["state"], "completed") + + def test_game_log_can_follow_internal_relative_plutainer_symlink(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + (app / "logs").mkdir() + (app / "runtime").mkdir() + (app / "runtime/game.log").write_text("first\nsecond\nthird\n") + (app / "logs/games_sp.log").symlink_to("../runtime/game.log") + result = self.agent.execute(self.operation("logs.read", {"path": "logs/games_sp.log", "lines": 2})) + self.assertEqual(result["state"], "completed") + self.assertEqual(json.loads(result["result"])["lines"], ["second", "third"]) + with self.assertRaises(AgentError): + module.managed_write(app, "logs/games_sp.log", b"must-not-write-through-link") + + def test_game_log_rejects_symlink_escaping_instance(self): + self.create() + app = self.agent.instance_dir(INSTANCE, 1) / "app" + (app / "logs").mkdir() + outside = Path(self.temp.name) / "host-secret" + outside.write_text("secret") + (app / "logs/games_sp.log").symlink_to(outside) + result = self.agent.execute(self.operation("logs.read", {"path": "logs/games_sp.log"})) + self.assertEqual(result["state"], "failed") + self.assertIn("outside", result["error"]) + + def test_console_log_falls_back_to_owned_docker_output_including_stderr(self): + self.create() + original = self.docker.call + def call(*args, **kwargs): + if args[:2] == ("container", "logs"): + self.assertTrue(kwargs["merge_stderr"]) + self.assertEqual(args[-1], self.agent.container_name(INSTANCE, 1)) + return "stdout-line\nstderr-line\n" + return original(*args, **kwargs) + self.docker.call = call + result = self.agent.execute(self.operation("logs.read", {"path": "logs/console.log"})) + self.assertEqual(result["state"], "completed") + self.assertEqual(json.loads(result["result"])["lines"], ["stdout-line", "stderr-line"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker-compose.yml b/docker-compose.yml index 13c33ff..125a55f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,35 +7,23 @@ services: restart: unless-stopped ports: - "3000:3000" - # Lets RCON_HOST=host.docker.internal reach game servers running on the - # same Komodo host. Docker Engine 20.10+ resolves this alias when the - # extra_hosts entry is present. - extra_hosts: - - "host.docker.internal:host-gateway" volumes: - # in prod (Komodo host), the game server tree is at /root/zombies - - /root/zombies:/root/zombies:rw - - /root/zombies/.electric-cherry:/data:rw + # Preserve the existing controller database by default. Set EC_DATA_DIR + # to a dedicated persistent directory for a new controller installation. + - ${EC_DATA_DIR:-/root/zombies/.electric-cherry}:/data:rw environment: - # SvelteKit's Node adapter caps request bodies at 512KB by default, - # which rejects real-world plugin .dll uploads with HTTP 413. Same - # trust boundary as the CSRF disable (see svelte.config.js) — this is - # LAN-only, so "Infinity" (no cap) is fine. NOTE: "0" does NOT mean - # unlimited in this adapter — it blocks anything > 0 bytes. SvelteKit's - # own error message calls this out: "To disable body size limits, - # specify Infinity rather than 0." If exposed to the public internet, - # replace with a concrete byte cap (e.g. "128M") instead. + ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000} + EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-} + EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-} + EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false} + # Large game files are streamed with manifest size and checksum limits. BODY_SIZE_LIMIT: "Infinity" - SERVERS_ROOT: /root/zombies/t5 - T6_SERVERS_ROOT: /root/zombies/t6 - REL_DEST: runtime/plutonium/storage/t5/raw/scripts/sp - BETA_ARCHIVE_DIR: /root/zombies/beta-archive DB_PATH: /data/cherry.db - KOMODO_ADDRESS: ${KOMODO_ADDRESS} - KOMODO_API_KEY: ${KOMODO_API_KEY} - KOMODO_API_SECRET: ${KOMODO_API_SECRET} - KOMODO_TARGET_SERVER: ${KOMODO_TARGET_SERVER} - PLUTAINER_T6_GAMEFILES_VOLUME: ${PLUTAINER_T6_GAMEFILES_VOLUME:-t6_game_files} - RCON_PASSWORD: ${RCON_PASSWORD} - RCON_HOST: ${RCON_HOST:-host.docker.internal} - RCON_TIMEOUT_MS: ${RCON_TIMEOUT_MS:-3000} + GAME_FILES_ROOT: /data/game-files + BETA_ARCHIVE_DIR: /data/beta-archive + # These are logical legacy paths; native instance files live on agents. + SERVERS_ROOT: /data/legacy/t5 + T6_SERVERS_ROOT: /data/legacy/t6 + PLUTAINER_IMAGE: ${PLUTAINER_IMAGE:-ghcr.io/ayymoss/plutainer:latest} + # Optional. Native instances otherwise receive individual random secrets. + RCON_PASSWORD: ${RCON_PASSWORD:-} diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000..b0c17ab --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,125 @@ +# Users and access + +Electric Cherry requires a user session for operator pages and management APIs. +HTTPS protects the connection; server-side role checks protect each operation. +There is no anonymous management mode or public registration. + +## First administrator and upgrades + +1. Generate a setup secret: `openssl rand -hex 32`. +2. Set `EC_SETUP_TOKEN` to that value in the controller environment (or Compose + `.env`), set `EC_PUBLIC_URL` to the external HTTPS URL, and start EC. For a + direct Node deployment, set `ORIGIN` to that same URL. +3. Visit EC. `/setup` asks for the setup token, a username, display name and + password. Only the holder of the token can create the first administrator. +4. Remove `EC_SETUP_TOKEN` after setup and restart/recreate the controller. + Setup remains closed once any user exists, even if the token is still set. +5. Use **Settings → Users** to create additional accounts. Share temporary + passwords privately; each user must change theirs on first login. + +Existing installations become protected on upgrade. Existing game servers and +agent enrollment credentials continue working independently of browser login. +Preserve the current database when upgrading: accounts and sessions are stored +there alongside the fleet metadata. Never publish a blank setup page with a +guessable token. An unset setup token holds setup closed. + +## Roles + +| Capability | Viewer | Operator | Administrator | +| --- | --- | --- | --- | +| Fleet, metrics, matches and operation summaries | Yes | Yes | Yes | +| Raw configs, file diffs, logs and telemetry diagnostics | No | Yes | Yes | +| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes | +| Scripts, plugins, configs, campaigns, betas and collections | Read summaries | Manage | Manage | +| Host setup, enrollment, credentials, runtime preparation and game-file import/cache | Read status | Read status | Manage | +| Server keys and global settings/maintenance policies | No | No | Manage | +| Create users, change roles, disable accounts and reset passwords | No | No | Yes | +| Own password and sign out | Yes | Yes | Yes | + +Operators can launch using server keys already saved by an administrator. They +cannot save keys indirectly through a launch request. An operator's content +permissions include executable scripts and plugins; assign that role only to +people trusted to operate the game fleet. + +Permissions are checked on the server for API calls, normal pages and Svelte +page-data requests. Hiding UI controls is a convenience, not the access boundary. +The controller prevents disabling or demoting the last enabled administrator. +Accounts are disabled rather than deleted so history retains their identity. + +## Passwords, sessions and recovery + +Passwords are 15–128 characters and stored using salted scrypt hashes. Temporary +passwords created or reset by administrators must be changed before any fleet +access. Password changes require the current password. + +Sessions use random tokens in HTTP-only cookies, with only token hashes stored in +SQLite. HTTPS uses a Secure `__Host-` cookie. Sessions expire after 12 hours, or +30 minutes without authenticated requests; background dashboard polling counts +as activity. Login replaces the existing browser session. Logout revokes it. +Changing a password, changing a role, disabling an account, or revoking that +user's sessions invalidates their existing sessions. Live event streams check +revocation every 15 seconds and then close. + +Login, initial setup and password verification are rate-limited with persistent +15-minute windows. The database stores audit entries for login results, account +changes and successful management requests without recording passwords, tokens +or request bodies. Administrators can review recent entries in Users. + +An administrator can reset another user's password from Users. If the only +administrator forgets their password, an operator with local controller/container +access can run the recovery command in an interactive terminal: + +```sh +docker compose exec electric-cherry node scripts/reset-admin.mjs admin +# Direct Node deployment, using the correct database: +DB_PATH=./data/cherry.db node scripts/reset-admin.mjs admin +``` + +Replace `admin` with the existing administrator's username. The command prompts +without echoing the password, resets that enabled administrator only, revokes +their sessions and records the recovery. It does not expose a remote recovery +endpoint or accept passwords as command arguments. + +## Agent and telemetry connections + +Host agents retain enrollment/host-token authentication under `/api/agent/`. +Installer and release files remain public so a new machine can enroll. A browser +login never substitutes for a host credential, and host tokens cannot manage EC. + +Game reporters must send a separate secret of at least 32 characters configured +as `EC_TELEMETRY_TOKEN`: + +```http +POST /api/telemetry +Authorization: Bearer +Content-Type: application/json +``` + +`X-EC-Telemetry-Token` is also supported. Cookies, query parameters and JSON fields +cannot supply it. A missing server token disables ingestion; a missing or wrong +request token is rejected before parsing or recording the payload. Existing JSON +event fields are unchanged. Configure your uploaded reporter/plugin, or a trusted +relay if that reporter cannot set headers, before upgrading. Use HTTPS for the +reporter-facing endpoint. Rotating this shared token requires updating reporters. + +## Proxy and local development + +Keep Node's internal port behind your TLS proxy and configure `ORIGIN` correctly. +Management writes require an exact matching `Origin` header, including login, +JSON requests and multipart uploads. Do not configure a proxy to replace arbitrary +incoming origins with your trusted origin. Machine endpoints use their own token +checks and do not require a browser Origin. + +Use the adapter's real client-address configuration only with a trusted proxy +that strips spoofed forwarding headers; login throttling otherwise uses the direct +peer address. Forwarded host or user headers do not authenticate users. + +Local HTTP browser development requires `EC_AUTH_ALLOW_HTTP=true` and a loopback +hostname (`localhost`, `127.0.0.1` or `::1`). This uses a separate development +cookie. It never enables HTTP login for a LAN/public hostname. Agent HTTP +development remains separately controlled by `EC_AGENT_ALLOW_HTTP` and the agent's +loopback-only option. + +The implementation follows OWASP's guidance for [authorization on every request](https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html#validate-the-permissions-on-every-request), +[password hashing](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#scrypt), +and [session management](https://cheatsheetseries.owasp.org/cheatsheets/Session_Management_Cheat_Sheet.html). diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md index 9db8107..5d3cb8d 100644 --- a/docs/beta-test-plans.md +++ b/docs/beta-test-plans.md @@ -6,7 +6,18 @@ beta on that server. ## Game reporter protocol -Send JSON to `POST /api/telemetry` with `Content-Type: application/json`. +Set `EC_TELEMETRY_TOKEN` on the Electric Cherry controller to a dedicated random +secret of at least 32 characters (for example, generate 64 hexadecimal characters +with `openssl rand -hex 32`). Configure the same secret in each game reporter or +its trusted local relay. Use a production HTTPS URL or a reverse proxy that +terminates TLS; browser login cookies do not authorize game telemetry. + +Send JSON to `POST /api/telemetry` with `Content-Type: application/json` and either +`Authorization: Bearer ` or +`X-EC-Telemetry-Token: `. If both headers are present, they must +contain the same secret. Keep the secret out of URLs, payloads and browser code, +and use a different secret from host-agent credentials or account passwords. +The JSON event format is unchanged. There are four events. Send them whether or not a beta is active. | Event | Fields | @@ -74,8 +85,10 @@ Send the final round and each player's final stats when the game ends. Success returns HTTP 200 with `{ "ok": true, "telemetry": { ... } }`. The returned telemetry contains the event and an Electric Cherry `receivedAt` -timestamp. Invalid JSON or fields return 400, an unknown/destroyed server 404, -and a non-JSON content type 415. +timestamp. A missing/invalid machine token returns 401; an unset or invalid +controller `EC_TELEMETRY_TOKEN` returns 503. Authentication happens before payload +validation. Authenticated invalid JSON or fields return 400, an unknown/destroyed +server 404, and a non-JSON content type 415. The script sends each event once, in gameplay order, directly to Electric Cherry. There is no discovery request, beta token, boot/session ID, sequence number, @@ -93,7 +106,12 @@ of beta activity. That page documents cumulative stat accounting, round credit, and the limits of identifying matches from this protocol. The game scripts are uploaded artifacts, not source maintained in this -repository. Their reporter should emit the four payloads above. +repository. Their reporter should emit the four payloads above and add the +authentication header. Existing anonymous reporters must be updated before +telemetry is accepted; if a reporter cannot set HTTP headers, configure a trusted +host-local relay to add the header. This repository does not bundle that relay or +assume a particular GSC HTTP implementation. Rotating the shared secret requires +updating the controller environment and all reporters/relays together. ### Telemetry diagnostics @@ -102,7 +120,9 @@ Open a server's **Diagnostics → Telemetry API requests** to inspect incoming name. Each entry includes the Electric Cherry receipt time, HTTP result, error reason, event, map, round, player count, content type, and submitted JSON. Rejected reports are included, even when the content type is wrong but the body is readable -JSON. Processing failures return 500 and record the internal error when diagnostic +JSON after authentication. Unauthorized requests are rejected before their bodies +are read and are not retained in per-server diagnostics. Authentication headers +and their secrets are never stored. Processing failures return 500 and record the internal error when diagnostic storage is available; failed database transactions do not update the scoreboard. The panel refreshes every five seconds while visible and has a manual refresh @@ -118,6 +138,8 @@ newest first, with caching disabled. Check the reporter's destination and server field. Calls that never reach Electric Cherry, malformed JSON, and bodies without a usable server name cannot appear in this server's log. Earlier requests are not backfilled. + Missing/invalid telemetry credentials also produce no retained request; inspect + the reporter's HTTP result for 401 or an unconfigured-controller 503. - **400 / 415:** inspect the error and payload. One invalid player's stats or a duplicate GUID rejects the entire report. These requests do not update the scoreboard. diff --git a/docs/native-hosts.md b/docs/native-hosts.md new file mode 100644 index 0000000..6ba7d41 --- /dev/null +++ b/docs/native-hosts.md @@ -0,0 +1,174 @@ +# Native host management + +Electric Cherry controls Docker/Plutainer through an outbound host agent. A native +instance never uses Komodo or a filesystem mounted into the controller. The +controller owns desired state, campaigns, collections, safety queues and history; +the agent owns local execution and its durable operation journal. + +## Install and launch + +1. Run the controller with persistent `/data` storage. Set `EC_PUBLIC_URL` to its + HTTPS origin and proxy TLS to port 3000. Set a random `EC_SETUP_TOKEN` and create + the first administrator at `/setup`. Management pages/APIs require user login + and roles; agent endpoints authenticate their own host credentials. See + [Users and access](authentication.md) for setup and password recovery. +2. Open **Hosts → Add Host**, choose a name, local game-port range and bind address, + and generate an enrollment. Run the displayed command on Debian 12/13 amd64. +3. The installer validates the platform, installs Python 3 and an absent Docker + installation, verifies the downloaded agent SHA-256 over HTTPS, enrolls it and + enables `electric-cherry-agent.service`. It needs no Git, Node.js or local build. +4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, import your + existing `t5_game_files` or `t6_game_files` Docker volume on an enrolled source + host. Name the library and version and record its DLC metadata. +5. Cache the published version on each destination host. Wait for verification. + Hosts report connection and readiness separately; deployment also checks the + particular library, image, capacity and ports you selected. +6. Launch a server or collection. Select the host, immutable game-file version(s) + and runtime image. EC selects lobby IDs and reserves UDP/TCP ports atomically. + Saved Plutonium server keys remain associated with their exact server names. + +A fresh destination host downloads centrally stored game files without manual +copying or per-container setup. EC does not supply the game content itself. + +The default host root is `/var/lib/electric-cherry`; installer `--data-dir` changes +it. Data is separated into `libraries///content`, +`instances///app`, and `agent/operations.sqlite3`. Base libraries +are mounted read-only and instance deletion removes only that instance's files. +The controller stores large files under `GAME_FILES_ROOT` (default beside the +SQLite database), with manifests/checksums and operation metadata in SQLite. + +The agent uses Plutainer's documented [game-file and app mounts](https://github.com/Ayymoss/Plutainer/blob/main/docs/volumes-and-configs.md). +Its log reader supports relative log links that resolve inside the instance app +directory, while file writes reject symlinks and paths outside that directory. + +## Identity, transport and credentials + +Enrollment tokens are random, hashed at rest, single-use and expire after 15 +minutes. Consumption and credential issuance are transactional. Enrollment only +works at the enrollment endpoint. Each host receives a distinct bearer credential +and the controller only stores its hash. Host and enrollment bearer credentials +are rejected outside `/api/agent/`. + +The agent uses HTTPS, operating-system certificate validation, outbound heartbeats +and polling; Docker has no exposed remote API. Agent endpoints reject HTTP unless +`EC_AGENT_ALLOW_HTTP=true` is explicitly enabled for development. The agent's +insecure development option is limited to loopback. Public release assets contain +no credentials. The release checksum is authenticated by the controller's HTTPS +origin; there is no separate offline release-signing infrastructure. + +Browser sessions do not authorize agents. Similarly, host credentials never grant +access to operator pages. Game telemetry uses a separate `EC_TELEMETRY_TOKEN`, +configured in each reporter's authorization header; see the +[telemetry contract](beta-test-plans.md#game-reporter-protocol). + +Hosts offers credential rotation and revocation. Rotation persists the new secret +on the agent before switching controller credentials, so a lost response can be +retried using the same proposed secret. Revocation affects one host only and does +not stop its running game containers. Re-enrollment requires a fresh token and an +explicit installer `--reenroll` when local credentials already exist. + +API outline: + +- `POST /api/agent/enroll`: consume enrollment token and return host credentials. +- `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches. +- `GET /api/agent/operations`: deliver/reconcile durable operation IDs. +- `POST /api/agent/operations/:id`: authenticated acceptance, progress or result. +- `/api/agent/imports/...` and `/api/agent/libraries/...`: host-scoped uploads and + authenticated game-file downloads. + +The operation envelope is independent of polling and can be carried over a +persistent transport later. No unrestricted shell operation is available. + +## Safety and durable operations + +Each remote request has an immutable ID and payload, plus an instance UUID and +generation where applicable. Reusing a key with a different request is rejected. +The controller records remote IDs before dispatch. The agent persists accepted, +running, completed, failed and unknown states in SQLite. Lost acknowledgments +return the recorded result; restarting the controller never creates a new +mutation simply because the previous reply was lost. + +The agent reconciles ambiguous results using owned Docker labels, generations and +persisted execution checkpoints when those observations prove the outcome. +Otherwise the result remains **Unknown** and holds dependent work. Running alone +is insufficient evidence that a particular restart happened. EC neither assumes +an unreachable server stopped nor launches a duplicate replacement elsewhere. + +Campaign bytes are staged and verified while players remain connected. Only +Provisioning can dispatch activation. It checks source eligibility and shared +capacity, and the agent repeats a fresh local RCON occupancy check immediately +before activation/restart, stop, removal or rebuild. Busy or unverifiable lobbies +wait indefinitely, including multi-hour Zombies sessions. Before images returned +by the agent populate existing campaign/beta rollback history. A beta cannot be +closed while its remote activation receipt is still unresolved. + +Rebuild uses a new generation, fresh instance storage and the latest promoted +configuration/scripts/plugins. It retains the selected game-file version and +runtime image. Old generation commands cannot target the replacement. Start, +restart and create complete only after post-operation health evidence; creation +and rebuild also verify the selected map and running Plutonium revision. The +base-game library and runtime revision are independent records. + +**Drain** blocks new deployments and queues safe retirement of existing lobbies. +Active beta ownership and other operations continue to gate retirement. Players +may still join until an eligible empty moment; draining does not kick them. +**Disable** holds agent dispatch. Neither mode, controller downtime nor agent +communication loss stops unrelated running containers. + +## Libraries and ports + +Imports scan the source volume and generate per-file SHA-256 manifests. Uploads +stream to temporary controller storage, are bounded by their manifest sizes and +verified before atomic publication. Published versions cannot be updated; +changing files requires a new version. Hosts download with HTTP Range resumption, +report progress, verify all checksums and atomically promote the complete cache. +Corrupt or incomplete content never becomes Ready. Cache inventory is reconciled +on heartbeats, including vanished caches. Older immutable versions remain usable +when their exact manifest is verified. + +Reservations include host ID, local bind address, protocol and port. Wildcard +binds conflict with concrete addresses on the same host. UDP and TCP reservations +for one launch commit together; the same number can be used independently on +another host. Optional public/NAT address and port are recorded separately and do +not change local Docker bindings. EC does not configure external NAT/firewalls. + +## Development and validation + +```sh +npm run agent:package +npm run check +npm test +npm run test:agent +npm run build +``` + +The build packages `agent/ec_agent.py` and `agent/install.sh` into the static +release, generates its checksum manifest and serves those same assets in the +Node-adapter image. The Python agent uses only its standard library. + +Automated tests cover enrollment/authentication, rotation/revocation, host +readiness, scoped ports, immutable library publication, streaming/ranges, +checksums, interrupted downloads, instance generation ownership, local occupancy +holds, controller restart/reconnect, ambiguous results, occupied campaign staging, +activation before images, collections and draining. Docker/UDP and host +observations are simulated in these tests. + +A real Debian/Docker/Plutonium acceptance run is still required before production +rollout. This workspace has no Docker engine or Debian game host, so automated +coverage is not evidence that a real T5/T6 server has been deployed here. Run: + +- Installer on Debian 12 and 13 amd64, then rerun without changing identity/data. +- T5 and T6 deployments and a mixed collection from a fresh destination host. +- Start, stop, restart, rebuild, destroy; verify RCON, map and runtime revision. +- Read console and game logs, including Plutainer's relative game-log links. +- File campaign and rollback, with a player joining between controller check and + agent activation, and during file activation before restart; verify that the + agent holds disruption. +- Drain occupied lobbies and verify shared libraries survive their retirement. +- Corrupt/interrupted library downloads, resumed transfer and verified promotion. +- Controller restart, agent restart and network loss before execution, during + execution and after execution but before acknowledgement. Verify the same + operation IDs and absence of duplicate containers/restarts. + +Legacy backend code remains for existing database registrations. New native +launches do not require its configuration, API, network or mounted directories. diff --git a/package.json b/package.json index 83199aa..48a8e97 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "electric-cherry", "version": "0.1.0", - "description": "T5/T6 Plutonium Zombies fleet manager — replaces push_scripts.sh with a live web UI", + "description": "T5/T6 Plutonium Zombies fleet manager \u2014 replaces push_scripts.sh with a live web UI", "private": true, "type": "module", "scripts": { @@ -12,7 +12,10 @@ "test": "vitest run", "test:watch": "vitest", "benchmark:leaderboard": "node scripts/benchmark-leaderboard.mjs", - "seed:fixture": "tsx scripts/seed-fixture.ts" + "seed:fixture": "tsx scripts/seed-fixture.ts", + "agent:package": "python3 agent/package_release.py", + "prebuild": "npm run agent:package", + "test:agent": "python3 -m unittest discover -s agent/tests -v" }, "dependencies": { "@fontsource/alfa-slab-one": "^5.3.0", diff --git a/scripts/reset-admin.mjs b/scripts/reset-admin.mjs new file mode 100644 index 0000000..db0340c --- /dev/null +++ b/scripts/reset-admin.mjs @@ -0,0 +1,55 @@ +#!/usr/bin/env node +// Local recovery only: requires filesystem/container access to the controller. +import Database from 'better-sqlite3'; +import { emitKeypressEvents } from 'node:readline'; +import { hashPassword } from '../src/lib/server/auth/passwords.js'; + +function secret(prompt) { + return new Promise((resolve, reject) => { + let value = ''; + process.stdout.write(prompt); + emitKeypressEvents(process.stdin); + process.stdin.setRawMode(true); + process.stdin.resume(); + const cleanup = () => { + process.stdin.off('keypress', listener); + process.stdin.setRawMode(false); + process.stdin.pause(); + process.stdout.write('\n'); + }; + const listener = (text, key) => { + if (key?.ctrl && key.name === 'c') { cleanup(); reject(new Error('Cancelled.')); } + else if (key?.name === 'return' || key?.name === 'enter') { cleanup(); resolve(value); } + else if (key?.name === 'backspace') value = [...value].slice(0,-1).join(''); + else if (text && !key?.ctrl && !key?.meta && !/[\x00-\x1f\x7f]/.test(text) && [...value].length < 128) value += text; + }; + process.stdin.on('keypress',listener); + }); +} + +try { + const username = process.argv[2]?.toLowerCase(); + if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username)) throw new Error('Usage: node scripts/reset-admin.mjs '); + if (!process.stdin.isTTY) throw new Error('Run this command in an interactive terminal. Passwords are never accepted as command arguments.'); + const database = new Database(process.env.DB_PATH || './data/cherry.db',{fileMustExist:true}); + database.pragma('busy_timeout=5000'); + const user = database.prepare("SELECT id FROM auth_users WHERE username=? AND role='admin' AND disabled=0").get(username); + if (!user) throw new Error('An enabled administrator with that username was not found.'); + const password = await secret('New administrator password (15–128 characters): '); + const confirmation = await secret('Repeat password: '); + if (password !== confirmation) throw new Error('Passwords did not match.'); + const passwordHash = await hashPassword(password); + database.transaction(() => { + const changed = database.prepare("UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0") + .run(passwordHash,Date.now(),user.id); + if (changed.changes !== 1) throw new Error('Administrator changed during recovery. Try again.'); + database.prepare('DELETE FROM auth_sessions WHERE user_id=?').run(user.id); + database.prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)') + .run('password.recovered',user.id,'{"source":"local-cli"}',Date.now()); + }).immediate(); + database.close(); + console.log('Administrator password reset. Existing sessions have been revoked.'); +} catch (cause) { + console.error(cause instanceof Error ? cause.message : 'Administrator recovery failed.'); + process.exitCode=1; +} diff --git a/src/app.d.ts b/src/app.d.ts index 9213ba0..c8e14e2 100644 --- a/src/app.d.ts +++ b/src/app.d.ts @@ -1,5 +1,10 @@ declare global { - namespace App {} + namespace App { + interface Locals { + user: import('./lib/auth.js').AuthUser | null; + sessionHash: string | null; + } + } } export {}; diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 1181933..25b4c22 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -2,6 +2,7 @@ import type { Handle } from '@sveltejs/kit'; import { migrate } from '$lib/server/db/migrate.js'; import { startWorker } from '$lib/server/pipeline/worker.js'; import { interruptDeployments } from '$lib/server/provisioning/history.js'; +import { authenticationHandle } from '$lib/server/auth/guard.js'; let booted = false; function boot(): void { @@ -15,7 +16,7 @@ function boot(): void { // SvelteKit calls handle() for every request; using the first request as the // boot trigger keeps the whole app single-process without needing an // out-of-band init hook. -export const handle: Handle = async ({ event, resolve }) => { +export const handle: Handle = async (input) => { boot(); - return resolve(event); + return authenticationHandle(input); }; diff --git a/src/lib/auth.ts b/src/lib/auth.ts new file mode 100644 index 0000000..ffca13d --- /dev/null +++ b/src/lib/auth.ts @@ -0,0 +1,17 @@ +export type UserRole = 'admin' | 'operator' | 'viewer'; + +export interface AuthUser { + id: string; + username: string; + displayName: string; + role: UserRole; + disabled: boolean; + mustChangePassword: boolean; + createdAt: string; + updatedAt: string; + lastLoginAt: string | null; +} + +export const roleLabels: Record = { + admin: 'Administrator', operator: 'Operator', viewer: 'Viewer' +}; diff --git a/src/lib/deployment-plan.ts b/src/lib/deployment-plan.ts index 86fb651..d853146 100644 --- a/src/lib/deployment-plan.ts +++ b/src/lib/deployment-plan.ts @@ -11,6 +11,7 @@ export interface LaunchFileSummary { } export interface LaunchRecipeSummary { + native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }; game: ZombieGame; mapSlug: string; mapName: string; diff --git a/src/lib/game-files.ts b/src/lib/game-files.ts new file mode 100644 index 0000000..27782db --- /dev/null +++ b/src/lib/game-files.ts @@ -0,0 +1,28 @@ +export type GameLibraryGame = 't5' | 't6'; +export type GameLibraryState = 'importing' | 'ready' | 'failed'; +export type LibraryCacheState = 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; + +export interface GameFileEntry { path: string; size: number; sha256: string } +export interface HostLibraryCache { + hostId: string; libraryVersionId: string; status: LibraryCacheState; + bytesDownloaded: number; bytesTotal: number; manifestSha256: string | null; + error: string | null; updatedAt: string; operationId: string | null; +} +export interface GameLibraryVersion { + id: string; libraryId: string; game: GameLibraryGame; version: string; + description: string; dlc: string[]; state: GameLibraryState; sizeBytes: number; + fileCount: number; manifestSha256: string | null; files: GameFileEntry[]; + createdAt: string; publishedAt: string | null; storageLocation: string; + sourceHostId: string; sourceVolume: string; error: string | null; operationId: string | null; + caches: HostLibraryCache[]; +} +export interface GameLibrary { + id: string; name: string; game: GameLibraryGame; description: string; + createdAt: string; versions: GameLibraryVersion[]; +} + +export function formatGameBytes(bytes: number): string { + if (!bytes) return '0 B'; + const unit = Math.min(4, Math.floor(Math.log(bytes) / Math.log(1024))); + return `${(bytes / 1024 ** unit).toFixed(unit ? 1 : 0)} ${['B', 'KiB', 'MiB', 'GiB', 'TiB'][unit]}`; +} diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts new file mode 100644 index 0000000..6ead592 --- /dev/null +++ b/src/lib/hosts.ts @@ -0,0 +1,29 @@ +export type HostMode = 'active' | 'draining' | 'disabled'; +export type HostReadiness = 'preparing' | 'ready' | 'degraded' | 'draining' | 'disabled'; +export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown'; + +export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string } +export interface HostInventory { + hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string; + cpu?: string; cpuCount?: number; memoryTotal?: number; memoryAvailable?: number; + storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean; + directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[]; + currentOperation?: string | null; [key: string]: unknown; +} +export interface Host { + id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; + bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string; + lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline'; + readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory; + enrolled: boolean; credentialActive: boolean; +} +export interface HostOperation { + id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null; + payload: Record; state: HostOperationState; result: unknown; error: string | null; + createdAt: string; updatedAt: string; +} + +export interface NativeDeploymentSelection { + hostId: string; libraryIds: Partial>; image: string; + bindAddress?: string; publicAddress?: string; publicPort?: number; +} diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index 44b79a5..1897bdb 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -6,7 +6,7 @@ export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'verify-map', 'verify-plutonium', 'verify-stopped'] as const; export type ProvisioningPhase = typeof provisioningPhases[number]; export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown'; -export type EvidenceSource = 'komodo' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null; +export type EvidenceSource = 'komodo' | 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null; export interface ProvisioningCheck { key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped'; diff --git a/src/lib/server/auth/access.ts b/src/lib/server/auth/access.ts new file mode 100644 index 0000000..dae65b9 --- /dev/null +++ b/src/lib/server/auth/access.ts @@ -0,0 +1,51 @@ +import type { UserRole } from '../../auth.js'; + +export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin'; + +const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/'); +const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD'; + +/** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname. + * Route IDs preserve the same policy for page-data requests and encoded URLs. + * Machine endpoints enforce their own host/enrollment/telemetry credentials. + */ +export function routePermission(routeId: string | null, method: string): RoutePermission { + method = method.toUpperCase(); + const reading = readMethod(method); + if (routeId === null) return reading ? 'viewer' : 'operator'; + // These cannot occur in this application's resolved route IDs. Fail closed + // if a future caller accidentally supplies a raw or noncanonical pathname. + if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId) + || routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin'; + + if (within(routeId, '/api/agent')) return 'machine'; + if (routeId === '/api/telemetry' && method === 'POST') return 'machine'; + if ((routeId === '/login' || routeId === '/setup') && reading) return 'public'; + if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public'; + if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated'; + + if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/auth') + || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin'; + + if (!reading && ['/api/hosts', '/api/settings', '/api/game-files'].some(base => within(routeId, base))) return 'admin'; + // These endpoints mix operational actions with global policy and schedule + // changes. Per-incident/job actions remain available to operators. + if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin'; + + // Summaries remain visible to viewers; raw configuration, historical file + // bytes, and unfiltered diagnostics may contain runtime credentials. + if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect' + || routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]' + || routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests') return 'operator'; + + return reading ? 'viewer' : 'operator'; +} + +/** A browser role never substitutes for a machine credential. */ +export function roleAllows(role: UserRole, permission: RoutePermission): boolean { + if (!['viewer', 'operator', 'admin'].includes(role)) return false; + if (permission === 'machine') return false; + if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true; + if (permission === 'operator') return role === 'operator' || role === 'admin'; + return permission === 'admin' && role === 'admin'; +} diff --git a/src/lib/server/auth/guard.ts b/src/lib/server/auth/guard.ts new file mode 100644 index 0000000..751e4c9 --- /dev/null +++ b/src/lib/server/auth/guard.ts @@ -0,0 +1,111 @@ +import { error, json, redirect, type Handle } from '@sveltejs/kit'; +import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js'; +import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js'; +import { roleAllows, routePermission, type RoutePermission } from './access.js'; + +function publicAsset(routeId: string | null, path: string, method: string) { + return routeId === null && ['GET','HEAD'].includes(method) && + (path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/install.sh','/agent/manifest.json'].includes(path)); +} + +/** Streaming responses must lose access when the underlying session does. */ +export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response { + if (!response.body) return response; + const reader = response.body.getReader(); + let timer: ReturnType | undefined; + let closed = false; + let closeStream: () => void = () => {}; + const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); }; + const stream = new ReadableStream({ + start(controller) { + closeStream = () => { + if (closed) return; closed = true; cleanup(); + void reader.cancel().catch(() => {}); controller.close(); + }; + timer = setInterval(() => { + try { + const current = readSessionHash(tokenHash); + if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream(); + } catch { closeStream(); } + },15_000); + signal.addEventListener('abort',closeStream,{once:true}); + if (signal.aborted) { closeStream(); return; } + }, + async pull(controller) { + try { + const next = await reader.read(); + if (closed) return; + if (next.done) { closed=true; cleanup(); controller.close(); } + else controller.enqueue(next.value); + } catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } } + }, + cancel() { closed=true; cleanup(); return reader.cancel(); } + }); + return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers}); +} + +export const authenticationHandle: Handle = async ({ event, resolve }) => { + event.locals.user = null; event.locals.sessionHash = null; + const routeId = event.route.id; + const method = event.request.method.toUpperCase(); + if (publicAsset(routeId,event.url.pathname,method)) return resolve(event); + const permission = routePermission(routeId,method); + if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) { + return json({error:'Host credentials only authorize the host agent API.'},{status:403}); + } + if (permission === 'machine') return resolve(event); + const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/'); + try { + requireUserTransport(event.url); + if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url); + } catch (cause) { if (api) return authError(cause); throw cause; } + + const session = readSession(event.cookies.get(sessionCookie(event.url))); + event.locals.user = session?.user??null; + event.locals.sessionHash = session?.tokenHash??null; + const setup = setupRequired(); + if (permission !== 'public') { + if (!session) { + if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}}); + redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`); + } + if (session.user.mustChangePassword && permission !== 'authenticated') { + if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}}); + redirect(303,'/account'); + } + if (!roleAllows(session.user.role,permission)) { + const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.'; + if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}}); + error(403,message); + } + // Launch APIs can also save credentials. Operators may launch with saved + // keys, but only administrators may introduce or replace server keys. + if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) { + try { + const body = await authBody(event.request.clone(),1024*1024); + if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) { + return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403}); + } + } catch (cause) { return authError(cause); } + } + } else if (['/login','/setup'].includes(routeId??'')) { + if (session) redirect(303,session.user.mustChangePassword?'/account':'/'); + if (routeId==='/setup'&&!setup) redirect(303,'/login'); + if (routeId==='/login'&&setup) redirect(303,'/setup'); + } + + let response = await resolve(event); + if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) { + recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname}); + } + if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) { + response = protectEventStream(response,session.tokenHash,permission,event.request.signal); + } + const headers = new Headers(response.headers); + headers.set('Cache-Control','no-store'); + headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', ')); + headers.set('Referrer-Policy','same-origin'); + headers.set('X-Content-Type-Options','nosniff'); + headers.set('X-Frame-Options','DENY'); + return new Response(response.body,{status:response.status,statusText:response.statusText,headers}); +}; diff --git a/src/lib/server/auth/http.ts b/src/lib/server/auth/http.ts new file mode 100644 index 0000000..ed36c3b --- /dev/null +++ b/src/lib/server/auth/http.ts @@ -0,0 +1,49 @@ +import { error, isHttpError, json, type Cookies } from '@sveltejs/kit'; +import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js'; +import type { AuthUser } from '../../auth.js'; + +export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; } +export function requireUserTransport(url: URL): void { + if (url.protocol === 'https:') return; + if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return; + error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.'); +} +export function requireSameOrigin(request: Request, url: URL): void { + if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') { + error(403, 'This action requires a same-origin request.'); + } +} +export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) { + cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:', + sameSite: 'lax', expires: new Date(result.session.expiresAt) }); + return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } }); +} +export function requireUser(user: AuthUser | null): AuthUser { + if (!user) throw new AuthError('Sign in to continue.',401); + return user; +} +export async function authBody(request: Request, maxBytes = 8192): Promise> { + if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415); + if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413); + const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.'); + const chunks: Uint8Array[] = []; let length = 0; + try { + for (;;) { + const { done, value } = await reader.read(); if (done) break; + length += value.byteLength; + if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); } + chunks.push(value); + } + } finally { reader.releaseLock(); } + let body: unknown; + try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); } + if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.'); + return body as Record; +} +export function authError(cause: unknown): Response { + if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} }); + const message = cause instanceof Error ? cause.message : 'Authentication request failed.'; + const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500; + return json({ error: status === 500 ? 'The authentication request could not be completed.' : message }, + { status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} }); +} diff --git a/src/lib/server/auth/index.ts b/src/lib/server/auth/index.ts new file mode 100644 index 0000000..3a06e28 --- /dev/null +++ b/src/lib/server/auth/index.ts @@ -0,0 +1,184 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; +import { db } from '../db/index.js'; +import { ensureAuthSchema } from './schema.js'; +import { hashPassword, verifyPassword } from './passwords.js'; +import type { AuthUser, UserRole } from '../../auth.js'; + +export const SESSION_COOKIE = '__Host-ec_session'; +export const SESSION_LIFETIME_MS = 12 * 60 * 60_000; +export const SESSION_IDLE_MS = 30 * 60_000; +const digest = (value: string) => createHash('sha256').update(value).digest('hex'); +function database() { const handle = db(); ensureAuthSchema(handle); return handle; } +type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number; + must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null }; +export class AuthError extends Error { + constructor(message: string, public status = 400) { super(message); } +} +function present(row: UserRow): AuthUser { + return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled, + mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() }; +} +function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; } +export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; } +export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); } +export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); } +function username(value: unknown): string { + if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.'); + return value.toLowerCase(); +} +function displayName(value: unknown, fallback: string): string { + const name = value === undefined ? fallback : value; + if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.'); + return name.trim(); +} +function role(value: unknown): UserRole { + if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.'); + return value as UserRole; +} +export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record = {}): void { + database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)') + .run(actorId, action, targetId, JSON.stringify(details), Date.now()); +} +export function listAuthAudit() { + return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt + FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all(); +} + +/** Persistent fixed windows are consumed before hashing, including failed setup. */ +export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void { + const now = Date.now(); const windowMs = 15 * 60_000; + const ipKey = digest(`${kind}:ip:${clientAddress}`); + const accountKey = digest(`${kind}:account:${account.toLowerCase()}`); + database().transaction(() => { + database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000); + for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) { + const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined; + if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429); + } + for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1) + ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END, + window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs); + }).immediate(); +} + +function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser { + if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409); + const id = randomUUID(); const now = Date.now(); + database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`) + .run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now); + return getUser(id)!; +} +export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } } +export async function bootstrapAdmin(input: Record, clientAddress: string): Promise { + consumeAuthAttempt(clientAddress, 'bootstrap', 'setup'); + if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409); + const expected = process.env.EC_SETUP_TOKEN; + if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503); + if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401); + const name = username(input.username); const label = displayName(input.displayName, name); + const passwordHash = await hashPassword(input.password); + return database().transaction(() => { + if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409); + const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false }); + recordAudit(user.id, 'setup', user.id); + return { user, session: createSession(user.id) }; + }).immediate(); +} +export async function authenticatePassword(input: Record, clientAddress: string): Promise { + const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : ''; + consumeAuthAttempt(clientAddress, name); + const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined; + const valid = await verifyPassword(input.password, row?.password_hash ?? null); + const result = database().transaction(() => { + const current = row ? userRow(row.id) : undefined; + if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) { + recordAudit(null, 'login.failed'); return null; + } + database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id); + recordAudit(current.id, 'login', current.id); + return { user: getUser(current.id)!, session: createSession(current.id) }; + }).immediate(); + if (!result) throw new AuthError('Invalid username or password.', 401); + return result; +} + +export function createSession(userId: string): { token: string; expiresAt: number } { + const now = Date.now(); const user = userRow(userId); + if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401); + const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS; + database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS); + database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now); + return { token, expiresAt }; +} +export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null { + if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null; + return readSessionHash(digest(token), touch); +} +export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null { + const now = Date.now(); + const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id + WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined; + if (!row) return null; + if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash); + return { user: present(row), tokenHash, expiresAt: row.expires_at }; +} +export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); } +export function revokeUserSessions(actorId: string, userId: string): void { + database().transaction(() => { + requireAdministrator(actorId); + if (!userRow(userId)) throw new AuthError('User not found.',404); + database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId); + recordAudit(actorId,'sessions.revoked',userId); + })(); +} +function requireAdministrator(actorId: string) { + const actor = userRow(actorId); + if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403); +} +export async function createUser(actorId: string, input: Record): Promise { + requireAdministrator(actorId); + const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role); + const passwordHash = await hashPassword(input.password); + return database().transaction(() => { + requireAdministrator(actorId); + const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true }); + recordAudit(actorId,'user.created',user.id,{role:userRole}); return user; + }).immediate(); +} +export async function updateUser(actorId: string, id: string, input: Record): Promise { + requireAdministrator(actorId); + if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.'); + if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.'); + const passwordHash = input.password === undefined ? null : await hashPassword(input.password); + return database().transaction(() => { + requireAdministrator(actorId); + const current = userRow(id); if (!current) throw new AuthError('User not found.',404); + const nextRole = input.role === undefined ? current.role : role(input.role); + const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled); + const label = displayName(input.displayName,current.display_name); + if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) { + const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id); + if (!other) throw new AuthError('Keep at least one enabled administrator.',409); + } + database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`) + .run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id); + if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id); + recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash}); + return getUser(id)!; + }).immediate(); +} +export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise { + const original = userRow(userId); + if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400); + if (currentPassword === newPassword) throw new AuthError('Choose a different password.'); + const passwordHash = await hashPassword(newPassword); + return database().transaction(() => { + const current = userRow(userId); + if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409); + database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId); + database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId); + recordAudit(userId,'password.changed',userId); + return { user: getUser(userId)!, session: createSession(userId) }; + }).immediate(); +} diff --git a/src/lib/server/auth/passwords.js b/src/lib/server/auth/passwords.js new file mode 100644 index 0000000..16bffae --- /dev/null +++ b/src/lib/server/auth/passwords.js @@ -0,0 +1,45 @@ +import { randomBytes, scrypt, timingSafeEqual } from 'node:crypto'; + +// OWASP's scrypt baseline: 128 MiB, r=8, p=1. Bound concurrent work so +// unauthenticated password attempts cannot allocate unlimited memory. +const parameters = { N: 131072, r: 8, p: 1, maxmem: 256 * 1024 * 1024 }; +let active = 0; + +/** @param {unknown} password */ +export function validatePassword(password) { + if (typeof password !== 'string' || [...password].length < 15 || [...password].length > 128) { + throw new Error('Use a password or passphrase between 15 and 128 characters.'); + } + return password; +} + +/** @param {string} password @param {string} salt @returns {Promise} */ +function derive(password, salt) { + if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.')); + active++; + return new Promise((resolve, reject) => { + scrypt(password, salt, 64, parameters, (error, result) => { + active--; + if (error) reject(error); else resolve(result); + }); + }); +} + +/** @param {unknown} password */ +export async function hashPassword(password) { + const validated = validatePassword(password); + const salt = randomBytes(32).toString('hex'); + const key = await derive(validated, salt); + return `scrypt$131072$8$1$${salt}$${key.toString('hex')}`; +} + +/** @param {unknown} password @param {string | null} encoded */ +export async function verifyPassword(password, encoded) { + if (typeof password !== 'string' || [...password].length > 128) return false; + const parts = encoded?.split('$'); + const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1' + && /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]); + // Missing accounts perform the same expensive operation as real accounts. + const result = await derive(password, valid ? parts[4] : '0'.repeat(64)); + return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid; +} diff --git a/src/lib/server/auth/schema.ts b/src/lib/server/auth/schema.ts new file mode 100644 index 0000000..22ab691 --- /dev/null +++ b/src/lib/server/auth/schema.ts @@ -0,0 +1,27 @@ +import type Database from 'better-sqlite3'; +const initialized = new WeakSet(); +export function ensureAuthSchema(handle: Database.Database): void { + if (initialized.has(handle)) return; + handle.exec(` + CREATE TABLE IF NOT EXISTS auth_users ( + id TEXT PRIMARY KEY, username TEXT NOT NULL COLLATE NOCASE UNIQUE, + display_name TEXT NOT NULL, password_hash TEXT NOT NULL, + role TEXT NOT NULL CHECK(role IN ('admin','operator','viewer')), + disabled INTEGER NOT NULL DEFAULT 0, must_change_password INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, last_login_at INTEGER + ); + CREATE TABLE IF NOT EXISTS auth_sessions ( + token_hash TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES auth_users(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS auth_sessions_user ON auth_sessions(user_id); + CREATE TABLE IF NOT EXISTS auth_rate_limits ( + key TEXT PRIMARY KEY, window_start INTEGER NOT NULL, attempts INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS auth_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, actor_id TEXT, action TEXT NOT NULL, + target_id TEXT, details TEXT NOT NULL, created_at INTEGER NOT NULL + ); + `); + initialized.add(handle); +} diff --git a/src/lib/server/auth/telemetry.ts b/src/lib/server/auth/telemetry.ts new file mode 100644 index 0000000..5698564 --- /dev/null +++ b/src/lib/server/auth/telemetry.ts @@ -0,0 +1,34 @@ +import { createHash, timingSafeEqual } from 'node:crypto'; + +export type TelemetryAuthentication = + | { ok: true } + | { ok: false; status: 401 | 503; error: string }; + +const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/; +const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value); +const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest(); + +/** Machine telemetry has its own credential; browser sessions never authorize it. */ +export function authenticateTelemetry(request: Request): TelemetryAuthentication { + const expected = process.env.EC_TELEMETRY_TOKEN ?? ''; + if (!validToken(expected)) { + return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' }; + } + + const authorization = request.headers.get('authorization'); + const custom = request.headers.get('x-ec-telemetry-token'); + const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1]; + const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' }); + + // Reject malformed/ambiguous credentials rather than falling back to a + // second header. A shared secret must never come from a URL, cookie or body. + if (authorization !== null && !bearer) return denied(); + if (custom !== null && !validToken(custom)) return denied(); + if (bearer !== undefined && !validToken(bearer)) return denied(); + const supplied = bearer ?? custom; + if (!supplied) return denied(); + const suppliedHash = hash(supplied); + if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied(); + if (!timingSafeEqual(suppliedHash, hash(expected))) return denied(); + return { ok: true }; +} diff --git a/src/lib/server/backends/files.ts b/src/lib/server/backends/files.ts new file mode 100644 index 0000000..99006eb --- /dev/null +++ b/src/lib/server/backends/files.ts @@ -0,0 +1,50 @@ +import * as fs from 'node:fs/promises'; +import { relative, resolve, sep } from 'node:path'; +import { config } from '../config.js'; +import { nativeInstance, nativeRequest } from './native.js'; + +function remote(path: string) { + for (const root of [config.serversRoot, config.t6ServersRoot]) { + if (!root) continue; + const rel = relative(resolve(root), resolve(path)); + if (!rel || rel.startsWith('..') || rel.startsWith(sep)) continue; + const [name, ...parts] = rel.split(sep); + const instance = nativeInstance(name); + if (instance) return { instance, path: parts.join('/') || '.' }; + } + return null; +} +function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); } + +export async function readFile(path: string): Promise { + const target = remote(path); + if (!target) return fs.readFile(path); + const result = await nativeRequest<{ contentBase64: string | null }>(target.instance, 'files.read', { path: target.path }); + return result.contentBase64 === null ? missing(path) : Buffer.from(result.contentBase64, 'base64'); +} + +export async function exists(path: string): Promise { + try { await stat(path); return true; } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; } +} + +export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> { + const target = remote(path); + if (!target) return fs.stat(path); + const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path }); + if (result === null || result.exists === false) return missing(path); + return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile }; +} + +export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> { + const target = remote(path); + if (!target) return fs.readdir(path, options); + const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path }); + return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory })); +} + +export async function nativeTail(path: string, lines: number) { + const target = remote(path); + if (!target) return undefined; + return nativeRequest(target.instance, 'logs.read', { path: target.path, lines }); +} diff --git a/src/lib/server/backends/native.ts b/src/lib/server/backends/native.ts new file mode 100644 index 0000000..d455a6e --- /dev/null +++ b/src/lib/server/backends/native.ts @@ -0,0 +1,41 @@ +import { randomUUID } from 'node:crypto'; +import { db } from '../db/index.js'; +import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js'; + +export interface NativeInstance { + id: string; server: string; host_id: string; generation: number; library_id: string; + image: string; bind_address: string; port: number; public_address: string | null; + public_port: number | null; created_at: string; retired_at: string | null; +} + +export function nativeInstance(server: string): NativeInstance | null { + return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null; +} + +export function nativeDeployment(id: string): boolean { + return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id); +} + +/** Only the agent receives host paths, Docker access, or game credentials. */ +export async function nativeRequest>(instance: NativeInstance, kind: string, payload: Record = {}, key?: string): Promise { + const host = getHost(instance.host_id); + if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.'); + const operation = queueHostOperation(instance.host_id, kind, payload, { + instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID() + }); + const result = await waitHostOperation(operation.id, 25_000); + if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`); + // Do not apply replies from a retired generation to a replacement server. + const current = nativeInstance(instance.server); + if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.'); + return result.result as T; +} + +export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> { + const instance = nativeInstance(server); + if (!instance) return { ok: false, reason: 'Native instance is not registered' }; + try { + const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect'); + return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` }; + } catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; } +} diff --git a/src/lib/server/backends/schema.ts b/src/lib/server/backends/schema.ts new file mode 100644 index 0000000..5d35823 --- /dev/null +++ b/src/lib/server/backends/schema.ts @@ -0,0 +1,20 @@ +import type Database from 'better-sqlite3'; + +export function migrateNativeInstances(handle: Database.Database): void { + handle.exec(` + CREATE TABLE IF NOT EXISTS native_instances ( + id TEXT PRIMARY KEY, server TEXT NOT NULL, host_id TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 1, library_id TEXT NOT NULL, + image TEXT NOT NULL, bind_address TEXT NOT NULL DEFAULT '0.0.0.0', + port INTEGER NOT NULL, public_address TEXT, public_port INTEGER, + created_at TEXT NOT NULL, retired_at TEXT + ); + CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL; + CREATE TABLE IF NOT EXISTS native_provisioning ( + deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id), + instance_id TEXT NOT NULL REFERENCES native_instances(id), generation INTEGER NOT NULL, + stage_operation_id TEXT, operation_id TEXT, files_json TEXT, + applied_at TEXT, verified_at TEXT + ); + `); +} diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 09f5c21..1b3cf4a 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -8,6 +8,7 @@ import { emptyMetrics } from '../fleet/beta-evaluation.js'; import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js'; import { initializeFleetSettings } from './fleet-settings.js'; import { migrateServerHistory } from './server-history.js'; +import { migrateNativeInstances } from '../backends/schema.js'; let _db: Database.Database | null = null; @@ -366,6 +367,7 @@ export function db(): Database.Database { ]); backfillLeaderboard(handle); initializeFleetSettings(handle); + migrateNativeInstances(handle); } catch (err) { handle.close(); throw err; diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index f4e3cd8..f9c3f36 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -10,8 +10,10 @@ async function exclusive(id: number, action: () => Promise): Promise { try { return await action(); } finally { operations.delete(id); } } -import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { existsSync } from 'node:fs'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { readFile, exists } from '../backends/files.js'; +import { getHostOperation } from '../hosts/index.js'; + import { join, posix } from 'node:path'; import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; @@ -204,6 +206,14 @@ export class BetaError extends Error { } } +function assertNativeActivationSettled(betaId: number): void { + const pending = db().prepare(`SELECT n.operation_id FROM native_provisioning n + JOIN provisioning_sources s ON s.operation_id=n.deployment_id + WHERE s.kind='beta' AND s.source_id=? AND n.operation_id IS NOT NULL + AND n.files_json IS NOT NULL AND n.applied_at IS NULL`).all(String(betaId)) as { operation_id: string }[]; + if (pending.some(row => getHostOperation(row.operation_id)?.state !== 'failed')) throw new BetaError('The host is still reconciling file activation. Wait for its durable before images before promoting or rolling back.', 'conflict'); +} + function nowIso(): string { return new Date().toISOString(); } @@ -563,6 +573,7 @@ export async function rollbackBeta(betaId: number, options: { restart?: 'manual' return exclusive(betaId, () => rollbackBetaInternal(betaId, options)); } async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise { + assertNativeActivationSettled(betaId); const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'running' && beta.state !== 'ready-promote') { @@ -658,7 +669,7 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual // Wipe archive + close out beta. const archive = betaArchiveOf(betaId); - if (existsSync(archive)) await rm(archive, { recursive: true, force: true }); + if (await exists(archive)) await rm(archive, { recursive: true, force: true }); // Per-version test-failed transition. Skip a version if (a) another beta // still gates it at ready-promote/promoted, or (b) it's the currently- @@ -764,6 +775,7 @@ export async function promoteBeta(betaId: number, reviewedToken?: string): Promi return exclusive(betaId, () => promoteBetaInternal(betaId, reviewedToken)); } async function promoteBetaInternal(betaId: number, reviewedToken?: string): Promise { + assertNativeActivationSettled(betaId); const test = storedPlan(betaId); if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote'); const beta = getBeta(betaId); @@ -842,7 +854,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom // Non-beta matching-scope server: current on-disk file (if any) // is the "before" state for a promote-then-rollback round trip. const targetPath = join(dest, script.target_rel_path); - if (existsSync(targetPath)) { + if (await exists(targetPath)) { const priorContent = await readFile(targetPath); const inv = getServerScript(name, script.target_rel_path); plan = { @@ -906,13 +918,13 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom hadFile = true; } } - if (!betaServerSet.has(name) && existsSync(targetPath)) { + if (!betaServerSet.has(name) && await exists(targetPath)) { // Capture an initial preview; activation records the actual before image. priorContent = await readFile(targetPath); priorSha = sha256Buffer(priorContent); hadFile = true; } - const requiresActivation = !betaServerSet.has(name) || !existsSync(targetPath) + const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath) || sha256Buffer(await readFile(targetPath)) !== composedSha; cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation }); } @@ -1038,7 +1050,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); const archive = betaArchiveOf(betaId); - if (existsSync(archive)) await rm(archive, { recursive: true, force: true }); + if (await exists(archive)) await rm(archive, { recursive: true, force: true }); const landed = Array.from(new Set([ ...plans.map((p) => p.server), diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index dc0355a..8a2fb8a 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -11,6 +11,8 @@ import { getServerKey } from './server-keys.js'; import { serverObservation } from './status.js'; import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; +import { nativeSelection, nextNativePort } from '../provisioning/native.js'; +import { nativeInstance } from '../backends/native.js'; function coverage(entries: BlueprintEntry[]) { type Server = { name: string; game: string | null; map_slug: string | null; map: string | null }; @@ -76,18 +78,25 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): C const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; const lobby = nextLobbyNumber(base, names); const name = lobby === null ? null : `${base}_${lobby}`; - const port = nextAvailablePort(ports, releasingServers); + const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers); if (name) names.push(name); if (port !== null) ports.push(port); return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) }; })); } -function blockersFor(servers: CollectionPreview['servers']): string[] { +function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] { const blockers: string[] = []; if (!servers.length) return blockers; - if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Configure Komodo credentials and a target host before launching.'); - if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.'); + if (entries.some(entry => entry.recipe.native)) { + for (const entry of entries) { + try { nativeSelection(entry.recipe.native as unknown as Record, entry.recipe.game); } + catch (error) { blockers.push((error as Error).message); } + } + } else { + if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.'); + if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.'); + } if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`); return blockers; @@ -95,7 +104,7 @@ function blockersFor(servers: CollectionPreview['servers']): string[] { function removalBlockersFor(removals: CollectionPreview['removals']): string[] { const blockers: string[] = []; - if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.'); + if (removals.some(server => !nativeInstance(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.'); for (const server of removals) { const blocker = activeBetaRemovalBlocker(server.name); if (blocker) blockers.push(blocker); @@ -103,14 +112,21 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] { return blockers; } -export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview { +export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record): CollectionPreview { if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.'); if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); return db().transaction((): CollectionPreview => { const { entries, blueprintName } = blueprintEntries(blueprintId); + if (selection?.hostId !== undefined || !config.komodo.address) { + const libraries = selection?.libraryIds as Record | undefined; + for (const entry of entries) { + entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); + entry.recipe.image = entry.recipe.native.image!; + } + } const { counts, retainedServers, removals } = collectionPlan(entries, mode); const servers = allocate(entries, removals.map(server => server.name)); - const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)]; + const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString()); @@ -141,7 +157,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const { retainedServers, removals } = collectionPlan(entries, preview.mode); const releasingServers = removals.map(server => server.name); const servers = allocate(entries, releasingServers); - const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)]; + const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) { throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.'); diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index eaa7a17..da2611f 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -1,4 +1,4 @@ -import { readdir, readFile, stat } from 'node:fs/promises'; +import { readdir, readFile, stat } from '../backends/files.js'; import { randomUUID } from 'node:crypto'; import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js'; import { join, posix, relative, sep } from 'node:path'; diff --git a/src/lib/server/fleet/files.ts b/src/lib/server/fleet/files.ts index f110afd..aa91499 100644 --- a/src/lib/server/fleet/files.ts +++ b/src/lib/server/fleet/files.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { readFile, stat } from 'node:fs/promises'; +import { readFile, stat } from '../backends/files.js'; export async function sha256(path: string): Promise { const buf = await readFile(path); diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 50f0f1e..2902498 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -9,6 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '. import { sha256Buffer } from './files.js'; export interface LaunchRecipe { + native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }; format: 1; game: ZombieGame; mapSlug: string; diff --git a/src/lib/server/fleet/logs.ts b/src/lib/server/fleet/logs.ts index 6200201..6786732 100644 --- a/src/lib/server/fleet/logs.ts +++ b/src/lib/server/fleet/logs.ts @@ -1,6 +1,7 @@ import { open, stat } from 'node:fs/promises'; import { resolve, sep } from 'node:path'; import { rootOfServer, gameOfServer } from '../config.js'; +import { nativeTail } from '../backends/files.js'; // Whitelisted log files exposed to the UI, mapped to their path relative // to each server's root directory. Adding a new supported log = one entry @@ -63,6 +64,8 @@ export async function tailLog( path: string, lines: number = DEFAULT_TAIL_LINES ): Promise { + const remote = await nativeTail(path, lines); + if (remote !== undefined) return remote; let st; try { st = await stat(path); diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index bfa7bd1..ffb51ce 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -3,6 +3,8 @@ import { existsSync } from 'node:fs'; import { config, destOf, serverDir, rootOfServer } from '../config.js'; import { db } from '../db/index.js'; import { parseServerName } from './map-aliases.js'; +import { nativeInstance } from '../backends/native.js'; +import { stat as managedStat } from '../backends/files.js'; /** * All server folders under the game roots. Used by seedServerRegistry to backfill @@ -46,6 +48,10 @@ export async function ensureDest( create: boolean ): Promise<{ ok: true; path: string } | { ok: false; reason: string }> { const dest = destOf(server); + if (nativeInstance(server)) { + try { return (await managedStat(dest)).isDirectory() ? { ok: true, path: dest } : { ok: false, reason: 'Agent script directory is unavailable' }; } + catch (error) { return { ok: false, reason: (error as Error).message }; } + } if (existsSync(dest)) { try { const s = await stat(dest); @@ -63,6 +69,10 @@ export async function ensureDest( // -- server registry -- export interface ServerRow { + native_instance_id?: string | null; + host_id?: string | null; + instance_generation?: number | null; + game_library_id?: string | null; name: string; komodo_type: 'deployment' | 'stack' | 'container'; komodo_name: string; @@ -146,11 +156,13 @@ export async function seedServerRegistry(): Promise { } export function getServer(name: string): ServerRow | undefined { - return db().prepare('SELECT * FROM servers WHERE name = ?').get(name) as ServerRow | undefined; + return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id + FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`).get(name) as ServerRow | undefined; } export function listServers(): ServerRow[] { - return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[]; + return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id + FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`).all() as ServerRow[]; } /** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */ diff --git a/src/lib/server/game-files/http.ts b/src/lib/server/game-files/http.ts new file mode 100644 index 0000000..f8f828c --- /dev/null +++ b/src/lib/server/game-files/http.ts @@ -0,0 +1,44 @@ +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import { json } from '@sveltejs/kit'; +import { getLibraryFile } from './index.js'; +import { assertRegularPath, parseByteRange } from './storage.js'; + +export function gameFileError(cause: unknown): Response { + const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400; + return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.' }, { status }); +} +export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise> { + if (!request.body) throw new Error('A JSON request body is required.'); + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; let bytes = 0; + try { + while (true) { + const next = await reader.read(); if (next.done) break; + bytes += next.value.byteLength; + if (bytes > maxBytes) { await reader.cancel(); throw new Error('Request body exceeds its size limit.'); } + chunks.push(next.value); + } + } finally { reader.releaseLock(); } + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Expected a JSON object.'); + return body; +} + +export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise { + const file = getLibraryFile(versionId, requestedPath); + if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 }); + const path = await assertRegularPath(file.storageLocation, file.path); + if ((await stat(path)).size !== file.size) return json({ error: 'Stored file failed size verification.' }, { status: 503 }); + const etag = `"${file.sha256}"`; + const headers = new Headers({ 'Content-Type': 'application/octet-stream', 'Accept-Ranges': 'bytes', ETag: etag, 'Cache-Control': 'private, no-cache', 'X-Content-SHA256': file.sha256 }); + if (request.headers.get('if-none-match') === etag) return new Response(null, { status: 304, headers }); + let range; + try { range = parseByteRange(request.headers.has('if-range') && request.headers.get('if-range') !== etag ? null : request.headers.get('range'), file.size); } + catch { headers.set('Content-Range', `bytes */${file.size}`); return new Response(null, { status: 416, headers }); } + headers.set('Content-Length', String(range ? range.end - range.start + 1 : file.size)); + if (range) headers.set('Content-Range', `bytes ${range.start}-${range.end}/${file.size}`); + const body = request.method === 'HEAD' ? null : Readable.toWeb(createReadStream(path, range ?? undefined)) as ReadableStream; + return new Response(body, { status: range ? 206 : 200, headers }); +} diff --git a/src/lib/server/game-files/index.ts b/src/lib/server/game-files/index.ts new file mode 100644 index 0000000..21f15b7 --- /dev/null +++ b/src/lib/server/game-files/index.ts @@ -0,0 +1,256 @@ +import { randomUUID } from 'node:crypto'; +import { mkdir, rename, stat } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { db } from '../db/index.js'; +import { config } from '../config.js'; +import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js'; +import { ensureGameFileSchema } from './schema.js'; +import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js'; +import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js'; + +function database() { const handle = db(); ensureGameFileSchema(handle); return handle; } +export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files')); +const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id); +type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null }; +type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null }; +const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id }); +function versionFromRow(row: VersionRow): GameLibraryVersion { + const files = JSON.parse(row.manifest) as GameFileEntry[]; + return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version, + description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes, + files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at, + publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id, + sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id, + caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) }; +} +export function getLibraryVersion(id: string): GameLibraryVersion | null { + const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined; + return row ? versionFromRow(row) : null; +} +export function getLibraryFile(versionId: string, path: string) { + validateGamePath(path); + return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation + FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`) + .get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null; +} +export function newestReadyLibrary(game: string): GameLibraryVersion | null { + const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined; + return row ? versionFromRow(row) : null; +} +export function listLibraries(): GameLibrary[] { + reconcileLibraryOperations(); + return (database().prepare('SELECT * FROM game_file_libraries ORDER BY name').all() as Array<{ id: string; name: string; game: GameLibraryGame; description: string; created_at: string }>).map(row => ({ + id: row.id, name: row.name, game: row.game, description: row.description, createdAt: row.created_at, + versions: (database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.id=? ORDER BY v.created_at DESC, v.rowid DESC').all(row.id) as VersionRow[]).map(versionFromRow) + })); +} +export function getHostLibraryCache(hostId: string, versionId: string): HostLibraryCache | null { + const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined; + return row ? cacheFromRow(row) : null; +} +export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion { + const version = getLibraryVersion(versionId); + if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.'); + const cache = getHostLibraryCache(hostId, versionId); + if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.'); + return version; +} + +function textField(input: unknown, label: string, max: number, required = true): string { + if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`); + return input.trim(); +} +export function createLibraryImport(input: Record): GameLibraryVersion { + const game = input.game; + if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.'); + const hostId = textField(input.hostId, 'source host', 128); + const host = getHost(hostId); + if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled source host.'); + const name = textField(input.name, 'library name', 160); + const description = textField(input.description ?? '', 'description', 4000, false); + const version = textField(input.version, 'version label', 120); + const dlcInput = input.dlc ?? []; + if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.'); + const dlc = dlcInput.map(value => textField(value, 'DLC name', 120)); + const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files'; + const handle = database(); + let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined; + if (library && library.game !== game) throw new Error('This library name is already used for another game.'); + const id = randomUUID(); const now = new Date().toISOString(); + handle.transaction(() => { + if (!library) { + library = { id: randomUUID(), game }; + handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now); + } + if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.'); + handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) + VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume); + })(); + try { + const operation = queueHostOperation(hostId, 'library.import', { + importId: id, game, sourceVolume, manifestUrl: `/api/agent/imports/${id}`, + uploadBaseUrl: `/api/agent/imports/${id}/files`, completeUrl: `/api/agent/imports/${id}/complete` + }, { idempotencyKey: `library-import:${id}` }); + handle.prepare('UPDATE game_file_versions SET operation_id=? WHERE id=?').run(operation.id, id); + } catch (cause) { + handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=?").run((cause as Error).message, id); + throw cause; + } + return getLibraryVersion(id)!; +} + +export function requireImportOwner(hostId: string, id: string): GameLibraryVersion { + const version = getLibraryVersion(id); + if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.'); + return version; +} +export function setImportManifest(hostId: string, id: string, input: unknown): GameLibraryVersion { + const version = requireImportOwner(hostId, id); + const files = validateManifest(input); const checksum = manifestChecksum(files); + if (version.manifestSha256) { + if (version.manifestSha256 !== checksum) throw new Error('An import manifest cannot change. Start a new version.'); + return version; + } + if (version.state !== 'importing') throw new Error('This import is not writable.'); + database().transaction(() => { + database().prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=?,size_bytes=? WHERE id=? AND state=\'importing\'').run(JSON.stringify(files), checksum, files.reduce((sum, file) => sum + file.size, 0), id); + const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)'); + for (const file of files) insert.run(id, file.path, file.size, file.sha256); + })(); + return getLibraryVersion(id)!; +} +export async function uploadImportFile(hostId: string, id: string, path: string, body: ReadableStream | null): Promise { + const file = getLibraryFile(id, path); + if (!file || file.sourceHostId !== hostId) throw new Error('File is not listed in a manifest belonging to this host.'); + if (file.state !== 'importing' || publishing.has(id)) throw new Error('This import is not writable.'); + await receiveGameFile(stagingRoot(id), file, body); +} + +const publishing = new Map>(); +export async function publishImport(hostId: string, id: string): Promise { + const version = requireImportOwner(hostId, id); + if (version.state === 'ready') return version; + if (version.state !== 'importing' || !version.files.length || !version.manifestSha256) throw new Error('Import needs a complete manifest before publication.'); + const existing = publishing.get(id); if (existing) return existing; + const task = (async () => { + let root = stagingRoot(id); + // A restart between the atomic directory rename and metadata commit is recoverable. + try { if ((await stat(version.storageLocation)).isDirectory()) root = version.storageLocation; } + catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; } + for (const file of version.files) await verifyGameFile(root, file); + if (root !== version.storageLocation) { + await mkdir(dirname(version.storageLocation), { recursive: true }); + await rename(root, version.storageLocation); + } + database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id); + return getLibraryVersion(id)!; + })(); + publishing.set(id, task); + try { return await task; } finally { publishing.delete(id); } +} + +/** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */ +export function importPublicationStatus(hostId: string, id: string) { + const version = requireImportOwner(hostId, id); + return { + status: version.state === 'ready' ? 'published' as const : version.state === 'failed' ? 'failed' as const : publishing.has(id) ? 'verifying' as const : 'pending' as const, + version: { id: version.id, game: version.game, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 }, + error: version.error + }; +} +export function requestImportPublication(hostId: string, id: string) { + const version = requireImportOwner(hostId, id); + if (version.state === 'importing' && !publishing.has(id)) { + if (!version.files.length || !version.manifestSha256) throw new Error('Import requires a manifest before publication.'); + void publishImport(hostId, id).catch(cause => { + database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'") + .run((cause instanceof Error ? cause.message : 'Library publication failed.').slice(0, 2000), id); + }); + } + return importPublicationStatus(hostId, id); +} + +export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void { + if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.'); + const handle = database(); + const now = new Date().toISOString(); + handle.transaction(() => { + const reported = new Set(); + for (const raw of input) { + if (!raw || typeof raw !== 'object') throw new Error('Invalid library cache report.'); + const id = raw.libraryVersionId ?? raw.id; + if (typeof id !== 'string') throw new Error('Invalid library version ID.'); + if (reported.has(id)) throw new Error('Duplicate version in host cache report.'); + reported.add(id); + const version = getLibraryVersion(id); if (!version || version.state !== 'ready') continue; + let status = raw.status ?? raw.state; + if (!['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes(status)) throw new Error('Invalid library cache state.'); + const checksum = typeof raw.manifestSha256 === 'string' ? raw.manifestSha256 : null; + let error = typeof raw.error === 'string' ? raw.error.slice(0, 2000) : null; + if (['ready', 'outdated'].includes(status) && checksum !== version.manifestSha256) { status = 'failed'; error = 'Host manifest checksum does not match the published version.'; } + const bytes = raw.bytesDownloaded ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0); + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) throw new Error('Invalid library download progress.'); + if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { status = 'failed'; error = 'Host reported an incomplete library as verified.'; } + handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at) + VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status, + bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at`) + .run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now); + } + if (options.snapshot) { + const cached = handle.prepare('SELECT version_id FROM host_library_caches WHERE host_id=?').all(hostId) as { version_id: string }[]; + for (const cache of cached) if (!reported.has(cache.version_id)) { + handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=? AND version_id=?") + .run(now, hostId, cache.version_id); + } + } + })(); +} + +export function queueLibraryDownload(hostId: string, versionId: string) { + const version = getLibraryVersion(versionId); + if (!version || version.state !== 'ready') throw new Error('Choose a published library version.'); + const host = getHost(hostId); + if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.'); + if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.'); + const prior = getHostLibraryCache(hostId, versionId); + if (prior?.operationId) { + const operation = getHostOperation(prior.operationId); + if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) return operation; + } + const operation = queueHostOperation(hostId, 'library.download', { + libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`, + filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`, + manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes + }); + updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: 0 }]); + database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId); + return operation; +} + +export function reconcileLibraryOperations(): void { + const handle = database(); + for (const row of handle.prepare("SELECT id,operation_id FROM game_file_versions WHERE state='importing' AND operation_id IS NOT NULL").all() as { id: string; operation_id: string }[]) { + const operation = getHostOperation(row.operation_id); + if (operation?.state === 'failed') handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'").run(operation.error ?? 'Host import failed.', row.id); + } + for (const cache of handle.prepare("SELECT * FROM host_library_caches WHERE status IN ('downloading','verifying') AND operation_id IS NOT NULL").all() as CacheRow[]) { + const operation = getHostOperation(cache.operation_id!); + if (operation?.state === 'failed') handle.prepare("UPDATE host_library_caches SET status='failed',error=?,updated_at=? WHERE host_id=? AND version_id=?").run(operation.error ?? 'Host download failed.', new Date().toISOString(), cache.host_id, cache.version_id); + } +} + +/** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */ +export function libraryOverview() { + const hosts = listHosts(); + return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({ + ...version, caches: hosts.map(host => { + const cached = getHostLibraryCache(host.id, version.id); + const latest = library.versions.find(item => item.state === 'ready'); + if (cached) return { ...cached, status: cached.status === 'ready' && latest?.id !== version.id ? 'outdated' as const : cached.status }; + return { + hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0, + bytesTotal: version.sizeBytes, manifestSha256: null, error: null, updatedAt: version.createdAt, operationId: null + }; + }) + })) })), hosts }; +} diff --git a/src/lib/server/game-files/schema.ts b/src/lib/server/game-files/schema.ts new file mode 100644 index 0000000..fcf7f03 --- /dev/null +++ b/src/lib/server/game-files/schema.ts @@ -0,0 +1,39 @@ +import type Database from 'better-sqlite3'; + +const initialized = new WeakSet(); +export function ensureGameFileSchema(handle: Database.Database): void { + if (initialized.has(handle)) return; + handle.exec(` + CREATE TABLE IF NOT EXISTS game_file_libraries ( + id TEXT PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE UNIQUE, + game TEXT NOT NULL CHECK(game IN ('t5','t6')), description TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS game_file_versions ( + id TEXT PRIMARY KEY, library_id TEXT NOT NULL REFERENCES game_file_libraries(id), + version TEXT NOT NULL, description TEXT NOT NULL, dlc TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('importing','ready','failed')), + size_bytes INTEGER NOT NULL DEFAULT 0, manifest TEXT NOT NULL DEFAULT '[]', + manifest_sha256 TEXT, created_at TEXT NOT NULL, published_at TEXT, + storage_location TEXT NOT NULL, source_host_id TEXT NOT NULL, source_volume TEXT NOT NULL, + error TEXT, operation_id TEXT, UNIQUE(library_id, version) + ); + CREATE TABLE IF NOT EXISTS host_library_caches ( + host_id TEXT NOT NULL, version_id TEXT NOT NULL REFERENCES game_file_versions(id), + status TEXT NOT NULL CHECK(status IN ('missing','downloading','verifying','ready','failed','outdated')), + bytes_downloaded INTEGER NOT NULL DEFAULT 0, bytes_total INTEGER NOT NULL DEFAULT 0, + manifest_sha256 TEXT, error TEXT, updated_at TEXT NOT NULL, operation_id TEXT, + PRIMARY KEY(host_id, version_id) + ); + CREATE TABLE IF NOT EXISTS game_file_entries ( + version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL, + size INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(version_id,path) + ); + INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256) + SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256') + FROM game_file_versions v,json_each(v.manifest) f; + CREATE TRIGGER IF NOT EXISTS game_file_versions_immutable BEFORE UPDATE ON game_file_versions + WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END; + `); + initialized.add(handle); +} diff --git a/src/lib/server/game-files/storage.ts b/src/lib/server/game-files/storage.ts new file mode 100644 index 0000000..135b24c --- /dev/null +++ b/src/lib/server/game-files/storage.ts @@ -0,0 +1,103 @@ +import { createHash } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { mkdir, lstat, rename, rm, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, resolve, sep } from 'node:path'; +import { Readable, Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { randomUUID } from 'node:crypto'; +import type { GameFileEntry } from '../../game-files.js'; + +export function validateGamePath(input: unknown): string { + if (typeof input !== 'string' || !input || input.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(input) || Buffer.from(input).toString('utf8') !== input + || isAbsolute(input) || input.split('/').some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) { + throw new Error('Game file paths must be relative, normalized paths without traversal.'); + } + return input; +} + +export function validateManifest(input: unknown): GameFileEntry[] { + if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a manifest containing 1–200000 files.'); + const paths = new Set(); + let total = 0; + const files = input.map(raw => { + if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.'); + const path = validateGamePath(raw.path); + if (!Number.isSafeInteger(raw.size) || raw.size < 0 || typeof raw.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(raw.sha256)) throw new Error(`Invalid size or SHA256 for ${path}.`); + const key = path.toLowerCase(); + if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`); + paths.add(key); total += raw.size; + if (!Number.isSafeInteger(total)) throw new Error('Library size exceeds the supported limit.'); + return { path, size: raw.size as number, sha256: raw.sha256 as string }; + }).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); + for (const file of files) { + const parts = file.path.toLowerCase().split('/'); + parts.pop(); + while (parts.length) { + if (paths.has(parts.join('/'))) throw new Error('A manifest path cannot be both a file and a directory.'); + parts.pop(); + } + } + return files; +} + +export const manifestChecksum = (files: GameFileEntry[]): string => createHash('sha256').update(JSON.stringify(files)).digest('hex'); + +export function gameFilePath(root: string, path: string): string { + const base = resolve(root); + const result = resolve(base, validateGamePath(path)); + if (!result.startsWith(base + sep)) throw new Error('Invalid game file path.'); + return result; +} + +/** Refuse symlinks at every existing component, including the managed root. */ +export async function assertRegularPath(root: string, path: string, allowMissing = false): Promise { + const result = gameFilePath(root, path); + let current = resolve(root); + const components = [current, ...validateGamePath(path).split('/').map(part => current = resolve(current, part))]; + for (let i = 0; i < components.length; i++) { + let info; + try { info = await lstat(components[i]); } + catch (cause) { if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue; throw cause; } + if (info.isSymbolicLink() || (i < components.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('Game storage contains a non-regular path.'); + } + return result; +} + +export async function verifyGameFile(root: string, file: GameFileEntry): Promise { + const path = await assertRegularPath(root, file.path); + const info = await stat(path); + if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`); + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + if (hash.digest('hex') !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`); +} + +/** Stream to a private temporary file; publish only after byte count and SHA256 match. */ +export async function receiveGameFile(root: string, file: GameFileEntry, body: ReadableStream | null): Promise { + if (!body) throw new Error('A file body is required.'); + const path = await assertRegularPath(root, file.path, true); + await mkdir(dirname(path), { recursive: true }); + const temp = `${path}.upload-${randomUUID()}`; + let size = 0; + const hash = createHash('sha256'); + const verifier = new Transform({ transform(chunk, _encoding, done) { + size += chunk.length; + if (size > file.size) return done(new Error('Upload exceeds the manifest size.')); + hash.update(chunk); done(null, chunk); + } }); + try { + await pipeline(Readable.fromWeb(body as never), verifier, createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true })); + if (size !== file.size || hash.digest('hex') !== file.sha256) throw new Error(`Upload checksum or size mismatch for ${file.path}.`); + await rename(temp, path); + } finally { await rm(temp, { force: true }); } +} + +export function parseByteRange(header: string | null, size: number): { start: number; end: number } | null { + if (!header) return null; + const match = /^bytes=(\d*)-(\d*)$/.exec(header); + if (!match || (!match[1] && !match[2]) || size === 0) throw new Error('Unsatisfiable range.'); + let start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2])); + let end = match[1] && match[2] ? Number(match[2]) : size - 1; + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start || (!match[1] && Number(match[2]) === 0)) throw new Error('Unsatisfiable range.'); + return { start, end: Math.min(end, size - 1) }; +} diff --git a/src/lib/server/hosts/http.ts b/src/lib/server/hosts/http.ts new file mode 100644 index 0000000..3f25b08 --- /dev/null +++ b/src/lib/server/hosts/http.ts @@ -0,0 +1,50 @@ +import { error, json } from '@sveltejs/kit'; +import type { Host } from '../../hosts.js'; +import { authenticateCredential } from './index.js'; + +export function requireAgentTransport(request: Request): void { + if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') error(426, 'Host agents require HTTPS.'); +} +export function authenticateAgent(request: Request): Host { + requireAgentTransport(request); + const authorization = request.headers.get('authorization'); + const token = authorization?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1]; + const host = token ? authenticateCredential(token) : null; + if (!host) error(401, 'A valid host credential is required.'); + return host; +} +export function requireManagementOrigin(request: Request): void { + const origin = request.headers.get('origin'); + if ((origin && origin !== new URL(request.url).origin) || request.headers.get('sec-fetch-site') === 'cross-site') error(403, 'Cross-origin host management requests are not allowed.'); + // Agent credentials only authorize agent endpoints, never host management. + if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) error(403, 'Agent credentials cannot manage the controller.'); +} +export async function objectBody(request: Request, maxBytes = 1024 * 1024): Promise> { + if (Number(request.headers.get('content-length')) > maxBytes) error(413, 'Request is too large.'); + const reader = request.body?.getReader(); + if (!reader) error(400, 'Request body must be valid JSON.'); + const chunks: Uint8Array[] = []; let received = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + received += value.byteLength; + if (received > maxBytes) { void reader.cancel().catch(() => {}); error(413, 'Request is too large.'); } + chunks.push(value); + } + } finally { reader.releaseLock(); } + const raw = Buffer.concat(chunks, received).toString('utf8'); + let body: unknown; + try { body = JSON.parse(raw); } catch { error(400, 'Request body must be valid JSON.'); } + if (!body || typeof body !== 'object' || Array.isArray(body)) error(400, 'Request body must be an object.'); + return body as Record; +} +export function hostApiError(cause: unknown): Response { + if (cause && typeof cause === 'object' && 'status' in cause && 'body' in cause) throw cause; + return json({ error: cause instanceof Error ? cause.message : 'Host request failed.' }, { status: 400 }); +} +export function installerCommand(controller: string, token: string): string { + const origin = new URL(controller).origin; + const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; + return `curl --fail --silent --show-error --proto '=https' ${quote(origin + '/api/agent/install.sh')} | sudo bash -s -- --controller ${quote(origin)} --token ${quote(token)}`; +} diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts new file mode 100644 index 0000000..97e7da2 --- /dev/null +++ b/src/lib/server/hosts/index.ts @@ -0,0 +1,248 @@ +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { isIP } from 'node:net'; +import { db } from '../db/index.js'; +import { ensureHostSchema } from './schema.js'; +import type { Host, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js'; + +export const AGENT_PROTOCOL_VERSION = 1; +export const HEARTBEAT_MAX_AGE_MS = 90_000; +const now = () => new Date().toISOString(); +const digest = (token: string) => createHash('sha256').update(token).digest('hex'); +function database() { const handle = db(); ensureHostSchema(handle); return handle; } +type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number }; +const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`; +function hostFromRow(row: HostRow): Host { + const inventory: HostInventory = JSON.parse(row.inventory); + const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS; + const reasons: string[] = []; + if (!row.enrolled_at) reasons.push('Agent has not enrolled.'); + else if (!row.credential_active) reasons.push('Agent credential has been revoked.'); + if (!connected) reasons.push('Agent heartbeat is unavailable or stale.'); + if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.'); + if (!inventory.dockerRunning) reasons.push('Docker is not ready.'); + if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.'); + if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.'); + const infrastructureReady = reasons.length === 0; + if (!inventory.runtimeImages?.length) reasons.push('Prepare a Plutainer runtime image.'); + if (!inventory.libraries?.some(library => library.state === 'ready')) reasons.push('Download and verify a game-file library.'); + if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.'); + if (row.mode === 'disabled') reasons.push('Host is disabled.'); + return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max, + bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes, + createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline', + readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at || (infrastructureReady && reasons.length) ? 'preparing' : reasons.length ? 'degraded' : 'ready', + readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active }; +} +export function listHosts(): Host[] { return (database().prepare(hostSql + ' ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); } +export function getHost(id: string): Host | null { + const row = database().prepare(hostSql + ' WHERE h.id = ?').get(id) as HostRow | undefined; + return row ? hostFromRow(row) : null; +} +function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; } +function textField(value: unknown, label: string, max = 200): string { + if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`); + return value.trim(); +} +function port(value: unknown): number { if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535) throw new Error('Ports must be whole numbers between 1 and 65535.'); return value; } +export function createHost(input: Record): Host { + const id = randomUUID(); const name = textField(input.name, 'Host name'); + const min = port(input.portMin ?? 28960); const max = port(input.portMax ?? 28980); + if (max < min) throw new Error('The last game port must not precede the first.'); + const bind = input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address'); + if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); + database().prepare('INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)').run(id, name, min, max, bind, now()); + return requireHost(id); +} +export function updateHost(id: string, input: Record): Host { + const host = requireHost(id); + const name = input.name === undefined ? host.name : textField(input.name, 'Host name'); + const mode = input.mode ?? host.mode; + if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.'); + const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax); + if (max < min) throw new Error('The last game port must not precede the first.'); + const bind = input.bindAddress === undefined ? host.bindAddress : textField(input.bindAddress, 'Bind address'); + if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); + const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address'); + const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes; + if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.'); + database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id); + return requireHost(id); +} + +export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } { + requireHost(hostId); + if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.'); + const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID(); + const expiresAt = new Date(Date.now() + ttlMs).toISOString(); + database().transaction(() => { + database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(now(), hostId); + database().prepare('INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)').run(id, hostId, digest(token), now(), expiresAt); + })(); + return { id, hostId, token, expiresAt }; +} +function issueCredential(hostId: string, credential = 'ec_host_' + randomBytes(32).toString('base64url')): string { + database().prepare('INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)').run(randomUUID(), hostId, digest(credential), now()); + return credential; +} +export function validateInventory(value: unknown): HostInventory { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Host inventory must be an object.'); + const inventory = value as HostInventory; + if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.'); + for (const key of ['hostname','agentVersion','os','architecture','cpu','dockerVersion']) { + if (inventory[key] !== undefined && (typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000)) throw new Error(`Invalid inventory ${key}.`); + } + for (const key of ['cpuCount','memoryTotal','memoryAvailable','storageTotal','storageAvailable','protocolVersion']) { + if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`); + } + for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); + if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); + if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library => + !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.'); + return inventory; +} +export function enrollHost(token: unknown, rawInventory: unknown): { hostId: string; credential: string; heartbeatSeconds: number } { + if (typeof token !== 'string' || token.length > 200) throw new Error('Enrollment token is invalid or expired.'); + const inventory = validateInventory(rawInventory); + return database().transaction(() => { + const enrollment = database().prepare('SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?').get(digest(token), now()) as { id: string; host_id: string } | undefined; + if (!enrollment) throw new Error('Enrollment token is invalid or expired.'); + const changed = database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL').run(now(), enrollment.id); + if (changed.changes !== 1) throw new Error('Enrollment token has already been consumed.'); + revokeCredentials(enrollment.host_id); + const credential = issueCredential(enrollment.host_id); + database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id); + return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 }; + })(); +} +export function authenticateCredential(token: string): Host | null { + if (!token || token.length > 200) return null; + const row = database().prepare('SELECT id,host_id FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL').get(digest(token)) as { id: string; host_id: string } | undefined; + if (!row) return null; + database().prepare('UPDATE ec_host_credentials SET last_used_at=? WHERE id=?').run(now(), row.id); + return getHost(row.host_id); +} +export function revokeCredentials(hostId: string): void { + requireHost(hostId); + database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId); +} +export function rotateCredential(hostId: string, nextCredential?: string): string { + requireHost(hostId); + if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential)) throw new Error('New credentials must contain 32 random bytes encoded as base64url.'); + return database().transaction(() => { + if (nextCredential) { + const existing = database().prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?').get(digest(nextCredential)) as { host_id: string; revoked_at: string | null } | undefined; + if (existing) { + if (existing.host_id !== hostId || existing.revoked_at) throw new Error('This credential cannot be reused.'); + return nextCredential; + } + } + revokeCredentials(hostId); return issueCredential(hostId, nextCredential); + })(); +} +export function heartbeat(hostId: string, value: unknown): Host { + requireHost(hostId); const inventory = validateInventory(value); + database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId); + return requireHost(hostId); +} +export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host { + const host = requireHost(hostId); const reasons = [...host.readinessReasons]; + if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.'); + if (options.libraryId && !host.inventory.libraries?.some(library => library.id === options.libraryId && library.state === 'ready')) reasons.push('The selected game library is not ready on this host.'); + if (options.image && !host.inventory.runtimeImages?.includes(options.image)) reasons.push('The selected runtime image is not cached on this host.'); + if (options.port !== undefined && !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)) reasons.push('The selected host port is not available.'); + if (reasons.length) throw new Error(reasons.join(' ')); + return host; +} + +type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string }; +function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } +export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; } +export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } +export function queueHostOperation(hostId: string, kind: string, payload: Record, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation { + const host = requireHost(hostId); + if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.'); + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.'); + if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); + return database().transaction(() => { + if (options.idempotencyKey) { + const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined; + if (existing) { + if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || existing.payload !== JSON.stringify(payload)) throw new Error('Operation idempotency key is already associated with a different request.'); + return operationFromRow(existing); + } + } + if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); + if (host.mode === 'disabled') throw new Error('Host is disabled. Enable it before requesting operations.'); + const id = options.id ?? randomUUID(); const timestamp = now(); + database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp); + database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); + return getHostOperation(id)!; + })(); +} +export function pendingHostOperations(hostId: string): HostOperation[] { + const host = requireHost(hostId); + if (host.mode === 'disabled' || !host.connected) return []; + // Accepted/running/unknown commands keep the same immutable ID. The agent + // consults its durable journal before acknowledging or executing a replay. + return database().transaction(() => { + const rows = database().prepare(`SELECT * FROM ec_host_operations WHERE host_id=? AND state IN ('pending','accepted','running','unknown') + AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create')) + ORDER BY COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`).all(hostId, Number(host.readiness === 'ready')) as OperationRow[]; + const deliveredAt = now(); + for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); + return rows.map(operationFromRow); + })(); +} +export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation { + return database().transaction(() => { + const operation = getHostOperation(id); + if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.'); + if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.'); + if (['completed','failed'].includes(operation.state)) return operation; + const state = input.state as HostOperationState; + if (operation.state === 'running' && state === 'accepted') return operation; + const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000); + const result = input.result === undefined ? null : JSON.stringify(input.result); + if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); + database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id); + if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error); + return getHostOperation(id)!; + })(); +} +export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs; + while (true) { + const operation = getHostOperation(id); + if (!operation) throw new Error('Host operation was not found.'); + if (['completed','failed','unknown'].includes(operation.state)) return operation; + if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`); + await new Promise(resolve => setTimeout(resolve, 200)); + } +} + +export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean { + const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress; + if (candidate < host.portMin || candidate > host.portMax) return false; + const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[]; + return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address))); +} +export function nextHostPort(hostId: string, additionalPorts: number[] = [], bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp'): number | null { + const host = requireHost(hostId); + for (let candidate = host.portMin; candidate <= host.portMax; candidate++) if (!additionalPorts.includes(candidate) && hostPortAvailable(hostId, candidate, bindAddress, protocol)) return candidate; + return null; +} +export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number { + const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp'; + if (!isIP(bind)) throw new Error('Bind address must be an IP address.'); + if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.'); + if (!instanceId) throw new Error('An immutable instance ID is required.'); + return database().transaction(() => { + const existing = database().prepare('SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?').get(hostId, instanceId, bind, protocol) as { port: number } | undefined; + if (existing && (options.port === undefined || options.port === existing.port)) return existing.port; + const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol); + if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) throw new Error('No available game port on the selected host.'); + database().prepare('INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)').run(hostId, instanceId, bind, protocol, allocated, now()); + return allocated; + })(); +} +export function releaseHostPorts(instanceId: string): void { database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId); } diff --git a/src/lib/server/hosts/releases.ts b/src/lib/server/hosts/releases.ts new file mode 100644 index 0000000..314a0fa --- /dev/null +++ b/src/lib/server/hosts/releases.ts @@ -0,0 +1,11 @@ +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +/** adapter-node copies public assets to build/client; Vite serves static in development. */ +export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'manifest.json'): Promise { + try { return await readFile(resolve('build/client/agent', file), 'utf8'); } + catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; + return readFile(resolve('static/agent', file), 'utf8'); + } +} diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts new file mode 100644 index 0000000..cf9e1b0 --- /dev/null +++ b/src/lib/server/hosts/schema.ts @@ -0,0 +1,47 @@ +import type Database from 'better-sqlite3'; + +const initialized = new WeakSet(); +export function ensureHostSchema(handle: Database.Database): void { + if (initialized.has(handle)) return; + handle.exec(` + CREATE TABLE IF NOT EXISTS ec_hosts ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, hostname TEXT NOT NULL DEFAULT '', + mode TEXT NOT NULL DEFAULT 'active' CHECK (mode IN ('active','draining','disabled')), + port_min INTEGER NOT NULL DEFAULT 28960, port_max INTEGER NOT NULL DEFAULT 28980, + bind_address TEXT NOT NULL DEFAULT '0.0.0.0', public_address TEXT NOT NULL DEFAULT '', + minimum_free_bytes INTEGER NOT NULL DEFAULT 1073741824, + created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}' + ); + CREATE TABLE IF NOT EXISTS ec_host_enrollments ( + id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, expires_at TEXT NOT NULL, consumed_at TEXT + ); + CREATE TABLE IF NOT EXISTS ec_host_credentials ( + id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, revoked_at TEXT, last_used_at TEXT + ); + CREATE TABLE IF NOT EXISTS ec_host_operations ( + id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), kind TEXT NOT NULL, + instance_id TEXT, generation INTEGER, idempotency_key TEXT, + payload TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending','accepted','running','completed','failed','unknown')), + result TEXT, error TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, last_delivered_at TEXT, + UNIQUE(host_id, idempotency_key) + ); + CREATE INDEX IF NOT EXISTS ec_host_operations_queue ON ec_host_operations(host_id, state, created_at); + CREATE TABLE IF NOT EXISTS ec_host_operation_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id), + state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT + ); + CREATE TABLE IF NOT EXISTS ec_host_ports ( + host_id TEXT NOT NULL REFERENCES ec_hosts(id), instance_id TEXT NOT NULL, + bind_address TEXT NOT NULL, protocol TEXT NOT NULL CHECK (protocol IN ('udp','tcp')), + port INTEGER NOT NULL CHECK (port BETWEEN 1 AND 65535), created_at TEXT NOT NULL, + PRIMARY KEY(host_id,bind_address,protocol,port) + ); + CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id); + `); + const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[]; + if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); + initialized.add(handle); +} diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index ed584b8..c7bd554 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -1,5 +1,6 @@ import { config, komodoConfigured } from '../config.js'; import { getServer } from '../fleet/servers.js'; +import { nativeHealth, nativeInstance } from '../backends/native.js'; // Ports `komodo_restart` in push_scripts.sh:622. Three shapes depending on // the tracked komodo_type: deployment | container | stack. @@ -353,6 +354,7 @@ async function fetchJsonWithSnippet( * the container endpoint isn't available. */ export async function komodoInspectHealthy(serverName: string): Promise { + if (getServer(serverName)?.native_instance_id) return nativeHealth(serverName); if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; const row = getServer(serverName); if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; diff --git a/src/lib/server/integrations/rcon.ts b/src/lib/server/integrations/rcon.ts index 93bfbca..515d6a3 100644 --- a/src/lib/server/integrations/rcon.ts +++ b/src/lib/server/integrations/rcon.ts @@ -5,6 +5,7 @@ import { getServer, setServerField } from '../fleet/servers.js'; import { latestGameSubmission } from '../fleet/lifecycle-evidence.js'; import { komodoResolveGamePort } from './komodo.js'; import { parsePlutoniumRevision } from '../../plutonium.js'; +import { nativeInstance, nativeRequest } from '../backends/native.js'; // Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like // \xff\xff\xff\xffrcon @@ -373,6 +374,26 @@ export function formatRawReply(buf: Buffer): string { * rather than just "unreachable". */ export async function rconStatusResult(server: string): Promise { + const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null; + if (native) { + const key = JSON.stringify(['agent', native.id, native.generation, latestGameSubmission(server)]); + const existing = statusInFlight.get(key); + if (existing) return existing; + const endpoint = { host: `agent:${native.host_id}`, port: native.port }; + const request = (async (): Promise => { + try { + const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' }); + const players = parsePlayerCount(reply.text); + if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) }; + const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text); + return { ok: true, endpoint, status: { players, + capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients), + map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null, + gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) }, rawReply: reply.text.slice(0, 4000) }; + } catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; } + })().finally(() => statusInFlight.delete(key)); + statusInFlight.set(key, request); return request; + } // Snapshot before any network work. A probe begun after a restart or a // replacement registration must not join one observing the old process. const createdAt = getServer(server)?.created_at; @@ -476,6 +497,14 @@ function positiveCapacity(value: string | undefined): number | null { /** Query the running process, not the updater's on-disk files. */ export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> { + const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null; + if (native) { + try { + const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'version' }); + const revision = parsePlutoniumRevision(reply.text); + return { revision, error: revision === null ? 'Agent RCON reply did not identify a runtime revision.' : null }; + } catch (error) { return { revision: null, error: (error as Error).message }; } + } const { endpoint } = await endpointFor(server); if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' }; const response = await sendRcon(endpoint.host, endpoint.port, 'version'); diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 50d7d58..3d02c8f 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { komodoConfigured } from '../config.js'; +import { nativeInstance } from '../backends/native.js'; import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; @@ -295,7 +296,7 @@ async function run(forceScan: boolean): Promise { const lastScan = setting('watchdog_last_scan'); if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return; setSetting('watchdog_last_scan', iso()); - if (!komodoConfigured()) { + if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) { setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return; } const errors: string[] = []; diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index dfd166c..25dfb79 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -6,12 +6,14 @@ import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, import { fleetSettings } from '../settings.js'; import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js'; import { readProgress, recordProgress, type ProgressUpdate } from './progress.js'; +import { bindNativeOperation } from './native.js'; export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string { const id = randomUUID(); db().transaction(() => { db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy); + bindNativeOperation(id, server); recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested', label: action === 'create' ? 'Server creation requested' : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } }); @@ -243,10 +245,10 @@ export function listDeployments() { export function interruptDeployments(): void { db().transaction(() => { - const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[]; + const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[]; const finishedAt = new Date().toISOString(); const error = 'The application restarted before this operation finished. Check the server before retrying.'; - db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running'`).run(finishedAt, error); + db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error); for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome', message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } }); })(); diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index ecdb551..548b43e 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -28,6 +28,8 @@ import { getVersion } from '../fleet/scripts.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; import { recordProgress, type ProgressUpdate } from './progress.js'; import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js'; +import { nativeInstance, nativeDeployment } from '../backends/native.js'; +import { previewNativeCreation, queueNativeCreation, queueNativeRebuild, cancelQueuedNativeCreation } from './native.js'; export type CreateServerFailure = | 'bad-request' @@ -49,6 +51,12 @@ export type DestroyServerResult = | { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] }; export interface CreateServerInput { + hostId?: string; + libraryId?: string; + image?: string; + bindAddress?: string; + publicAddress?: string; + publicPort?: number; game?: ZombieGame; // omitted by older clients: T5 gameMode: GameMode; // 'zombie' — only value supported today mapSlug: string; // 'cosmodrome' @@ -167,7 +175,8 @@ interface PreviewRow { } /** Read-only checks; a preview never reserves a slot or writes credentials. */ -export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview { +export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview { + if (input.hostId !== undefined || !config.komodo.address) return previewNativeCreation(input); if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.'); let recipe: LaunchRecipe; let blueprintName: string | null = null; @@ -208,6 +217,8 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue // releasingServers is internal to atomic collection queueing; HTTP callers cannot set it. export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult { + const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null; + if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input); return db().transaction((): CreateServerResult => { const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined; if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' }; @@ -243,6 +254,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st /** Reserve the same identity and port before removing any part of the instance. */ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult { + if (nativeInstance(name)) return queueNativeRebuild(name, triggeredBy); const blocker = activeBetaRemovalBlocker(name); if (blocker) return { ok: false, kind: 'conflict', message: blocker }; const server = getServer(name); @@ -535,6 +547,8 @@ function hasActiveCreation(name: string): boolean { } export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } { + const cancelled = cancelQueuedNativeCreation(name, triggeredBy); + if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled }; const blocker = activeBetaRemovalBlocker(name); if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') @@ -597,6 +611,7 @@ async function runDestructions(): Promise { } async function processDestruction(operation: DestructionJob): Promise { + if (nativeDeployment(operation.id)) return; if (creating || destroying.has(operation.server)) return; if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return; destroying.add(operation.server); @@ -859,6 +874,7 @@ async function runCreations(): Promise { // deployment. A known interrupted startup must retain its readiness gate. const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[]; for (const { id } of interrupted) { + if (nativeDeployment(id)) continue; if (resumeVerification(id)) continue; // A queued removal does not make the interrupted container disappear. // Keep its slot until removal finishes, without resuming normal work or @@ -883,6 +899,7 @@ async function runCreations(): Promise { WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[]; for (const job of checking) { const id = job.deployment_id; + if (nativeDeployment(id)) continue; if (getServer(job.server)?.game_port !== job.port) { finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' }); continue; @@ -980,6 +997,7 @@ async function runCreations(): Promise { const queue = creationQueue(); const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length); const eligible = queue.queued.filter(({ id }) => { + if (nativeDeployment(id)) return false; const blocker = creationBlocker(id); if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker }); else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity', diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts new file mode 100644 index 0000000..839d667 --- /dev/null +++ b/src/lib/server/provisioning/native.ts @@ -0,0 +1,404 @@ +import { randomBytes, randomUUID } from 'node:crypto'; +import { posix } from 'node:path'; +import { db } from '../db/index.js'; +import { config, relDestForKind, serverDir } from '../config.js'; +import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; +import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js'; +import { getLibraryVersion, requireReadyLibrary } from '../game-files/index.js'; +import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; +import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; +import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; +import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js'; +import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js'; +import { recordProgress } from './progress.js'; +import { fleetSettings } from '../settings.js'; +import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; +import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js'; +import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js'; +import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js'; +import { sha256Buffer } from '../fleet/files.js'; +import type { StagedFileChange } from '../fleet/staged-files.js'; +import type { LaunchPreview } from '../../deployment-plan.js'; +import type { ProvisioningAction } from '../../provisioning.js'; + +export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number } +type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; +interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; + instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null } +interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } + +export function nativeSelection(input: Record, game: string): NativeSelection { + if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.'); + if (typeof input.libraryId !== 'string' || !input.libraryId) throw new Error('Select a published game-file version.'); + const version = getLibraryVersion(input.libraryId); + if (!version || version.game !== game || version.state !== 'ready') throw new Error('Choose a published game-file version for the selected game.'); + const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; + if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); + const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 }); + requireReadyLibrary(host.id, version.id); + const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress; + if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.'); + if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.'); + if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.'); + return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress, + publicPort: input.publicPort as number | undefined }; +} + +export function previewNativeCreation(input: Record): LaunchPreview { + let recipe: NativeRecipe; + let blueprintName: string | null = null; + if (typeof input.deploymentId === 'string') { + const previous = recipeForDeployment(input.deploymentId); + if (!previous) throw new Error('The recorded deployment is unavailable.'); + recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug); + blueprintName = previous.blueprintName; + } else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); + else { + if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); + recipe = captureRecipe(input.game, input.mapSlug); + } + const selection = nativeSelection(input, recipe.game); + recipe.native = selection; recipe.image = selection.image!; + const base = `${recipe.game}_zombie_${recipe.mapSlug}`; + const lobby = nextLobbyNumber(base); + const name = lobby === null ? null : `${base}_${lobby}`; + const port = nextNativePort(selection); + const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); + db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)') + .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); + const checks: LaunchPreview['checks'] = [ + { label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` }, + { label: 'Game files', state: 'pass', detail: `Immutable library version ${selection.libraryId} is ready on the selected host.` }, + { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' }, + { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, + { label: 'Configuration', state: 'pass', detail: recipe.configSource } + ]; + return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks, + ready: checks.every(check => check.state !== 'blocked') }; +} + +export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null { + const excluded = [...taken]; + for (;;) { + const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp'); + if (port === null) return null; + // Both protocols are published by Plutainer and reserved together. + const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp'); + if (tcp === port) return port; + if (tcp !== null && tcp < port) { excluded.push(tcp); continue; } + excluded.push(port); + } +} + +export function queueNativeCreation(input: CreateServerInput): CreateServerResult { + try { + return db().transaction(() => { + const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as + { id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined; + if (input.previewId && !preview) throw new Error('Review a fresh launch preview.'); + if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id }; + if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.'); + const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined; + const game = input.game ?? 't5'; + if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.'); + const selection = nativeSelection((saved?.native ?? input) as unknown as Record, game); + const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! }; + const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`); + if (lobby === null) throw new Error('All lobby IDs are reserved.'); + const name = `${game}_zombie_${input.mapSlug}_${lobby}`; + const port = nextNativePort(selection); + if (!port) throw new Error('No available host port.'); + if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.'); + const key = input.plutoKey?.trim() || getServerKey(name); + if (!key) throw new Error(`Save a Plutonium server key for ${name}.`); + saveServerKey(name, key); + const instanceId = randomUUID(); + reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' }); + reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' }); + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at) + VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port, + selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString()); + const id = beginDeployment(name, 'create', input.createdBy); + db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id); + db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') + .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap); + db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null); + if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id); + deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' }); + return { ok: true as const, name, port, deploymentId: id }; + })(); + } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } +} + +export function bindNativeOperation(id: string, server: string): void { + const instance = nativeInstance(server); + if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation); +} + +/** Cancelling an unsubmitted launch releases only its local reservation. */ +export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null { + const instance = nativeInstance(server); + if (!instance || getServer(server)) return null; + return db().transaction(() => { + const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id + WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined; + if (!creation) return null; + const id = beginDeployment(server, 'destroy', triggeredBy); + const now = new Date().toISOString(); + db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?") + .run(now, now, creation.id); + db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id); + releaseHostPorts(instance.id); + deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted'); + finishDeployment(id, { ok: true }); + return id; + })(); +} + +export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult { + try { + const instance = nativeInstance(name)!; const server = getServer(name); + const blocker = activeBetaRemovalBlocker(name); + if (blocker) throw new Error(blocker); + if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.'); + requireReadyLibrary(instance.host_id, instance.library_id); + const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image, + native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } }; + return db().transaction(() => { + const id = beginDeployment(name, 'rebuild', triggeredBy); + db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); + db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') + .run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap); + db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe)); + deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' }); + return { ok: true as const, name, port: instance.port, deploymentId: id }; + })(); + } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } +} + +function filesFor(job: Job): FilePlan[] { + const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s + ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server + WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[]; + return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path), + contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null, + checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha })); +} + +/** Agent receipts are the authoritative before images; do not read a controller mount. */ +function applyReceipt(job: Job, result: Record) { + if (job.applied_at || !job.files_json) return; + const plans = JSON.parse(job.files_json) as FilePlan[]; + const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined; + db().transaction(() => { + for (const plan of plans) { + const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path); + if (!snapshot) throw new Error('Agent activation receipt is missing a before image.'); + const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange; + if (!row || row.state === 'applied') continue; + const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); + const prior = getServerScript(job.server, row.rel_path); + if (row.source_kind === 'beta') { + if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`) + .run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length); + db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=? + AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind); + db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path); + db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); + } + if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?') + .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); + if (row.content === null) removeScript(job.server, row.rel_path); + else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null, + sha256Buffer(row.content), row.content.length, row.inventory_source); + db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id); + } + db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); + })(); +} + +function createPayload(instance: NativeInstance, recipe: LaunchRecipe) { + const key = getServerKey(instance.server); + if (!key) throw new Error('The saved server key was removed.'); + return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address, + rconPassword: config.rcon.password || randomBytes(32).toString('hex'), + env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', + PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key }, + files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) }; +} + +function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); } +function complete(job: Job) { + db().transaction(() => { + db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); + db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id); + finishDeployment(job.id, { ok: true }); + })(); +} + +export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise { + // Draining prevents admission immediately and retires existing lobbies only through the normal safety queue. + for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) { + const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[]; + for (const instance of instances) { + if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue; + if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue; + const id = beginDeployment(instance.server, 'destroy', 'host-drain'); + db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); + deploymentStage(id, 'Host draining; waiting for a fresh empty lobby'); + } + } + const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at + FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id + WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[]; + for (const job of jobs) { + try { await processJob(job, eligibility); } + catch (error) { hold(job, (error as Error).message); } + } +} + +async function processJob(job: Job, eligibility: (job: Job) => string | null) { + let instance = nativeInstance(job.server); + if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) { + finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; + } + const host = getHost(instance.host_id); + if (!host?.connected || host.mode === 'disabled') { + db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server); + hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return; + } + if (job.operation_id) { + const operation = getHostOperation(job.operation_id); + if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; } + if (operation.state === 'failed') { + // The agent reports failed only when no mutation began. Ambiguous + // post-mutation failures remain unknown and retain their capacity. + db().transaction(() => { + db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id); + const files = JSON.parse(job.files_json ?? '[]') as FilePlan[]; + for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id); + finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' }); + })(); + return; + } + if (operation.state !== 'completed') { + const blocked = operation.result as { message?: string } | null; + hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return; + } + if (job.action === 'destroy') { + db().transaction(() => { + deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } }); + db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id); + releaseHostPorts(instance!.id); + db().prepare('DELETE FROM servers WHERE name=?').run(job.server); + complete(job); + })(); + return; + } + if (job.action === 'rebuild' && instance.generation === job.generation) { + db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation); + instance = nativeInstance(job.server)!; + } + applyReceipt(job, (operation.result ?? {}) as Record); + if (job.state !== 'awaiting-health') { + db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id); + recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' }); + } + const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect'); + if (job.action === 'stop') { + if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; } + db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server); + complete(job); return; + } + if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; } + const status = await probeServerStatus(job.server, operation.updatedAt); + if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; } + if (job.action === 'create' || job.action === 'rebuild') { + const recipe = recipeForDeployment(job.id)!.recipe; + if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; } + await checkRelease(); + const revision = await probeVersion(job.server); const target = freshRelease(); + if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; } + if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; } + } + db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server); + recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state }, + { key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] }); + complete(job); return; + } + const reason = eligibility(job); + if (reason) { hold(job, reason); return; } + if (job.action === 'destroy' || job.action === 'rebuild') { + const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; } + } + const files = job.files_json ? JSON.parse(job.files_json) as FilePlan[] : filesFor(job); + if (files.length && !job.stage_operation_id) { + const stage = queueHostOperation(instance.host_id, 'files.stage', { stageId: job.id, files }, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }); + db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?').run(stage.id, JSON.stringify(files), job.id); + hold(job, 'Staging and verifying files on the host.'); return; + } + if (job.stage_operation_id) { + const stage = getHostOperation(job.stage_operation_id); + if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; } + if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; } + if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) { + finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return; + } + } + const settings = fleetSettings(); + const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL + AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n; + if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } + if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } + if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 }); + if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id); + if (job.action !== 'create' && job.action !== 'start') { + // Fast controller check keeps occupied lobbies out of the operation slots. + // The agent repeats this immediately before its local mutation. + const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect'); + if (!['stopped','exited','created','dead','missing'].includes(health.state)) { + const occupancy = await probePlayerEvidence(job.server); + const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s + JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id + WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id); + if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; } + if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; } + } + } + const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; + if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return; + db().transaction(() => { + const live = nativeInstance(job.server); + if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.'); + if (job.action === 'destroy' || job.action === 'rebuild') { + const beta = activeBetaRemovalBlocker(job.server); + if (beta) { hold(job, beta); return; } + } + if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) { + finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return; + } + if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') { + db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) + DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id); + } + let payload: Record = files.length ? { stageId: job.id } : {}; + if (job.action === 'create' || job.action === 'rebuild') { + const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe; + const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image }; + payload = createPayload(instance!, recipe); + if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1; + db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); + if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) + VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), + recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port); + for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create'); + } + const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload, + { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id }); + db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id); + for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id); + db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); + deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null, + command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } }); + })(); +} diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 079de3d..7de2f8c 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -7,6 +7,7 @@ import { queueRebuildServer } from './instances.js'; import { restartRequestInFlight } from './restarts.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; import { recordProgress } from './progress.js'; +import { nativeInstance } from '../backends/native.js'; export interface LifecycleOperation { id: string; server: string; action: string; state: string; stage: string; @@ -56,9 +57,10 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle // stale file preconditions after a beta; its failure must not strand // the rollback that would restore the server. const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id); - if (source.kind === 'campaign' && !row.restarted_at && !rollback) { + if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) { id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL) AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id @@ -135,7 +137,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; - if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' }; + if (!nativeInstance(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' }; return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; if (blocker) return { ok: false as const, status: 409, error: blocker }; @@ -184,7 +186,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract { const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server') .all() as RestartSlot[]; for (const slot of slots) { + if (slot.operation_id && nativeDeployment(slot.operation_id)) continue; if (requestsInFlight.has(slot.server)) continue; // Recovery capacity transferred to a rebuild remains reserved until // teardown. Its health is verified by the creation worker, never here. diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index 8bd57d7..ecd2283 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -12,6 +12,8 @@ import { cancelProvisioning } from './requests.js'; import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; import { recordProgress } from './progress.js'; +import { nativeInstance, nativeDeployment } from '../backends/native.js'; +import { processNativeProvisioning } from './native.js'; const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']); export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? ''); @@ -98,6 +100,19 @@ export function processProvisioning(): Promise { export const processLifecycle = processProvisioning; async function run(): Promise { + await processNativeProvisioning(operation => { + if (operation.action === 'create') return null; + if (operation.action === 'destroy') { + const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL + AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1`).get(operation.server, operation.id); + return other ? 'Waiting for the current server operation.' : null; + } + const typed = operation as Operation; + const eligibility = sourceEligibility(typed); + if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; } + if (eligibility === 'held') return 'Waiting for the requester to resume.'; + return blocker(operation.server, operation.id, typed.action)?.message ?? null; + }); await processCreations(); await processDestructions(); await refreshRestartRollout(); @@ -105,6 +120,7 @@ async function run(): Promise { WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health')) ORDER BY started_at, rowid`).all() as Operation[]; for (const operation of operations) { + if (nativeDeployment(operation.id)) continue; try { await processOperation(operation); } catch (err) { // Observation failures are retried; remote mutations are never replayed. diff --git a/src/lib/ui/components/DeployCollectionDialog.svelte b/src/lib/ui/components/DeployCollectionDialog.svelte index ad90ebf..c2cc938 100644 --- a/src/lib/ui/components/DeployCollectionDialog.svelte +++ b/src/lib/ui/components/DeployCollectionDialog.svelte @@ -6,12 +6,19 @@ import Button from './primitives/Button.svelte'; import LaunchRecipe from './LaunchRecipe.svelte'; import ServerIdentity from './ServerIdentity.svelte'; - import type { CollectionMode, CollectionPreview } from '$lib/deployment-plan.js'; + import NativeHostSelection from './NativeHostSelection.svelte'; + import type { NativeDeploymentSelection } from '$lib/hosts.js'; + import type { ZombieGame } from '$lib/map-aliases.js'; + import type { BlueprintSummary, CollectionMode, CollectionPreview } from '$lib/deployment-plan.js'; let { open = $bindable(false), blueprintId = '' }: { open?: boolean; blueprintId?: string } = $props(); let dialog: HTMLDialogElement | undefined = $state(); let preview = $state(null); let mode = $state('fill'); + let games = $state([]); + let collectionName = $state('Launch collection'); + let hostSelection = $state({ hostId: '', libraryIds: {}, image: 'ghcr.io/ayymoss/plutainer:latest' }); + let hostReady = $state(false); let confirmRemovals = $state(false); let keys = $state>({}); let busy = $state(false); @@ -28,16 +35,31 @@ $effect(() => { if (!dialog) return; - if (open && !wasOpen) { wasOpen = true; mode = 'fill'; keys = {}; preview = null; keyNotice = ''; dialog.showModal(); void review(); } + if (open && !wasOpen) { wasOpen = true; mode = 'fill'; keys = {}; preview = null; keyNotice = ''; dialog.showModal(); void initialize(); } else if (!open && wasOpen) { wasOpen = false; session++; dialog.close(); } }); + async function initialize() { + const current = ++session; busy = true; error = ''; games = []; hostReady = false; + hostSelection = { hostId: '', libraryIds: {}, image: 'ghcr.io/ayymoss/plutainer:latest' }; + try { + const response = await fetch('/api/blueprints'); const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not load collection.'); + if (current !== session) return; + const blueprint = (body.blueprints as BlueprintSummary[]).find(item => item.id === blueprintId); + if (!blueprint) throw new Error('This collection is no longer available.'); + games = [...new Set(blueprint.entries.map(entry => entry.game))]; collectionName = blueprint.name; + } catch (cause) { if (current === session) error = (cause as Error).message; } + finally { if (current === session) busy = false; } + } + async function review() { + if (!hostReady) { error = 'Choose a ready host and cached game files for each game in this collection.'; return; } const current = ++session; const previous = preview; busy = true; error = ''; keyNotice = ''; confirmRemovals = false; try { - const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode }) }); + const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, hostId: hostSelection.hostId, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.'); if (current === session) { @@ -73,16 +95,17 @@ { if (isDialogBackdrop(event, dialog) && !busy) open = false; }} oncancel={event => { if (busy) event.preventDefault(); }} onclose={() => open = false}>
-
Collection launch

{preview?.blueprintName ?? 'Preparing your collection'}

+
Collection launch

{preview?.blueprintName ?? collectionName}

+ {#if games.length} { preview = null; confirmRemovals = false; }} />{/if}

{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this collection across both games.' : 'Create missing lobbies and keep all existing servers.'}

{#if error}{/if} - {#if !preview}

{busy ? 'Finding a lobby and port for every server…' : 'Refresh to prepare another preview.'}

{#if !busy}{/if} + {#if !preview}

{busy ? 'Preparing the collection…' : 'Select a host and game files, then review capacity and server keys.'}

{#if !busy}{/if} {:else} {#if keyNotice}

{keyNotice}

{/if} {#if expired}

This preview expired. Refresh before launch; keys remain entered for unchanged IDs.

{/if} @@ -122,7 +145,7 @@
{#if preview.servers.length}

{keysReady} / {preview.servers.length} keys available

{/if}

Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.

{#if keysReady < preview.servers.length}{/if}
{:else}

{preview.mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet already meets or exceeds every target in this collection.'} No servers will be created or removed.

{/if} -
Setup and launch requirements

Each server uses the latest promoted setup when its turn begins. The host must have the selected games and DLC. This preview checks settings and capacity; it does not probe game files, images, credentials, or network access.

{#each preview.entries as entry}
{/each}
+
Setup and launch requirements

Each new server uses the selected host, immutable game-file version, and prepared runtime. Host readiness, cached files, runtime availability, and local port reservations are checked again before deployment.

{#each preview.entries as entry}
{/each}
{/if}
diff --git a/src/lib/ui/components/DeployServerDialog.svelte b/src/lib/ui/components/DeployServerDialog.svelte index 0cd337f..907a115 100644 --- a/src/lib/ui/components/DeployServerDialog.svelte +++ b/src/lib/ui/components/DeployServerDialog.svelte @@ -6,6 +6,8 @@ import Button from './primitives/Button.svelte'; import LaunchRecipe from './LaunchRecipe.svelte'; import DeployCollectionDialog from './DeployCollectionDialog.svelte'; + import NativeHostSelection from './NativeHostSelection.svelte'; + import type { NativeDeploymentSelection } from '$lib/hosts.js'; import type { BlueprintSummary, LaunchPreview } from '$lib/deployment-plan.js'; import type { ZombieMapEntry, ZombieGame } from '$lib/map-aliases.js'; @@ -21,6 +23,8 @@ let maps = $derived(catalogs[game]); let selectedBlueprint = $derived(blueprints.find(b => b.id === blueprintId)); let preview = $state(null); + let hostSelection = $state({ hostId: '', libraryIds: {}, image: 'ghcr.io/ayymoss/plutainer:latest' }); + let hostReady = $state(false); let plutoKey = $state(''); let busy = $state(false); let loaded = $state(false); @@ -50,6 +54,7 @@ blueprintId = initialBlueprintId; game = initialGame; mapSlug = initialMapSlug; + hostSelection = { hostId: '', libraryIds: {}, image: 'ghcr.io/ayymoss/plutainer:latest' }; hostReady = false; try { const [mapRes, blueprintRes] = await Promise.all([fetch('/api/maps'), fetch('/api/blueprints')]); if (!mapRes.ok || !blueprintRes.ok) throw new Error('Could not load maps and collections. Try again.'); @@ -59,8 +64,13 @@ blueprints = blueprintBody.blueprints; if (!maps.some(m => m.slug === mapSlug)) mapSlug = maps[0]?.slug ?? ''; if (blueprintId && !blueprints.some(b => b.id === blueprintId)) throw new Error('This collection is no longer available. Close this dialog and choose another starting point.'); + if (sourceDeploymentId) { + const source = await fetch(`/api/provisioning/${encodeURIComponent(sourceDeploymentId)}`).then(response => response.json()); + if (current !== session) return; + if (!source.recipe) throw new Error('The original launch recipe is unavailable.'); + game = source.recipe.game; mapSlug = source.recipe.mapSlug; + } loaded = true; - if (sourceDeploymentId) await review(); } catch (err) { if (current === session) error = (err as Error).message; } } @@ -69,12 +79,15 @@ if (!sourceDeploymentId && selectedBlueprint) { open = false; collectionOpen = true; return; } + if (!hostReady) { error = 'Choose a ready host, cached game-file version, and prepared runtime.'; return; } const current = session; const previousName = preview?.name; busy = true; error = ''; savedName = ''; keyNotice = ''; try { const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : blueprintId ? { blueprintId } : { game, mapSlug }) }); + body: JSON.stringify({ ...(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : { game, mapSlug }), + hostId: hostSelection.hostId, libraryId: hostSelection.libraryIds[game], image: hostSelection.image, + bindAddress: hostSelection.bindAddress || undefined, publicAddress: hostSelection.publicAddress || undefined, publicPort: hostSelection.publicPort }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.'); if (current === session) { @@ -137,7 +150,7 @@ {#if preview.hasSavedKey}

Saved key ready

{:else}{/if}

{preview.hasSavedKey ? 'The saved key for this server ID will be reused automatically. Electric Cherry does not check whether Plutonium accepts it. Update it in Keys.' : 'Add the key here. It is saved for this exact server ID when you launch, including if provisioning fails.'} Manage saved keys in a new tab; your launch stays open here.

-
{#if !sourceDeploymentId}{/if}
+
{#if !preview.hasSavedKey && !plutoKey.trim()}

Enter this server’s key to enable launch.

{/if}

Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Lobby and port availability are checked again at launch.

@@ -147,14 +160,15 @@ {/if} {:else if sourceDeploymentId} -

{busy ? 'Preparing the current promoted setup…' : 'Review the latest promoted setup for this map with a fresh lobby and port.'}

+

Choose a host for the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.

{:else}
{ event.preventDefault(); void review(); }}> {#if selectedBlueprint}

{selectedBlueprint.totalServers} target server{selectedBlueprint.totalServers === 1 ? '' : 's'}

{#each selectedBlueprint.entries as entry}

{entry.count}× {entry.mapName} · {entry.game.toUpperCase()}

{/each}

Review missing servers and their keys together. Every new server uses the latest promoted scripts, plugins, and configuration.

{#if selectedBlueprint.description}

{selectedBlueprint.description}

{/if}
{:else}

Pick a map to review its current promoted scripts, plugins, and configuration. You can save the map as a collection for future launches.

{/if} + {#if !selectedBlueprint}{/if}
Know what you’re launching.

Review the current promoted setup, available lobby, and launch prerequisites before creating your server.

-
+
{/if} diff --git a/src/lib/ui/components/DestroyServerModal.svelte b/src/lib/ui/components/DestroyServerModal.svelte index ed1f3ac..11ad0f6 100644 --- a/src/lib/ui/components/DestroyServerModal.svelte +++ b/src/lib/ui/components/DestroyServerModal.svelte @@ -32,7 +32,7 @@
{ event.preventDefault(); void destroy(); }}>

Queue server removal?

{name}

-

This stops the container, deletes its Komodo deployment, and removes its server files and campaign details. Provisioning history and the saved server key are retained.

+

This stops and removes the container, its writable server files, and campaign details. Shared game libraries, provisioning history, and the saved server key are retained.

{#if players !== null && players > 0}{players} player{players === 1 ? '' : 's'} currently online.{/if} Removal waits until a live check confirms the server is empty. Follow progress or cancel the waiting request on the provisioning page. diff --git a/src/lib/ui/components/LifecycleDialog.svelte b/src/lib/ui/components/LifecycleDialog.svelte index 7796f81..41d46b1 100644 --- a/src/lib/ui/components/LifecycleDialog.svelte +++ b/src/lib/ui/components/LifecycleDialog.svelte @@ -53,7 +53,7 @@ {#each runtimeActions as value}{/each} -

{action === 'destroy' ? 'Destroy the container, delete its Komodo deployment, and remove all server files and registration after the server is empty. Provisioning history and the key saved separately in Keys are retained.' : runtimeDescriptions[action]}

+

{action === 'destroy' ? 'Destroy the container and remove its writable server files and registration after the server is empty. Shared game libraries, provisioning history, and the key saved separately in Keys are retained.' : runtimeDescriptions[action]}

{#if destructive}{/if} {#if error}{/if}
diff --git a/src/lib/ui/components/NativeHostSelection.svelte b/src/lib/ui/components/NativeHostSelection.svelte new file mode 100644 index 0000000..2c1d0e5 --- /dev/null +++ b/src/lib/ui/components/NativeHostSelection.svelte @@ -0,0 +1,41 @@ + + +
+

Host and game files

+ {#if error}{/if} + {#if loading}

Loading host readiness and libraries…

{/if} + + {#if !loading && !hosts.some(candidate => candidate.readiness === 'ready')}

No deployment-ready hosts. Set up a host, then refresh.

{/if} + {#if host}

{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}

{/if} + {#each games as game}{/each} +

Import or download game files to a host, then refresh. Each launch stays pinned to its selected library version.

+ + {#if host && !imageReady}

Prepare this runtime image from host setup before launch.

{/if} + {#if network}
Network overrides

Public mapping does not change the local Docker port reservation.

{/if} +
+ + diff --git a/src/lib/ui/components/ProvisioningProgress.svelte b/src/lib/ui/components/ProvisioningProgress.svelte index 1e500aa..8086f3f 100644 --- a/src/lib/ui/components/ProvisioningProgress.svelte +++ b/src/lib/ui/components/ProvisioningProgress.svelte @@ -9,7 +9,7 @@ const progress = $derived(detail.progress); let expanded = $state>({}); const evidenceLabels = { pass: 'Passed', waiting: 'Waiting', fail: 'Failed', unknown: 'Unknown' }; - const sources = { komodo: 'Komodo', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' }; + const sources = { komodo: 'Komodo', agent: 'Host agent', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' };
diff --git a/src/lib/ui/stores/fleet.ts b/src/lib/ui/stores/fleet.ts index f6af079..5733645 100644 --- a/src/lib/ui/stores/fleet.ts +++ b/src/lib/ui/stores/fleet.ts @@ -9,7 +9,7 @@ export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperati }); export const fleetError = writable(''); export const fleetRefreshing = writable(false); -export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string } | null>(null); +export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null); export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) => snapshot.servers.map(server => { diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts new file mode 100644 index 0000000..c55e153 --- /dev/null +++ b/src/routes/+layout.server.ts @@ -0,0 +1,4 @@ +import { setupRequired } from '$lib/server/auth/index.js'; +import type { LayoutServerLoad } from './$types'; + +export const load: LayoutServerLoad = ({ locals }) => ({ user: locals.user ?? null, setupRequired: setupRequired() }); diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 54b3a1d..5863dd6 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -15,17 +15,24 @@ import { resolveConfirmation } from '$ui/confirm.js'; import '$ui/styles/components.css'; import { enableViewTransitions } from '$ui/motion/viewTransition.js'; + import type { AuthUser } from '$lib/auth.js'; // Side-effect import — puts @fontsource CSS on Vite's module graph so // the referenced woff/woff2 files get emitted into the build. import '$ui/styles/fonts.js'; - interface Props { children: import('svelte').Snippet } - let { children }: Props = $props(); + interface Props { children: import('svelte').Snippet; data: { user: AuthUser | null; setupRequired: boolean } } + let { children, data }: Props = $props(); + const publicPage = $derived(['/login', '/setup'].includes($page.url.pathname)); + const sessionReady = $derived(!!data.user && !data.user.mustChangePassword && !publicPage); let menuOpen = $state(false); + let accountOpen = $state(false); + let signingOut = $state(false); + let signOutError = $state(''); let notificationsOpen = $state(false); let helpOpen = $state(false); const unread = $derived($notificationHistory.filter(item => !item.read).length); - let menuButton: HTMLButtonElement; + let menuButton = $state(); + let accountButton = $state(); function openUtility(panel: 'notifications' | 'help') { if (menuOpen) { menuButton?.focus(); menuOpen = false; } @@ -33,32 +40,51 @@ else helpOpen = true; } - afterNavigate(() => { menuOpen = false; resolveConfirmation(false); }); + afterNavigate(() => { menuOpen = false; accountOpen = false; resolveConfirmation(false); }); function closeMenu(event: KeyboardEvent) { + if (event.key === 'Escape' && accountOpen) { accountOpen = false; accountButton?.focus(); } if (event.key === 'Escape' && menuOpen) { menuOpen = false; menuButton?.focus(); } } - onMount(() => { - const stopTimeZone = startTimeZonePreference(); + async function signOut() { + if (signingOut) return; + signingOut = true; signOutError = ''; + try { + const response = await fetch('/api/auth/logout', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' }); + if (!response.ok) { const body = await response.json(); throw new Error(body.error ?? 'Could not sign out. Try again.'); } + try { sessionStorage.removeItem('electric-cherry:notifications'); } catch { /* Optional local history. */ } + notificationHistory.set([]); + window.location.assign('/login'); + } catch (cause) { signOutError = (cause as Error).message; signingOut = false; } + } + + $effect(() => { + if (!sessionReady) return; const stopNotifications = startNotificationHistory(); const stopPipeline = startPipelineStream(); const stopServers = startServerStream(); - const stopFleetMonitor = startFleetMonitor(); + const stopFleet = startFleetMonitor(); + return () => { stopNotifications(); stopFleet(); stopPipeline?.(); stopServers?.(); }; + }); + + onMount(() => { + const stopTimeZone = startTimeZonePreference(); // Two-key nav shortcuts (Gmail-style): `g f` fleet, `g d` provisioning, // `g b` betas, `g s` scripts. Ignored while typing in inputs. const shortcuts: Record = { f: '/', d: '/provisioning', a: '/campaigns', w: '/maintenance', b: '/betas', - k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics' + k: '/server-keys', s: '/scripts', c: '/configs', p: '/plugins', l: '/leaderboard', h: '/settings', m: '/metrics', o: '/hosts' }; let armed = false; let armTimer: ReturnType | null = null; const onKey = (e: KeyboardEvent) => { + if (!sessionReady) return; const target = e.target as HTMLElement; if (target && (['INPUT', 'TEXTAREA', 'SELECT'].includes(target.tagName) || target.isContentEditable || target.closest('dialog'))) return; if (e.metaKey || e.ctrlKey || e.altKey) return; @@ -74,6 +100,7 @@ armed = false; if (armTimer) { clearTimeout(armTimer); armTimer = null; } const dest = shortcuts[e.key]; + if ((e.key === 'h' || e.key === 'k') && data.user?.role !== 'admin') return; if (dest) { e.preventDefault(); goto(dest); @@ -83,10 +110,6 @@ window.addEventListener('keydown', onKey); return () => { stopTimeZone(); - stopNotifications(); - stopFleetMonitor(); - stopPipeline?.(); - stopServers?.(); window.removeEventListener('keydown', onKey); if (armTimer) clearTimeout(armTimer); }; @@ -98,7 +121,8 @@ const tabs = [ { href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') }, - { href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins)(\/|$)/.test(p) }, + { href: '/hosts', label: 'Hosts', match: (p: string) => /^\/hosts(\/|$)/.test(p) }, + { href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins|game-files)(\/|$)/.test(p) }, { href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|maintenance|watchdog|server-keys)(\/|$)/.test(p) }, { href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') }, { href: '/settings', label: 'Settings', match: (p: string) => p.startsWith('/settings') }, @@ -106,29 +130,33 @@ { href: '/matches', label: 'Match History', match: (p: string) => p.startsWith('/matches') }, { href: '/metrics', label: 'Metrics', match: (p: string) => p === '/metrics' } ]; - const secondary = $derived(tabs[1].match($page.url.pathname) - ? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }] - : tabs[2].match($page.url.pathname) - ? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/maintenance', label: 'Maintenance' }, { href: '/server-keys', label: 'Keys' }] - : tabs[4].match($page.url.pathname) ? [{ href: '/settings', label: 'Preferences' }] : []); + const visibleTabs = $derived(tabs.filter(tab => tab.label !== 'Settings' || data.user?.role === 'admin')); + const secondary = $derived(tabs.find(tab => tab.label === 'Library')!.match($page.url.pathname) + ? [{ href: '/scripts', label: 'Scripts' }, { href: '/configs', label: 'Configs' }, { href: '/plugins', label: 'Plugins' }, { href: '/game-files', label: 'Game Files' }] + : tabs.find(tab => tab.label === 'Operations')!.match($page.url.pathname) + ? [{ href: '/provisioning', label: 'Provisioning' }, { href: '/collections', label: 'Collections' }, { href: '/campaigns', label: 'Campaigns' }, { href: '/maintenance', label: 'Maintenance' }, ...(data.user?.role === 'admin' ? [{ href: '/server-keys', label: 'Keys' }] : [])] + : tabs.find(tab => tab.label === 'Settings')!.match($page.url.pathname) ? [{ href: '/settings', label: 'Preferences' }, ...(data.user?.role === 'admin' ? [{ href: '/settings/users', label: 'Users' }] : [])] : []);
+ {#if publicPage} +
+ {:else} - {#if secondary.length}
{#each secondary as item}{item.label}{/each}
{/if} -
+ {#if sessionReady && secondary.length}
{#each secondary as item}{item.label}{/each}
{/if} + {/if} +
+ {#if sessionReady && data.user?.role === 'viewer'}{/if} {@render children()}
-
{#each [['g f','Fleet'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g w','Maintenance'],['g b','Betas'],['g h','Settings'],['g k','Keys'],['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}
{keys}
{label}
{/each}
+
{#each [['g f','Fleet'],['g o','Hosts'],['g s','Scripts'],['g c','Configs'],['g p','Plugins'],['g d','Provisioning'],['g a','Campaigns'],['g w','Maintenance'],['g b','Betas'],...(data.user?.role === 'admin' ? [['g h','Settings'],['g k','Keys']] : []),['g l','Leaderboard'],['g m','Metrics'],['?','This help']] as [keys,label]}
{keys}
{label}
{/each}
@@ -193,6 +228,16 @@ } .shell { min-height: 100vh; } + .auth-brand { display:flex; justify-content:center; padding:48px 20px 20px; } + main.auth-main { max-width:520px; padding-top:16px; } + .viewer-note { display:flex; gap:8px 16px; flex-wrap:wrap; padding:14px 18px; margin-bottom:24px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface); font-size:var(--text-sm); color:var(--text-secondary); } .viewer-note strong { color:var(--text-primary); } + .account-menu { position:relative; } + .account-button { display:flex; align-items:center; gap:8px; max-width:220px; overflow:hidden; text-overflow:ellipsis; } + .role { text-transform:capitalize; color:var(--text-tertiary); font-size:var(--text-xs); } + .account-links { position:absolute; right:0; top:calc(100% + 8px); min-width:200px; z-index:30; display:grid; padding:8px; background:var(--surface); border:1px solid var(--border-strong); border-radius:var(--radius-md); box-shadow:0 12px 32px #0005; } + .account-links a,.account-links button { text-align:left; padding:10px 12px; color:var(--text-primary); background:none; border:0; border-radius:var(--radius-sm); font:inherit; font-size:var(--text-sm); cursor:pointer; text-decoration:none; } + .account-links a:hover,.account-links button:hover { background:var(--surface-raised); } + .account-links p { padding:0 12px; font-size:var(--text-xs); color:var(--state-danger); } nav { position: sticky; @@ -272,7 +317,7 @@ } /* The full navigation needs more room than a phone or portrait tablet. */ - @media (max-width: 1200px) { + @media (max-width: 1500px) { nav { flex-wrap: wrap; justify-content: space-between; @@ -303,6 +348,9 @@ .tabs a { display: flex; align-items: center; min-height: 44px; border-radius: var(--radius-md); } .nav-tools { display: none; } .nav-tools.expanded { display: flex; width: 100%; padding-top: 8px; } + .nav-tools.account-only { display:flex; } + .account-menu { margin-left:auto; } + .account-links { position:static; margin-top:8px; } } @media (max-width: 720px) { main { padding: var(--space-6) max(var(--space-4), env(safe-area-inset-right)) calc(var(--space-16) + env(safe-area-inset-bottom)) max(var(--space-4), env(safe-area-inset-left)); } diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 4febfc6..d3ba8e7 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -31,6 +31,7 @@ let deployOpen = $state(false); let initialGame = $state('t5'); let initialMapSlug = $state(''); + const canOperate = $derived($page.data.user?.role === 'admin' || $page.data.user?.role === 'operator'); const filters: { id: FleetFilter; label: string }[] = [ { id: 'all', label: 'All' }, { id: 'occupied', label: 'Occupied' }, { id: 'empty', label: 'Empty' }, { id: 'attention', label: 'Needs attention' }, { id: 'testing', label: 'Testing' }, { id: 'pinned', label: 'Pinned' } @@ -128,10 +129,10 @@ {#snippet actions()} - + {#if canOperate}{/if} {/snippet} - + {#if canOperate}{/if} {#if $fleetError} {/if} - {#if $fleetIntegrations && (!$fleetIntegrations.komodo || !$fleetIntegrations.rcon)} + {#if $fleetIntegrations?.native && !$fleetIntegrations.hostsReady} +
Prepare a game host · {$fleetIntegrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}Review hosts →
+ {:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !$fleetIntegrations.native && (!$fleetIntegrations.komodo || !$fleetIntegrations.rcon)}
Finish system setup · {!$fleetIntegrations.komodo ? 'Komodo' : ''}{!$fleetIntegrations.komodo && !$fleetIntegrations.rcon ? ' and ' : ''}{!$fleetIntegrations.rcon ? 'RCON' : ''} not configured.Review setup →
{/if} @@ -176,7 +179,7 @@ {#if servers.length === 0} {#snippet actions()} - + {#if canOperate}{/if} {/snippet} @@ -205,7 +208,7 @@ {#each groups as [key, group] (key)}
{#if grouping !== 'none'}
{#if grouping === 'map'}{group.game.toUpperCase()}{/if}

{group.title}

{group.servers.length} {group.servers.length === 1 ? 'lobby' : 'lobbies'} · {group.servers.some(s => freshServer(s, $uiNow)) ? group.servers.reduce((sum, s) => sum + (freshServer(s, $uiNow) ? s.last_players ?? 0 : 0), 0) : '—'} players · {group.servers.filter(s => freshServer(s, $uiNow) && s.last_players === 0).length} empty

- {#if group.launch}{/if} + {#if canOperate && group.launch}{/if}
{/if} {#if view === 'cards'}
{#each group.servers as server (server.name)} diff --git a/src/routes/account/+page.svelte b/src/routes/account/+page.svelte new file mode 100644 index 0000000..6e774ff --- /dev/null +++ b/src/routes/account/+page.svelte @@ -0,0 +1,57 @@ + + +My account · Electric Cherry +
+ {#if user} +

{user.displayName}

@{user.username} · {user.role}

+ {#if user.mustChangePassword}

Your account uses a temporary password. Choose a new password before continuing to Electric Cherry.

{/if} +

{user.mustChangePassword ? 'Choose your password' : 'Change password'}

+ {#if error}{/if}{#if notice}

{notice}

{/if} +
+ + + +

Use 15–128 characters and a password you do not use elsewhere.

+ +
+
+
+ {/if} +
+ + diff --git a/src/routes/api/agent/credentials/+server.ts b/src/routes/api/agent/credentials/+server.ts new file mode 100644 index 0000000..8795144 --- /dev/null +++ b/src/routes/api/agent/credentials/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { rotateCredential } from '$lib/server/hosts/index.js'; +import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; +export async function POST({ request }: { request: Request }) { + const host = authenticateAgent(request); + try { + const body = await objectBody(request); + if (typeof body.credential !== 'string') throw new Error('Persist a randomly generated replacement credential before requesting rotation.'); + rotateCredential(host.id, body.credential); + return json({ ok: true }, { headers: { 'Cache-Control': 'no-store' } }); + } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/agent/enroll/+server.ts b/src/routes/api/agent/enroll/+server.ts new file mode 100644 index 0000000..19b1d87 --- /dev/null +++ b/src/routes/api/agent/enroll/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import { enrollHost } from '$lib/server/hosts/index.js'; +import { hostApiError, objectBody, requireAgentTransport } from '$lib/server/hosts/http.js'; +export async function POST({ request }: { request: Request }) { + requireAgentTransport(request); + try { const body = await objectBody(request); return json(enrollHost(body.token, body.inventory), { headers: { 'Cache-Control': 'no-store' } }); } + catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/agent/heartbeat/+server.ts b/src/routes/api/agent/heartbeat/+server.ts new file mode 100644 index 0000000..f405084 --- /dev/null +++ b/src/routes/api/agent/heartbeat/+server.ts @@ -0,0 +1,16 @@ +import { json } from '@sveltejs/kit'; +import { heartbeat } from '$lib/server/hosts/index.js'; +import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; +export async function POST({ request }: { request: Request }) { + const authenticated = authenticateAgent(request); + try { + const body = await objectBody(request); + const host = heartbeat(authenticated.id, body.inventory); + const inventory = body.inventory as Record; + if (Array.isArray(inventory.libraryCaches)) { + const { updateHostLibraryCaches } = await import('$lib/server/game-files/index.js'); + updateHostLibraryCaches(host.id, inventory.libraryCaches, { snapshot: true }); + } + return json({ host, heartbeatSeconds: 15, serverTime: new Date().toISOString() }); + } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/agent/imports/[id]/+server.ts b/src/routes/api/agent/imports/[id]/+server.ts new file mode 100644 index 0000000..be4db8c --- /dev/null +++ b/src/routes/api/agent/imports/[id]/+server.ts @@ -0,0 +1,15 @@ +import { json } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { requireImportOwner, setImportManifest } from '$lib/server/game-files/index.js'; +import { gameFileError, readGameJson } from '$lib/server/game-files/http.js'; +export function GET({ request, params }: { request: Request; params: { id: string } }) { + try { return json(requireImportOwner(authenticateAgent(request).id, params.id)); } + catch (cause) { return gameFileError(cause); } +} +export async function PUT({ request, params }: { request: Request; params: { id: string } }) { + try { + const host = authenticateAgent(request); + const body = await readGameJson(request, 64 * 1024 * 1024); + return json(setImportManifest(host.id, params.id, body.files)); + } catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/agent/imports/[id]/complete/+server.ts b/src/routes/api/agent/imports/[id]/complete/+server.ts new file mode 100644 index 0000000..6fec060 --- /dev/null +++ b/src/routes/api/agent/imports/[id]/complete/+server.ts @@ -0,0 +1,15 @@ +import { json } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { importPublicationStatus, requestImportPublication } from '$lib/server/game-files/index.js'; +import { gameFileError } from '$lib/server/game-files/http.js'; +export function POST({ request, params }: { request: Request; params: { id: string } }) { + try { + const result = requestImportPublication(authenticateAgent(request).id, params.id); + return json(result, { status: result.status === 'verifying' || result.status === 'pending' ? 202 : 200 }); + } + catch (cause) { return gameFileError(cause); } +} +export function GET({ request, params }: { request: Request; params: { id: string } }) { + try { return json(importPublicationStatus(authenticateAgent(request).id, params.id)); } + catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/agent/imports/[id]/files/[...path]/+server.ts b/src/routes/api/agent/imports/[id]/files/[...path]/+server.ts new file mode 100644 index 0000000..4a03658 --- /dev/null +++ b/src/routes/api/agent/imports/[id]/files/[...path]/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { uploadImportFile } from '$lib/server/game-files/index.js'; +import { gameFileError } from '$lib/server/game-files/http.js'; +export async function PUT({ request, params }: { request: Request; params: { id: string; path: string } }) { + try { + const host = authenticateAgent(request); + await uploadImportFile(host.id, params.id, params.path, request.body); + return json({ uploaded: true }); + } catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/agent/install.sh/+server.ts b/src/routes/api/agent/install.sh/+server.ts new file mode 100644 index 0000000..53dbef1 --- /dev/null +++ b/src/routes/api/agent/install.sh/+server.ts @@ -0,0 +1,2 @@ +import { agentReleaseAsset } from '$lib/server/hosts/releases.js'; +export async function GET() { return new Response(await agentReleaseAsset('install.sh'), { headers: { 'Content-Type': 'text/x-shellscript', 'Cache-Control': 'no-cache' } }); } diff --git a/src/routes/api/agent/libraries/[id]/files/[...path]/+server.ts b/src/routes/api/agent/libraries/[id]/files/[...path]/+server.ts new file mode 100644 index 0000000..2d8f91c --- /dev/null +++ b/src/routes/api/agent/libraries/[id]/files/[...path]/+server.ts @@ -0,0 +1,7 @@ +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { gameFileError, serveGameFile } from '$lib/server/game-files/http.js'; +export async function GET({ request, params }: { request: Request; params: { id: string; path: string } }) { + try { authenticateAgent(request); return await serveGameFile(request, params.id, params.path); } + catch (cause) { return gameFileError(cause); } +} +export const HEAD = GET; diff --git a/src/routes/api/agent/libraries/[id]/manifest/+server.ts b/src/routes/api/agent/libraries/[id]/manifest/+server.ts new file mode 100644 index 0000000..92792f5 --- /dev/null +++ b/src/routes/api/agent/libraries/[id]/manifest/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { getLibraryVersion } from '$lib/server/game-files/index.js'; +import { gameFileError } from '$lib/server/game-files/http.js'; +export function GET({ request, params }: { request: Request; params: { id: string } }) { + try { + authenticateAgent(request); + const version = getLibraryVersion(params.id); + if (!version || version.state !== 'ready') return json({ error: 'Published library not found.' }, { status: 404 }); + return json({ id: version.id, game: version.game, files: version.files, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 }); + } catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/agent/libraries/[id]/status/+server.ts b/src/routes/api/agent/libraries/[id]/status/+server.ts new file mode 100644 index 0000000..ffeda98 --- /dev/null +++ b/src/routes/api/agent/libraries/[id]/status/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +import { updateHostLibraryCaches } from '$lib/server/game-files/index.js'; +import { gameFileError, readGameJson } from '$lib/server/game-files/http.js'; +export async function POST({ request, params }: { request: Request; params: { id: string } }) { + try { + const host = authenticateAgent(request); const body = await readGameJson(request); + updateHostLibraryCaches(host.id, [{ ...body, libraryVersionId: params.id }]); + return json({ ok: true }); + } catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/agent/operations/+server.ts b/src/routes/api/agent/operations/+server.ts new file mode 100644 index 0000000..1bc3e2e --- /dev/null +++ b/src/routes/api/agent/operations/+server.ts @@ -0,0 +1,7 @@ +import { json } from '@sveltejs/kit'; +import { pendingHostOperations } from '$lib/server/hosts/index.js'; +import { authenticateAgent } from '$lib/server/hosts/http.js'; +export function GET({ request }: { request: Request }) { + const host = authenticateAgent(request); + return json({ operations: pendingHostOperations(host.id), serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } }); +} diff --git a/src/routes/api/agent/operations/[id]/+server.ts b/src/routes/api/agent/operations/[id]/+server.ts new file mode 100644 index 0000000..10fb3e6 --- /dev/null +++ b/src/routes/api/agent/operations/[id]/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import { reportHostOperation } from '$lib/server/hosts/index.js'; +import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; +export async function POST({ request, params }: { request: Request; params: { id: string } }) { + const host = authenticateAgent(request); + try { const body = await objectBody(request, 128 * 1024 * 1024); const operation = reportHostOperation(host.id, params.id, { state: body.state, result: body.result, error: body.error }); return json({ operation: { id: operation.id, state: operation.state } }); } + catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/agent/release/[file]/+server.ts b/src/routes/api/agent/release/[file]/+server.ts new file mode 100644 index 0000000..9f4dc7e --- /dev/null +++ b/src/routes/api/agent/release/[file]/+server.ts @@ -0,0 +1,6 @@ +import { error } from '@sveltejs/kit'; +import { agentReleaseAsset } from '$lib/server/hosts/releases.js'; +export async function GET({ params }: { params: { file: string } }) { + if (!['ec-agent.py','manifest.json'].includes(params.file)) error(404, 'Release asset not found.'); + return new Response(await agentReleaseAsset(params.file as 'ec-agent.py' | 'manifest.json'), { headers: { 'Content-Type': params.file.endsWith('.json') ? 'application/json' : 'text/plain', 'Cache-Control': 'no-cache' } }); +} diff --git a/src/routes/api/auth/login/+server.ts b/src/routes/api/auth/login/+server.ts new file mode 100644 index 0000000..66e2342 --- /dev/null +++ b/src/routes/api/auth/login/+server.ts @@ -0,0 +1,10 @@ +import type { RequestHandler } from './$types'; +import { authenticatePassword, revokeSession } from '$lib/server/auth/index.js'; +import { authBody, authError, sessionCookie, sessionResponse } from '$lib/server/auth/http.js'; +export const POST: RequestHandler = async ({ request, cookies, url, getClientAddress }) => { + try { + const result = await authenticatePassword(await authBody(request),getClientAddress()); + revokeSession(cookies.get(sessionCookie(url))); + return sessionResponse(result,cookies,url); + } catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/auth/logout/+server.ts b/src/routes/api/auth/logout/+server.ts new file mode 100644 index 0000000..1f21d3a --- /dev/null +++ b/src/routes/api/auth/logout/+server.ts @@ -0,0 +1,9 @@ +import { json, type RequestHandler } from '@sveltejs/kit'; +import { recordAudit, revokeSession } from '$lib/server/auth/index.js'; +import { sessionCookie } from '$lib/server/auth/http.js'; +export const POST: RequestHandler = ({ cookies, url, locals }) => { + revokeSession(cookies.get(sessionCookie(url))); + cookies.delete(sessionCookie(url),{path:'/'}); + recordAudit(locals.user?.id??null,'logout',locals.user?.id??null); + return json({ ok:true }); +}; diff --git a/src/routes/api/auth/me/+server.ts b/src/routes/api/auth/me/+server.ts new file mode 100644 index 0000000..4b7f9ca --- /dev/null +++ b/src/routes/api/auth/me/+server.ts @@ -0,0 +1,2 @@ +import { json, type RequestHandler } from '@sveltejs/kit'; +export const GET: RequestHandler = ({ locals }) => json({ user:locals.user },{headers:{'Cache-Control':'no-store'}}); diff --git a/src/routes/api/auth/password/+server.ts b/src/routes/api/auth/password/+server.ts new file mode 100644 index 0000000..7e710ab --- /dev/null +++ b/src/routes/api/auth/password/+server.ts @@ -0,0 +1,11 @@ +import type { RequestHandler } from './$types'; +import { changePassword, consumeAuthAttempt } from '$lib/server/auth/index.js'; +import { authBody, authError, requireUser, sessionResponse } from '$lib/server/auth/http.js'; +export const POST: RequestHandler = async ({ request, cookies, locals, url, getClientAddress }) => { + try { + const user = requireUser(locals.user); + consumeAuthAttempt(getClientAddress(),user.id,'password'); + const body = await authBody(request); + return sessionResponse(await changePassword(user.id,body.currentPassword,body.newPassword),cookies,url); + } catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/auth/setup/+server.ts b/src/routes/api/auth/setup/+server.ts new file mode 100644 index 0000000..145cae2 --- /dev/null +++ b/src/routes/api/auth/setup/+server.ts @@ -0,0 +1,7 @@ +import type { RequestHandler } from './$types'; +import { bootstrapAdmin } from '$lib/server/auth/index.js'; +import { authBody, authError, sessionResponse } from '$lib/server/auth/http.js'; +export const POST: RequestHandler = async ({ request, cookies, url, getClientAddress }) => { + try { return sessionResponse(await bootstrapAdmin(await authBody(request),getClientAddress()),cookies,url); } + catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/deployment-collections/preview/+server.ts b/src/routes/api/deployment-collections/preview/+server.ts index 7e41516..fdf3cb5 100644 --- a/src/routes/api/deployment-collections/preview/+server.ts +++ b/src/routes/api/deployment-collections/preview/+server.ts @@ -4,6 +4,6 @@ import { previewCollection } from '$lib/server/fleet/collections.js'; export async function POST({ request }: { request: Request }) { try { const body = await request.json(); - return json(previewCollection(body?.blueprintId, body?.mode)); + return json(previewCollection(body?.blueprintId, body?.mode, body)); } catch (err) { return json({ error: (err as Error).message }, { status: 400 }); } } diff --git a/src/routes/api/game-files/+server.ts b/src/routes/api/game-files/+server.ts new file mode 100644 index 0000000..72a634b --- /dev/null +++ b/src/routes/api/game-files/+server.ts @@ -0,0 +1,10 @@ +import { json } from '@sveltejs/kit'; +import { createLibraryImport, libraryOverview } from '$lib/server/game-files/index.js'; +import { gameFileError, readGameJson } from '$lib/server/game-files/http.js'; +import { requireManagementOrigin } from '$lib/server/hosts/http.js'; + +export function GET() { return json(libraryOverview()); } +export async function POST({ request }: { request: Request }) { + try { requireManagementOrigin(request); return json({ version: createLibraryImport(await readGameJson(request)) }, { status: 202 }); } + catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/game-files/[id]/+server.ts b/src/routes/api/game-files/[id]/+server.ts new file mode 100644 index 0000000..7fd6521 --- /dev/null +++ b/src/routes/api/game-files/[id]/+server.ts @@ -0,0 +1,6 @@ +import { json } from '@sveltejs/kit'; +import { getLibraryVersion } from '$lib/server/game-files/index.js'; +export function GET({ params }: { params: { id: string } }) { + const version = getLibraryVersion(params.id); + return version ? json({ version }) : json({ error: 'Library version not found.' }, { status: 404 }); +} diff --git a/src/routes/api/game-files/[id]/download/+server.ts b/src/routes/api/game-files/[id]/download/+server.ts new file mode 100644 index 0000000..3459ceb --- /dev/null +++ b/src/routes/api/game-files/[id]/download/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { queueLibraryDownload } from '$lib/server/game-files/index.js'; +import { gameFileError, readGameJson } from '$lib/server/game-files/http.js'; +import { requireManagementOrigin } from '$lib/server/hosts/http.js'; +export async function POST({ request, params }: { request: Request; params: { id: string } }) { + try { + requireManagementOrigin(request); + const body = await readGameJson(request); + if (typeof body.hostId !== 'string') throw new Error('Choose a target host.'); + return json({ operation: queueLibraryDownload(body.hostId, params.id) }, { status: 202 }); + } catch (cause) { return gameFileError(cause); } +} diff --git a/src/routes/api/hosts/+server.ts b/src/routes/api/hosts/+server.ts new file mode 100644 index 0000000..4d37409 --- /dev/null +++ b/src/routes/api/hosts/+server.ts @@ -0,0 +1,13 @@ +import { json } from '@sveltejs/kit'; +import { createEnrollment, createHost, listHosts } from '$lib/server/hosts/index.js'; +import { hostApiError, installerCommand, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js'; + +export function GET({ request }: { request: Request }) { requireManagementOrigin(request); return json({ hosts: listHosts() }); } +export async function POST({ request }: { request: Request }) { + requireManagementOrigin(request); + try { + const host = createHost(await objectBody(request)); + const enrollment = createEnrollment(host.id); + return json({ host, enrollment: { ...enrollment, command: installerCommand(request.url, enrollment.token) } }, { status: 201, headers: { 'Cache-Control': 'no-store' } }); + } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/hosts/[id]/+server.ts b/src/routes/api/hosts/[id]/+server.ts new file mode 100644 index 0000000..d2bfaac --- /dev/null +++ b/src/routes/api/hosts/[id]/+server.ts @@ -0,0 +1,14 @@ +import { error, json } from '@sveltejs/kit'; +import { getHost, listHostOperations, updateHost } from '$lib/server/hosts/index.js'; +import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js'; +type Event = { request: Request; params: { id: string } }; +export function GET({ request, params }: Event) { + requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.'); + // Payloads and results can contain RCON keys or staged credentials. History + // exposes metadata only; agent APIs remain the only command delivery path. + return json({ host, operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) }); +} +export async function PATCH({ request, params }: Event) { + requireManagementOrigin(request); + try { return json({ host: updateHost(params.id, await objectBody(request)) }); } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/hosts/[id]/credentials/+server.ts b/src/routes/api/hosts/[id]/credentials/+server.ts new file mode 100644 index 0000000..3538d51 --- /dev/null +++ b/src/routes/api/hosts/[id]/credentials/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { queueHostOperation, revokeCredentials } from '$lib/server/hosts/index.js'; +import { hostApiError, requireManagementOrigin } from '$lib/server/hosts/http.js'; +type Event = { request: Request; params: { id: string } }; +export function DELETE({ request, params }: Event) { + requireManagementOrigin(request); + try { revokeCredentials(params.id); return json({ ok: true }); } catch (cause) { return hostApiError(cause); } +} +export function POST({ request, params }: Event) { + requireManagementOrigin(request); + try { const operation = queueHostOperation(params.id, 'credentials.rotate', {}); return json({ operationId: operation.id }, { status: 202 }); } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/hosts/[id]/enrollment/+server.ts b/src/routes/api/hosts/[id]/enrollment/+server.ts new file mode 100644 index 0000000..9e3d6bd --- /dev/null +++ b/src/routes/api/hosts/[id]/enrollment/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import { createEnrollment } from '$lib/server/hosts/index.js'; +import { hostApiError, installerCommand, requireManagementOrigin } from '$lib/server/hosts/http.js'; +export function POST({ request, params }: { request: Request; params: { id: string } }) { + requireManagementOrigin(request); + try { const enrollment = createEnrollment(params.id); return json({ enrollment: { ...enrollment, command: installerCommand(request.url, enrollment.token) } }, { headers: { 'Cache-Control': 'no-store' } }); } + catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/hosts/[id]/operations/+server.ts b/src/routes/api/hosts/[id]/operations/+server.ts new file mode 100644 index 0000000..543e04d --- /dev/null +++ b/src/routes/api/hosts/[id]/operations/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { queueHostOperation } from '$lib/server/hosts/index.js'; +import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js'; +export async function POST({ request, params }: { request: Request; params: { id: string } }) { + requireManagementOrigin(request); + try { + const body = await objectBody(request); + if (body.kind !== 'runtime.pull' || typeof body.image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9./_:@-]{0,511}$/.test(body.image)) throw new Error('Provide a runtime.pull request and a valid container image.'); + const operation = queueHostOperation(params.id, 'runtime.pull', { image: body.image }); + return json({ operationId: operation.id }, { status: 202 }); + } catch (cause) { return hostApiError(cause); } +} diff --git a/src/routes/api/servers/+server.ts b/src/routes/api/servers/+server.ts index 66462d4..cde2c19 100644 --- a/src/routes/api/servers/+server.ts +++ b/src/routes/api/servers/+server.ts @@ -7,17 +7,24 @@ import { queueCreateServer } from '$lib/server/provisioning/index.js'; import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js'; import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js'; import { runningVersions } from '$lib/server/integrations/plutonium.js'; +import { listHosts } from '$lib/server/hosts/index.js'; +import { nativeInstance } from '$lib/server/backends/native.js'; export async function GET() { const rconHost = config.rcon.host; + const hosts = listHosts(); + const hostsById = new Map(hosts.map(host => [host.id, host])); const versions = new Map(runningVersions().map(version => [version.server, version])); const servers = listServers().map((s) => { + const instance = s.native_instance_id ? nativeInstance(s.name) : null; return { ...serverObservation(s), + host: s.host_id ? hostsById.get(s.host_id) ?? null : null, + network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null, plutonium: versions.get(s.name), // Derived endpoint — same logic as endpointFor() but without asking // Komodo. If we don't have a port yet, endpoint is null. - endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, + endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId })) }; }); @@ -26,6 +33,9 @@ export async function GET() { serverNow: Date.now(), reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server), integrations: { + native: hosts.length > 0 || !komodoConfigured(), + hostsReady: hosts.filter(host => host.readiness === 'ready').length, + hostsTotal: hosts.length, rcon: rconConfigured(), komodo: komodoConfigured(), rconTimeoutMs: config.rcon.timeoutMs, @@ -36,6 +46,7 @@ export async function GET() { export async function POST({ request }: { request: Request }) { const body = (await request.json().catch(() => ({}))) as { + hostId?: string; libraryId?: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number; game?: string; gameMode?: string; mapSlug?: string; @@ -56,7 +67,7 @@ export async function POST({ request }: { request: Request }) { plutoKey: body.plutoKey, expectedName: body.expectedName, previewId: body.previewId, - createdBy: 'ui' + createdBy: 'ui', hostId: body.hostId, libraryId: body.libraryId, image: body.image, bindAddress: body.bindAddress, publicAddress: body.publicAddress, publicPort: body.publicPort }); if (!result.ok) { const status = result.kind === 'bad-request' ? 400 : result.kind === 'unconfigured' ? 503 diff --git a/src/routes/api/system-status/+server.ts b/src/routes/api/system-status/+server.ts index 3135485..70e7973 100644 --- a/src/routes/api/system-status/+server.ts +++ b/src/routes/api/system-status/+server.ts @@ -6,12 +6,31 @@ import { serverObservation } from '$lib/server/fleet/status.js'; import { freshStatusTimestamp } from '$lib/server-freshness.js'; import { tcpProbe } from '$lib/server/integrations/netProbe.js'; import type { SetupCheck, SystemStatus } from '$lib/system-status.js'; +import { listHosts } from '$lib/server/hosts/index.js'; +import { listLibraries } from '$lib/server/game-files/index.js'; export async function GET({ url }: { url: URL }) { const servers = listServers(); const readyServers = servers.filter(server => serverObservation(server).health === 'online').length; const rconServers = servers.filter(server => !server.last_error && freshStatusTimestamp(server.last_seen_at, Date.now()) && server.container_state === 'running').length; const savedKeys = listServerKeys().length; + const hosts = listHosts(); + if (hosts.length || !komodoConfigured()) { + const readyHosts = hosts.filter(host => host.readiness === 'ready'); + const connected = hosts.filter(host => host.connected); + const versions = listLibraries().flatMap(library => library.versions).filter(version => version.state === 'ready'); + const cached = versions.filter(version => version.caches.some(cache => cache.status === 'ready' && readyHosts.some(host => host.id === cache.hostId))); + const runtimes = connected.filter(host => host.inventory.runtimeImages?.length); + const checks: SetupCheck[] = [ + { id: 'hosts', title: 'Host agents', state: readyHosts.length ? 'ready' : hosts.length ? 'attention' : 'unconfigured', detail: `${connected.length} connected, ${readyHosts.length} deployment-ready, ${hosts.length} enrolled or awaiting enrollment.`, href: '/hosts', action: hosts.length ? 'Review hosts' : 'Add host' }, + { id: 'runtime', title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length ? `${runtimes.length} connected hosts report cached runtime images.` : 'Prepare a Plutainer image on the host before creating a server.', href: '/hosts', action: 'Prepare runtime' }, + { id: 'game-files', title: 'Verified game-file libraries', state: cached.length ? 'ready' : 'attention', detail: `${versions.length} published versions; ${cached.length} have a verified cache on a ready host.`, href: '/game-files', action: 'Manage game files' }, + { id: 'rcon', title: 'Agent-local game connection', state: rconServers ? 'ready' : 'unverified', detail: rconServers ? `${rconServers} servers have fresh RCON responses.` : 'The host agent checks RCON locally after deployment. Controller access to game ports is not required.', href: servers.length ? '/' : '/hosts', action: servers.length ? 'Review fleet' : 'Review hosts' }, + { id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Validity is checked with Plutonium at startup.`, href: '/server-keys', action: 'Manage keys' }, + { id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Registered servers are awaiting fresh healthy reports. Inspect their host and diagnostics.' : 'Choose a ready host and a game library when creating your first server.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' } + ]; + return json({ checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys } satisfies SystemStatus, { headers: { 'cache-control': 'no-store' } }); + } const missing = [!config.komodo.address && 'KOMODO_ADDRESS', !config.komodo.apiKey && 'KOMODO_API_KEY', !config.komodo.apiSecret && 'KOMODO_API_SECRET'].filter(Boolean); const connection: SetupCheck = { id: 'komodo', title: 'Komodo connection', state: komodoConfigured() ? 'unverified' : 'unconfigured', detail: komodoConfigured() ? 'Credentials configured. Run connection check to verify host reachability.' : `Set ${missing.join(', ')} in the app environment.`, href: '#setup-guide', action: 'Setup instructions' }; if (komodoConfigured() && url.searchParams.get('probe') === '1') { diff --git a/src/routes/api/telemetry/+server.ts b/src/routes/api/telemetry/+server.ts index fdf856a..b14b206 100644 --- a/src/routes/api/telemetry/+server.ts +++ b/src/routes/api/telemetry/+server.ts @@ -6,8 +6,18 @@ import { recordTelemetryRequest } from '$lib/server/fleet/telemetry-requests.js' import { recordGameEvent } from '$lib/server/fleet/beta-tests.js'; import { db } from '$lib/server/db/index.js'; import { emit } from '$lib/server/pipeline/bus.js'; +import { authenticateTelemetry } from '$lib/server/auth/telemetry.js'; export async function POST({ request }: { request: Request }) { + const authentication = authenticateTelemetry(request); + if (!authentication.ok) { + // Unauthenticated bodies are never parsed, attributed to a server, or + // retained in telemetry diagnostics; they cannot affect game evidence. + return json({ ok: false, error: authentication.error }, { + status: authentication.status, + headers: { 'Cache-Control': 'no-store' } + }); + } const receivedAt = new Date().toISOString(); const contentType = request.headers.get('content-type'); let value: unknown; diff --git a/src/routes/api/users/+server.ts b/src/routes/api/users/+server.ts new file mode 100644 index 0000000..d63981f --- /dev/null +++ b/src/routes/api/users/+server.ts @@ -0,0 +1,8 @@ +import { json, type RequestHandler } from '@sveltejs/kit'; +import { createUser, listUsers } from '$lib/server/auth/index.js'; +import { authBody, authError, requireUser } from '$lib/server/auth/http.js'; +export const GET: RequestHandler = () => json({ users:listUsers() }); +export const POST: RequestHandler = async ({ request, locals }) => { + try { return json({ user:await createUser(requireUser(locals.user).id,await authBody(request)) },{status:201}); } + catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/users/[id]/+server.ts b/src/routes/api/users/[id]/+server.ts new file mode 100644 index 0000000..d238f92 --- /dev/null +++ b/src/routes/api/users/[id]/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { updateUser } from '$lib/server/auth/index.js'; +import { authBody, authError, requireUser } from '$lib/server/auth/http.js'; +export const PATCH: RequestHandler = async ({ request, params, locals }) => { + try { return json({ user:await updateUser(requireUser(locals.user).id,params.id,await authBody(request)) }); } + catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/users/[id]/revoke-sessions/+server.ts b/src/routes/api/users/[id]/revoke-sessions/+server.ts new file mode 100644 index 0000000..2ef0e5d --- /dev/null +++ b/src/routes/api/users/[id]/revoke-sessions/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { revokeUserSessions } from '$lib/server/auth/index.js'; +import { authError, requireUser } from '$lib/server/auth/http.js'; +export const POST: RequestHandler = ({ params, locals }) => { + try { revokeUserSessions(requireUser(locals.user).id,params.id); return json({ok:true}); } + catch (cause) { return authError(cause); } +}; diff --git a/src/routes/api/users/audit/+server.ts b/src/routes/api/users/audit/+server.ts new file mode 100644 index 0000000..4f6e667 --- /dev/null +++ b/src/routes/api/users/audit/+server.ts @@ -0,0 +1,3 @@ +import { json } from '@sveltejs/kit'; +import { listAuthAudit } from '$lib/server/auth/index.js'; +export function GET() { return json({ events:listAuthAudit() }); } diff --git a/src/routes/collections/+page.svelte b/src/routes/collections/+page.svelte index 23a6a01..af08b79 100644 --- a/src/routes/collections/+page.svelte +++ b/src/routes/collections/+page.svelte @@ -29,6 +29,7 @@ let tab = $state('collections'); let limit = $state(25); let launchLimit = $state(10); + const canOperate = $derived($page.data.user?.role === 'admin' || $page.data.user?.role === 'operator'); const activeLaunches = $derived(launches.filter(launch => launch.active > 0)); const launchHistory = $derived(launches.filter(launch => launch.active === 0)); const visibleLaunches = $derived(tab === 'active' ? activeLaunches : launchHistory); @@ -92,10 +93,10 @@ Collections | Electric Cherry
- {#snippet actions()}{/snippet} + {#snippet actions()}{#if canOperate}{/if}{/snippet} - - + {#if canOperate} + {/if} { launchLimit = 10; saveContext(); }} ariaLabel="Collection views" idPrefix="collection-tabs" panelId="collection-panel" />
{#if tab === 'collections'} @@ -103,7 +104,7 @@

Your collections {blueprints.length}

{#if blueprintError}{/if} {#if blueprintsLoading}

Loading collections…

- {:else if !blueprints.length}
Plan the whole lineup.

Add maps from either game, choose the total lobbies each needs, and fill any gaps from one collection.

+ {:else if !blueprints.length}
No collections yet.

Collections define the maps and lobby counts for a fleet. Operators can create and launch them.

{#if canOperate}{/if}
{:else}
{#each blueprints.slice(0,limit) as blueprint (blueprint.id)} @@ -115,11 +116,11 @@

{blueprint.name}

{blueprint.description || 'Fill missing lobbies or match your fleet to this lineup exactly.'}

    {#each blueprint.entries as entry}
  • {entry.count}× {entry.mapName} {entry.game.toUpperCase()}
    • {/each}
-
+ {#if canOperate}
-
+
{/if} {/each}
diff --git a/src/routes/game-files/+page.svelte b/src/routes/game-files/+page.svelte new file mode 100644 index 0000000..49c267c --- /dev/null +++ b/src/routes/game-files/+page.svelte @@ -0,0 +1,120 @@ + + +Game Files | Electric Cherry +
+ + {#snippet actions()}{#if canManage}{/if}{/snippet} + + {#if error}{/if} + {#if notice}

{notice}

{/if} + {#if !canManage}

Administrators import game files and distribute verified copies to hosts.

{/if} + {#if canManage && showImport} +
+

Import an existing game volume

+

The selected host scans its {game}_game_files Docker volume, calculates SHA256 checksums, and streams a copy into controller storage. Publication requires every file to pass verification.

+
+
+ + + + + + +
+ + {#if !onlineHosts.length}

Enroll an online source host in Hosts before importing.

{/if} +
+
+ {/if} +
{libraries.length} libraries · Base-game files and runtime images are versioned separately.
+ {#each matching as library (library.id)} +
+

{library.name}

{library.game.toUpperCase()} · {library.description}

{#if canManage}{/if}
+ {#each library.versions as item (item.id)} +
+

{item.version}

{item.state === 'ready' ? 'Published' : item.state === 'importing' ? 'Importing' : 'Failed'}
+

{formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files · Created {new Date(item.createdAt).toLocaleString()}

+ {#if item.dlc.length}

Content: {item.dlc.join(', ')}

{/if} + {#if item.error}

{item.error}

{/if} +
Version details
Immutable version ID
{item.id}
Manifest SHA256
{item.manifestSha256 ?? 'Waiting for source scan'}
Controller storage
{item.storageLocation}
Imported from
{hostName(item.sourceHostId)} / {item.sourceVolume}
View complete file manifest
+ {#if item.state === 'ready'} + {#if canManage}
{/if} +
+ {#each item.caches as cache}{:else}{/each} +
HostCache stateProgressDetail
{hostName(cache.hostId)}{cache.status}{!hosts.find(host => host.id === cache.hostId)?.connected ? ' · offline' : ''}{#if cache.status === 'downloading' || cache.status === 'verifying'}{formatGameBytes(cache.bytesDownloaded)} / {formatGameBytes(cache.bytesTotal)}{:else if cache.status === 'ready' || cache.status === 'outdated'}Verified{cache.status === 'outdated' ? ' · older version' : ''}{:else}—{/if}{cache.error ?? ''}
Enroll a host to cache this version.
+ {:else if item.state === 'importing'}

The agent operation continues independently of this page. An interrupted import is reconciled when the host reconnects.

{/if} +
+ {/each} +
+ {:else}{#if loaded}{:else}

Loading game libraries…

{/if}{/each} +
+ + diff --git a/src/routes/hosts/+page.svelte b/src/routes/hosts/+page.svelte new file mode 100644 index 0000000..9c871e3 --- /dev/null +++ b/src/routes/hosts/+page.svelte @@ -0,0 +1,52 @@ + + +Hosts · Electric Cherry +
+ + {#if error}{/if} + {#if canManage}
+

Add host

Run the installation command on a Debian 12 or 13 amd64 machine. Each host connects to this controller over HTTPS.

+
+ + + + +
+ {#if enrollment}

Install the agent

This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.

{enrollment.command}
Open host setup →
{/if} +
{:else}

Administrators manage host setup, runtime preparation, and credentials.

{/if} + {#if loading}

Loading hosts…

{:else if !hosts.length}

No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}

{:else} +
{#each hosts as host}

{host.name}

{host.readiness}

{host.hostname || 'Awaiting enrollment'} · {host.connectionState}

{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}

Game ports {host.portMin}–{host.portMax}

{#if host.readinessReasons.length}

{host.readinessReasons[0]}

{/if}{/each}
+ {/if} +
+ + diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte new file mode 100644 index 0000000..c4ff756 --- /dev/null +++ b/src/routes/hosts/[id]/+page.svelte @@ -0,0 +1,55 @@ + + +{host?.name ?? 'Host'} · Electric Cherry +
← All hosts + {#if error}{/if}{#if notice}

{notice}

{/if} + {#if loading}

Loading host…

{:else if host} +

{host.connectionState} · {host.readiness}

Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}
+ {#if host.readinessReasons.length}
    {#each host.readinessReasons as reason}
  • {reason}
    • {/each}
{:else}

Connected and ready. Each deployment also checks its game library, runtime, and ports.

{/if} + {#if canManage}
{/if} +

Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.

+
+

Inventory

+
Hostname
{host.hostname || 'Pending'}
Agent
{host.inventory.agentVersion ?? 'Unknown'}
Platform
{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}
CPU
{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)
Memory available / total
{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}
Game storage available / total
{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}
Docker
{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}
Current operation
{host.inventory.currentOperation ?? 'None reported'}
+

Host configuration

{#if canManage}
+
+
{:else}
Friendly name
{host.name}
Game ports
{host.portMin}–{host.portMax}
Local bind address
{host.bindAddress}
Public / NAT address
{host.publicAddress || 'Not configured'}
Minimum free storage
{bytes(host.minimumFreeBytes)}

Host configuration is managed by administrators.

{/if}

Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.

+

Game files and runtime

Manage game-file libraries →

{#if host.inventory.libraries?.length}
    {#each host.inventory.libraries as library}
  • {library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}
    • {/each}
{:else}

No cached game libraries reported.

{/if} + {#if canManage}
{ event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}>
{/if}

Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}

+

Agent access

Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.

{#if canManage}
+ {#if enrollment}

Expires {new Date(enrollment.expiresAt).toLocaleString()}

{enrollment.command}
{/if}{:else}

Administrators manage enrollment and agent credentials.

{/if} +
+

Operation history

{#if !operations.length}

No remote operations yet.

{:else}
{#each operations as operation}{/each}
OperationStateUpdatedResult
{operation.kind}{operation.id}{operation.state}{new Date(operation.updatedAt).toLocaleString()}{operation.error ?? '—'}
{/if}
+ {/if} +
+ + diff --git a/src/routes/login/+page.svelte b/src/routes/login/+page.svelte new file mode 100644 index 0000000..4a059cc --- /dev/null +++ b/src/routes/login/+page.svelte @@ -0,0 +1,46 @@ + + +Sign in · Electric Cherry +
+ + {#if error}{/if} +
+
+ + + +
+
+

Need access or a password reset? Contact an Electric Cherry administrator.

+ {#if $page.data.setupRequired}Set up the first administrator →{/if} +
+ + diff --git a/src/routes/maintenance/[id]/+page.svelte b/src/routes/maintenance/[id]/+page.svelte index 9869ddd..6f24e66 100644 --- a/src/routes/maintenance/[id]/+page.svelte +++ b/src/routes/maintenance/[id]/+page.svelte @@ -117,7 +117,7 @@
First detected
{$timeFormat.dateTime(incident.first_seen_at)}
Last observed
{$timeFormat.dateTime(incident.last_seen_at)}
{#if incident.finished_at}
Finished
{$timeFormat.dateTime(incident.finished_at)}
{/if} - {#if evidence.containerState}
Observed Komodo state
{evidence.containerState}
{/if} + {#if evidence.containerState}
Observed container state
{evidence.containerState}
{/if} {#if evidence.reason && evidence.reason !== incident.summary}

Observed reason: {evidence.reason}

{/if} {#if !incident.server_deleted_at}
Diagnose connectionOpen server logs
{/if} diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index 1eeab5e..afe4931 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -77,6 +77,9 @@ rcon_status?: import('$lib/server-observation.js').GameObservation; telemetry?: import('$lib/telemetry.js').GameTelemetry; name: string; + native_instance_id?: string | null; host_id?: string | null; instance_generation?: number | null; game_library_id?: string | null; + host?: import('$lib/hosts.js').Host | null; + network?: { bindAddress: string; localPort: number; publicAddress: string | null; publicPort: number | null } | null; komodo_type: string; komodo_name: string; komodo_server: string | null; game_port: number | null; rcon_host: string | null; game: string | null; map: string | null; @@ -103,6 +106,7 @@ let name = $derived($page.params.name ?? ''); let server = $derived(($fleetServers.find(s => s.name === name) as Server | undefined) ?? null); + const native = $derived(!!server?.native_instance_id); let integrations = $derived($fleetIntegrations); let history = $state([]); let inventory = $state([]); @@ -311,8 +315,8 @@ {#if $fleetError}{/if} {#if hasDiagnosticIssue}
-
{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}

Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {integrations && !integrations.rcon ? 'RCON is not configured.' : ''}

-
{#if integrations && (!integrations.rcon || !integrations.komodo)}{/if}
+
{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}

Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : !native && integrations && !integrations.rcon ? 'RCON is not configured.' : ''}

+
{#if native && server.host_id}{:else if integrations && (!integrations.rcon || !integrations.komodo)}{/if}
{/if}
@@ -382,7 +386,9 @@

Current player count: {server.status_source === 'telemetry' ? 'game telemetry' : server.status_source === 'rcon' ? 'RCON' : 'unavailable'}. RCON probes below show that connection's separate results.

- {#if integrations} + {#if native} +
Game connectionRCON runs locally through the host agent
Host agent{server.host?.connectionState ?? 'unknown'}
Last heartbeat{server.host?.lastHeartbeat ? ago(server.host.lastHeartbeat, $uiNow) : 'Not received'}
+ {:else if integrations}
Endpoint{endpointLabel(effectiveEndpoint)}
RCON password{integrations.rcon ? 'configured' : 'missing'}
@@ -396,7 +402,7 @@ {effectiveReason} {errorDetail}
-
{rconFixHint(effectiveReason)}
+
{native ? 'Inspect the host agent connection and game logs. RCON is sent locally by the agent; public game-port forwarding does not affect this check.' : rconFixHint(effectiveReason)}
{:else if effectiveErr}
Error @@ -412,19 +418,19 @@ {/if}
- {#if effectiveEndpoint} + {#if effectiveEndpoint || native} {/if} - {#if !effectiveEndpoint && integrations?.komodo} + {#if !native && !effectiveEndpoint && integrations?.komodo} {/if} - + {:else if server.host_id}{/if}
{#if rconProbe?.ok} @@ -438,7 +444,7 @@
{/if} - {#if netDiagnosis} + {#if !native && netDiagnosis} {#if netDiagnosis.ok && netDiagnosis.results}
@@ -484,7 +490,7 @@ {/if} {/if} - {#if komodoProbe} + {#if !native && komodoProbe} {#if komodoProbe.ok}
Resolved @@ -521,8 +527,8 @@

Configuration

-
Komodo{server.komodo_type} · {server.komodo_name}
-
RCON override{server.rcon_host ?? '(default)'}
+ {#if native}
Host{server.host?.name ?? server.host_id}
Instance{server.native_instance_id}
Generation{server.instance_generation}
Game-file version{server.game_library_id}
{#if server.network}
Local UDP binding{server.network.bindAddress}:{server.network.localPort}
Public / NAT endpoint{server.network.publicAddress || server.host?.publicAddress || server.host?.hostname || 'Not configured'}:{server.network.publicPort ?? server.network.localPort}
{/if} + {:else}
Komodo{server.komodo_type} · {server.komodo_name}
RCON override{server.rcon_host ?? '(default)'}
{/if}
Game port{server.game_port ?? '(unresolved)'}
Game{server.game ?? '(unset)'}
Map{server.map ?? '(unset)'}
diff --git a/src/routes/settings/users/+page.svelte b/src/routes/settings/users/+page.svelte new file mode 100644 index 0000000..bb1afcb --- /dev/null +++ b/src/routes/settings/users/+page.svelte @@ -0,0 +1,154 @@ + + +Users · Electric Cherry +
+ + {#snippet actions()}{#if administrator}{/if}{/snippet} + + {#if !administrator}Go to your account + {:else} + {#if error}{/if}{#if notice}

{notice}

{/if} +
{#each roles as item}

{item.label}

{item.description}

{/each}
+ {#if loading}

Loading users…

+ {:else if !users.length}

No users could be loaded.

+ {:else} +
{#each users as user (user.id)}
+

{user.displayName} {#if user.id === $page.data.user?.id}You{/if}

@{user.username}

+
{user.role === 'admin' ? 'Administrator' : user.role}{user.disabled ? 'Disabled' : 'Enabled'}{#if user.mustChangePassword && !user.disabled}Password change required{/if}
+ +
+
{/each}
+ {/if} +
+

Account activity

The latest 200 sign-in and account events.

+ {#if auditError}{/if} + {#if auditLoading && !events.length}

Loading account activity…

+ {:else if !events.length}

No account events recorded.

+ {:else}
+ {#each events as event (event.id)}{@const details = auditDetails(event.details)}{/each} +
TimeEventByAccountDetails
{new Date(event.createdAt).toLocaleString()}{actionLabels[event.action] ?? event.action}{event.actor ? `@${event.actor}` : 'Unauthenticated'}{event.targetId ? users.find(user => user.id === event.targetId)?.username ?? event.targetId : '—'}{#if details}
View details
{details}
{:else}—{/if}
{/if} +
+ {/if} +
+ + +
+ {#if formError}{/if} + +

3–64 characters: letters, numbers, periods, underscores, and hyphens. Start with a letter or number.

+ +

{roleDescription}

+ +

15–128 characters. Save it before adding the user; it will not be displayed again.

+ +
+
+ + + +
+ {#if formError}{/if} + +

{roleDescription}

+

Disabled users cannot sign in. Running game servers continue operating.

+ {#if editing?.id === $page.data.user?.id}

Use My account to change your own password. Changes to your access can sign you out.

+ {:else}

Leave blank to keep the current password. A reset requires the user to choose a new password at their next sign-in.

{/if} +
+
+ + + diff --git a/src/routes/setup/+page.svelte b/src/routes/setup/+page.svelte new file mode 100644 index 0000000..297ac6f --- /dev/null +++ b/src/routes/setup/+page.svelte @@ -0,0 +1,44 @@ + + +Initial setup · Electric Cherry +
+ +

Use the setup token configured by the controller owner. This step is available only until the first administrator is created.

+ {#if error}{/if} +
+ +

Provided through the controller’s EC_SETUP_TOKEN setting.

+ +

3–64 characters. Start with a letter or number; periods, underscores, and hyphens are allowed.

+ + +

Use 15–128 characters. A long, unique passphrase works well.

+ + +
+ +
+ + diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py new file mode 100644 index 0000000..db8ab0c --- /dev/null +++ b/static/agent/ec-agent.py @@ -0,0 +1,1231 @@ +#!/usr/bin/env python3 +"""Electric Cherry host agent. Python 3 standard library; no remote shell API. + +Each operation is recorded before execution. An interrupted operation becomes +unknown and is reported for reconciliation, never automatically executed twice. +""" + +import argparse +import base64 +import contextlib +import hashlib +import fcntl +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import shutil +import signal +import socket +import sqlite3 +import stat as stat_module +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +VERSION = "1.0.0" +LABEL = "io.electric-cherry" +MAX_FILE = 32 * 1024 * 1024 +ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") + + +class AgentError(Exception): + pass + + +class Held(AgentError): + """A reversible prerequisite prevented any disruptive action.""" + + +class Rejected(AgentError): + """A definitive precondition failure, before any destructive action.""" + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def file_digest(path): + h = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def status_players(text): + lines = text.splitlines() + separator = next((index for index, line in enumerate(lines) if re.match(r"^\s*-+\s+-+", line)), None) + if separator is not None and any(re.search(r"\bnum\s+score\s+ping\b", line, re.I) for line in lines[:separator]): + rows = [line.strip() for line in lines[separator + 1:] if line.strip()] + if any(not re.match(r"^\d+\s+", row) for row in rows): + return None + return len(rows) + info = next((index for index, line in enumerate(lines) if line.startswith("\\") and "\\mapname\\" in line), None) + if info is not None: + return sum(bool(line.strip()) for line in lines[info + 1:]) + # A truncated header or an unfamiliar dialect is never proof of emptiness. + return None + + +def identifier(value): + if not isinstance(value, str) or not ID_RE.fullmatch(value): + raise AgentError("Invalid identifier") + return value + + +def instance_identity(instance_id, generation): + try: + if str(uuid.UUID(instance_id)) != instance_id.lower(): + raise ValueError() + except (ValueError, AttributeError, TypeError): + raise AgentError("Instance ID must be a UUID") from None + if isinstance(generation, bool) or not isinstance(generation, int) or generation < 1: + raise AgentError("Invalid instance generation") + return instance_id.lower(), generation + + +def scoped(root, value, allow_root=False): + """Reject traversal AND existing symlinks, including dangling symlinks.""" + if not isinstance(value, str) or "\\" in value or "\x00" in value: + raise AgentError("Invalid managed path") + rel = PurePosixPath(value) + if rel.is_absolute() or any(p in ("..", "") for p in rel.parts): + raise AgentError("Path is outside the managed directory") + if value not in ("", ".") and str(rel) != value: + raise AgentError("Managed paths must use their canonical relative form") + if (not rel.parts or str(rel) == ".") and not allow_root: + raise AgentError("A file path is required") + root = Path(root) + current = root + if current.is_symlink(): + raise AgentError("Managed root must not be a symlink") + for part in rel.parts: + current = current / part + if current.is_symlink(): + raise AgentError("Symlinks are not allowed in managed file operations") + return current + + +def atomic_bytes(path, data, mode=0o600): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + try: + with open(tmp, "xb") as stream: + os.chmod(tmp, mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + finally: + with contextlib.suppress(FileNotFoundError): + tmp.unlink() + + +def atomic_json(path, value): + atomic_bytes(path, json.dumps(value, sort_keys=True).encode()) + + +@contextlib.contextmanager +def managed_parent(root, value, create=False, uid=1000, gid=1000): + """Use dirfds so an instance cannot race a symlink into a host-root write.""" + scoped(root, value) + parts = PurePosixPath(value).parts + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + fd = os.open(root, flags) + try: + for part in parts[:-1]: + if create: + with contextlib.suppress(FileExistsError): + os.mkdir(part, 0o755, dir_fd=fd) + if os.geteuid() == 0: + os.chown(part, uid, gid, dir_fd=fd, follow_symlinks=False) + child = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = child + yield fd, parts[-1] + finally: + os.close(fd) + + +@contextlib.contextmanager +def managed_open(root, value): + with managed_parent(root, value) as (parent, name): + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) + try: + if not stat_module.S_ISREG(os.fstat(fd).st_mode): + raise AgentError("Requested managed path is not a regular file") + with os.fdopen(fd, "rb", closefd=False) as stream: + yield stream + finally: + os.close(fd) + + +def managed_read(root, value): + try: + with managed_open(root, value) as stream: + if os.fstat(stream.fileno()).st_size > MAX_FILE: + raise Rejected("Managed file exceeds the 32 MiB read limit") + content = stream.read(MAX_FILE + 1) + if len(content) > MAX_FILE: + raise Rejected("Managed file exceeds the 32 MiB read limit") + return content + except FileNotFoundError: + return None + + +def managed_write(root, value, content, uid=1000, gid=1000): + if content is None: + try: + with managed_parent(root, value) as (parent, name): + with contextlib.suppress(FileNotFoundError): + os.unlink(name, dir_fd=parent) + os.fsync(parent) + except FileNotFoundError: + pass + return + with managed_parent(root, value, create=True, uid=uid, gid=gid) as (parent, name): + prior = None + try: + prior = os.stat(name, dir_fd=parent, follow_symlinks=False) + if not stat_module.S_ISREG(prior.st_mode): + raise Rejected("Managed target is not a regular file") + except FileNotFoundError: + pass + temp_name = ".ec-" + uuid.uuid4().hex + fd = os.open(temp_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent) + try: + os.fchmod(fd, prior.st_mode & 0o777 if prior else 0o644) + if os.geteuid() == 0: + os.fchown(fd, prior.st_uid if prior else uid, prior.st_gid if prior else gid) + with os.fdopen(fd, "wb", closefd=False) as stream: + stream.write(content) + stream.flush() + os.fsync(fd) + os.replace(temp_name, name, src_dir_fd=parent, dst_dir_fd=parent) + os.fsync(parent) + finally: + os.close(fd) + with contextlib.suppress(FileNotFoundError): + os.unlink(temp_name, dir_fd=parent) + + +class Journal: + def __init__(self, path): + self.db = sqlite3.connect(path) + self.db.row_factory = sqlite3.Row + self.db.execute("PRAGMA journal_mode=WAL") + self.db.execute("PRAGMA synchronous=FULL") + self.db.execute("CREATE TABLE IF NOT EXISTS operations (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)") + if "checkpoint" not in {row[1] for row in self.db.execute("PRAGMA table_info(operations)")}: + self.db.execute("ALTER TABLE operations ADD COLUMN checkpoint TEXT") + self.db.execute("UPDATE operations SET state='unknown', error='Agent restarted during execution; inspect before reconciliation', updated=? WHERE state='running'", (time.time(),)) + self.db.commit() + + def accept(self, request): + op_id = identifier(request.get("id")) + immutable = {key: request.get(key) for key in ("id", "kind", "instanceId", "generation", "payload")} + canonical = json.dumps(immutable, sort_keys=True, separators=(",", ":")) + fingerprint = digest(canonical.encode()) + row = self.db.execute("SELECT * FROM operations WHERE id=?", (op_id,)).fetchone() + if row: + if row["fingerprint"] != fingerprint: + raise AgentError("Operation ID was reused with different content") + return dict(row), False + self.db.execute("INSERT INTO operations(id,fingerprint,request,state,updated) VALUES(?,?,?,'accepted',?)", (op_id, fingerprint, canonical, time.time())) + self.db.commit() + return self.get(op_id), True + + def get(self, op_id): + row = self.db.execute("SELECT * FROM operations WHERE id=?", (op_id,)).fetchone() + if row is None: + raise AgentError("Unknown operation") + return dict(row) + + def update(self, op_id, state, result=None, error=None): + self.db.execute("UPDATE operations SET state=?, result=?, error=?, updated=? WHERE id=?", (state, json.dumps(result) if result is not None else None, error, time.time(), op_id)) + self.db.commit() + return self.get(op_id) + + def close(self): + self.db.close() + + def checkpoint(self, op_id, value): + self.db.execute("UPDATE operations SET checkpoint=?, updated=? WHERE id=?", (json.dumps(value), time.time(), op_id)) + self.db.commit() + + +class Docker: + def call(self, *args, timeout=120, tolerate_missing=False, merge_stderr=False): + try: + result = subprocess.run(["docker", *args], stdout=subprocess.PIPE, stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE, timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired): + raise AgentError("Docker command unavailable or timed out") from None + if result.returncode: + # Never include arbitrary Docker stderr: create errors may echo secrets. + if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")): + return None + raise AgentError("Docker rejected the requested operation") + return result.stdout.decode("utf-8", "replace") + + def inspect(self, name): + raw = self.call("container", "inspect", name, tolerate_missing=True) + return json.loads(raw)[0] if raw is not None else None + + +class Agent: + def __init__(self, config, docker=None): + self.config = config + self.root = Path(config.get("root", "/var/lib/electric-cherry")).resolve() + for directory in ("agent", "instances", "libraries"): + (self.root / directory).mkdir(parents=True, exist_ok=True) + os.chmod(self.root / "agent", 0o700) + self.journal = Journal(self.root / "agent" / "operations.sqlite3") + self.docker = docker or Docker() + self.stopping = threading.Event() + self.current_operation = None + self.library_progress = {} + self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") + self.executing_id = None + + def instance_dir(self, instance_id, generation): + instance_id, generation = instance_identity(instance_id, generation) + return self.root / "instances" / instance_id / str(generation) + + def container_name(self, instance_id, generation): + instance_id, generation = instance_identity(instance_id, generation) + return f"ec-{instance_id}-g{generation}" + + def inspect(self, instance_id, generation): + name = self.container_name(instance_id, generation) + info = self.docker.inspect(name) + if info is None: + return {"exists": False, "running": False, "healthy": False, "state": "missing", "instanceId": instance_id, "generation": generation} + labels = info.get("Config", {}).get("Labels", {}) or {} + if labels.get(LABEL + ".managed") != "true" or labels.get(LABEL + ".instance") != instance_id or labels.get(LABEL + ".generation") != str(generation): + raise AgentError("Container ownership or generation does not match") + state = info.get("State", {}) + status = state.get("Status", "unknown") + healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + if status == "running" and not healthy: + status = state.get("Health", {}).get("Status", "unknown") + return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")} + + def checkpoint(self, value): + if self.executing_id: + self.journal.checkpoint(self.executing_id, value) + + def mark_removed(self, instance_id, generation): + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if not marker.exists() or json.loads(marker.read_text())["generation"] <= generation: + atomic_json(marker, {"generation": generation, "removed": True}) + + def make_writable(self, path, directory=False): + if os.geteuid() == 0: + os.chown(path, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + os.chmod(path, 0o755 if directory else 0o644) + elif directory: + os.chmod(path, 0o777) + + def spec(self, instance_id, generation): + path = self.instance_dir(instance_id, generation) / "spec.json" + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + raise AgentError("Managed instance metadata is unavailable") from None + + def request(self, method, path, payload=None, credential=True): + url = self.config["controller"].rstrip("/") + path + if urllib.parse.urlsplit(url).scheme != "https" and not self.config.get("allowInsecureLocalhost", False): + raise AgentError("Controller URL must use HTTPS") + if urllib.parse.urlsplit(url).scheme != "https" and urllib.parse.urlsplit(url).hostname not in ("127.0.0.1", "localhost", "::1"): + raise AgentError("Insecure development transport is limited to loopback") + headers = {"Content-Type": "application/json", "User-Agent": "electric-cherry-agent/" + VERSION} + if credential: + headers["Authorization"] = "Bearer " + self.config["credential"] + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + raise AgentError("Controller redirects are not allowed") + with urllib.request.build_opener(NoRedirect()).open(req, timeout=35) as response: + return json.load(response) + except (OSError, ValueError): + raise AgentError("Controller request failed") from None + + def inventory(self): + usage = shutil.disk_usage(self.root) + memory = {} + try: + for line in Path("/proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + memory[key] = int(value.strip().split()[0]) * 1024 + except (OSError, ValueError): + pass + docker_version, docker_running = "", False + try: + info = json.loads(self.docker.call("info", "--format", "{{json .}}", timeout=15)) + docker_version, docker_running = info.get("ServerVersion", ""), True + except (AgentError, ValueError): + pass + libraries, caches = [], [] + for marker in (self.root / "libraries").glob("*/*/manifest.json"): + try: + value = json.loads(marker.read_text()) + libraries.append({"id": value["libraryId"], "state": "ready"}) + caches.append({"libraryVersionId": value["libraryId"], "status": "ready", "bytesDownloaded": value["size"], "bytesTotal": value["size"], "manifestSha256": value.get("manifestSha256")}) + except (OSError, ValueError, KeyError): + continue + progress = list(self.library_progress.values()) + libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress) + caches.extend(progress) + images = [] + if docker_running: + with contextlib.suppress(AgentError): + lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines() + image_ids = set() + for line in lines: + parts = line.split("|") + if len(parts) != 4: + continue + repository, tag, image_digest, image_id = parts + image_ids.add(image_id) + if repository != "" and tag != "": + images.append(repository + ":" + tag) + if repository != "" and image_digest != "": + images.append(repository + "@" + image_digest) + aliases = self.root / "agent" / "runtime-images.json" + if aliases.exists(): + with contextlib.suppress(OSError, ValueError): + images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) + os_name = platform.system() + with contextlib.suppress(OSError): + for line in Path("/etc/os-release").read_text().splitlines(): + if line.startswith("PRETTY_NAME="): + os_name = line.split("=", 1)[1].strip('"') + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + + def safe_to_disrupt(self, instance_id, generation): + health = self.inspect(instance_id, generation) + if health["state"] in ("missing", "stopped", "exited", "created", "dead"): + return + if not health["running"]: + raise Held("Container state is unknown; disruption held") + result = self.rcon(instance_id, generation, "status") + if not result.get("ok") or result.get("players") is None: + raise Held("Fresh local player occupancy is unavailable; disruption held") + if result["players"] != 0: + raise Held("Players are present; disruption held") + + def rcon(self, instance_id, generation, command): + if not isinstance(command, str) or not command or len(command) > 2048 or any(ch in command for ch in ("\x00", "\r", "\n")): + raise AgentError("Invalid RCON command") + saved = self.spec(instance_id, generation) + password = saved.get("rconPassword") or saved.get("env", {}).get("PLUTAINER_RCON_PASSWORD") + if not password: + return {"ok": False, "reason": "unconfigured", "players": None, "text": ""} + bind = saved.get("bindAddress", "0.0.0.0") + host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind + family = socket.AF_INET6 if ":" in host else socket.AF_INET + chunks = [] + with socket.socket(family, socket.SOCK_DGRAM) as sock: + sock.settimeout(3) + try: + sock.connect((host, int(saved["port"]))) + sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n") + while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024: + chunks.append(sock.recv(65535)) + sock.settimeout(0.15) + except socket.timeout: + pass + except OSError: + return {"ok": False, "reason": "unreachable", "players": None, "text": ""} + if any(not chunk.startswith(b"\xff\xff\xff\xff") for chunk in chunks): + return {"ok": False, "reason": "invalid-reply", "players": None, "text": ""} + text = "\n".join(chunk[4:].decode("utf-8", "replace").removeprefix("print\n") for chunk in chunks) + if not text or re.search(r"(bad|invalid|wrong).{0,20}(password|rcon)", text, re.I): + return {"ok": False, "reason": "timeout" if not text else "authentication", "players": None, "text": text} + players = status_players(text) if command == "status" else None + return {"ok": True, "text": text, "players": players} + + def validate_spec(self, payload): + game = payload.get("game") + if game not in ("t5", "t6"): + raise AgentError("Only T5 and T6 are supported") + library = self.library_dir(game, payload.get("libraryId")) + self.verify_library(library) + image = payload.get("image", "") + if not isinstance(image, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", image): + raise AgentError("Invalid runtime image") + port = payload.get("port") + allowed = self.config.get("portRange", {"min": 28960, "max": 29060}) + if isinstance(port, bool) or not isinstance(port, int) or not allowed["min"] <= port <= allowed["max"]: + raise AgentError("Port is outside the configured host range") + bind = payload.get("bindAddress", "0.0.0.0") + try: + import ipaddress + ipaddress.ip_address(bind) + except (ValueError, TypeError): + raise AgentError("Invalid local bind address") from None + env = dict(payload.get("env", {})) + if payload.get("rconPassword"): + env["PLUTAINER_RCON_PASSWORD"] = payload["rconPassword"] + if not isinstance(env, dict) or any(not re.fullmatch(r"[A-Z][A-Z0-9_]*", key) or not isinstance(value, str) or "\x00" in value or "\n" in value for key, value in env.items()): + raise AgentError("Invalid runtime environment") + # Runtime download is a separate preparation operation. A deployment + # uses the already prepared image even during a registry outage. + self.docker.call("image", "inspect", image) + files = payload.get("files", []) + if not isinstance(files, list) or len(files) > 10000: + raise Rejected("Invalid initial file list") + total, paths = 0, set() + for item in files: + scoped(self.root / "agent" / "validation", item["path"]) + if item["path"] in paths: + raise Rejected("Duplicate initial file path") + paths.add(item["path"]) + total += len(self.decode_file(item)) + if total > 64 * 1024 * 1024: + raise Rejected("Initial files exceed the 64 MiB operation limit") + return game, library, image, port, bind, env + + def create(self, instance_id, generation, payload): + directory = self.instance_dir(instance_id, generation) + current_marker = directory.parent / "current.json" + if current_marker.exists(): + current = json.loads(current_marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed"): + raise AgentError("Instance generation has been superseded or removed") + existing = self.inspect(instance_id, generation) + if existing["exists"]: + if self.spec(instance_id, generation) != payload: + raise AgentError("Instance already exists with a different specification") + return existing + game, library, image, port, bind, env = self.validate_spec(payload) + app = directory / "app" + app.mkdir(parents=True, exist_ok=True) + self.make_writable(app, directory=True) + self.write_initial_files(app, payload.get("files", [])) + atomic_json(directory / "spec.json", payload) + env_path = directory / "environment" + atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) + name = self.container_name(instance_id, generation) + spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) + args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] + bind_arg = "[" + bind + "]" if ":" in bind else bind + for protocol in ("udp", "tcp"): + args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) + args.append(image) + self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) + self.docker.call(*args, timeout=120) + self.docker.call("container", "start", name) + atomic_json(current_marker, {"generation": generation}) + return self.inspect(instance_id, generation) + + def write_initial_files(self, root, files): + if not isinstance(files, list) or len(files) > 10000: + raise AgentError("Invalid file list") + for item in files: + content = self.decode_file(item) + managed_write(root, item["path"], content, int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + + def decode_file(self, item): + try: + content = base64.b64decode(item["contentBase64"], validate=True) + except (ValueError, KeyError, TypeError): + raise AgentError("Invalid encoded file content") from None + if len(content) > MAX_FILE: + raise AgentError("File exceeds staging size limit") + if item.get("sha256") and item["sha256"] != digest(content): + raise AgentError("Staged file checksum mismatch") + return content + + def lifecycle(self, kind, instance_id, generation, payload): + if kind == "instance.create": + return self.create(instance_id, generation, payload) + if kind == "instance.inspect": + return self.inspect(instance_id, generation) + health = self.inspect(instance_id, generation) + name = self.container_name(instance_id, generation) + before = [] + if kind == "instance.start": + if not health["exists"]: + raise AgentError("Instance does not exist") + if payload.get("stageId"): + self.safe_to_disrupt(instance_id, generation) + self.checkpoint({"phase": "activating", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), "restartRequired": health["running"]}) + before = self.activate(instance_id, generation, payload["stageId"]) + health = self.inspect(instance_id, generation) + self.safe_to_disrupt(instance_id, generation) + restart_required = bool(payload.get("stageId") and health["running"]) + self.checkpoint({"phase": "starting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt"), "restartRequired": restart_required}) + if restart_required: + self.docker.call("container", "restart", "--time", "30", name) + else: + self.docker.call("container", "start", name) + elif kind in ("instance.stop", "instance.restart", "instance.remove", "instance.rebuild"): + if kind == "instance.rebuild": + next_generation = payload.get("nextGeneration", payload.get("newGeneration")) + if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation: + raise AgentError("Rebuild must use a newer immutable generation") + spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration")} + self.validate_spec(spec) + self.safe_to_disrupt(instance_id, generation) + self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")}) + if payload.get("stageId") and kind == "instance.restart": + before = self.activate(instance_id, generation, payload["stageId"]) + if kind in ("instance.remove", "instance.rebuild"): + if health["exists"]: + self.docker.call("container", "stop", "--time", "30", name) + self.docker.call("container", "rm", name) + if kind == "instance.rebuild": + result = self.create(instance_id, next_generation, spec) + shutil.rmtree(self.instance_dir(instance_id, generation), ignore_errors=False) + return result + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + self.mark_removed(instance_id, generation) + elif health["exists"]: + if kind == "instance.restart" and payload.get("stageId"): + self.safe_to_disrupt(instance_id, generation) + self.docker.call("container", "stop" if kind.endswith("stop") else "restart", "--time", "30", name) + else: + raise AgentError("Unsupported instance operation") + return {**self.inspect(instance_id, generation), "before": before} + + def stage(self, instance_id, generation, payload, operation_id): + directory = self.instance_dir(instance_id, generation) + self.spec(instance_id, generation) + stage_id = identifier(payload.get("stageId", operation_id)) + stage = directory / "stages" / stage_id + files = payload.get("files") + if not isinstance(files, list) or not files or len(files) > 10000: + raise AgentError("Invalid staged file list") + planned = [] + paths = set() + total_size = 0 + for item in files: + scoped(directory / "app", item["path"]) + if item["path"] in paths: + raise AgentError("Duplicate staged file path") + paths.add(item["path"]) + deleting = item.get("delete") is True or item.get("contentBase64") is None + content = None if deleting else self.decode_file(item) + total_size += len(content) if content is not None else 0 + if total_size > 64 * 1024 * 1024: + raise Rejected("Staged file content exceeds the 64 MiB operation limit") + planned.append({"id": item.get("id"), "path": item["path"], "delete": deleting, "sha256": None if deleting else digest(content), "checkBefore": item.get("checkBefore", "expectedSha256" in item), "expectedBeforeSha": item.get("expectedBeforeSha", item.get("expectedSha256")), "contentBase64": None if deleting else base64.b64encode(content).decode()}) + manifest = {"stageId": stage_id, "files": planned, "state": "staged"} + marker = stage / "manifest.json" + if marker.exists(): + existing = json.loads(marker.read_text()) + if existing["files"] != planned: + raise AgentError("Stage ID already contains different files") + return {"stageId": stage_id, "files": len(planned), "state": existing["state"]} + atomic_json(marker, manifest) + return {"stageId": stage_id, "files": len(planned), "state": "staged"} + + def activate(self, instance_id, generation, stage_id): + directory = self.instance_dir(instance_id, generation) + stage = directory / "stages" / identifier(stage_id) + marker = stage / "manifest.json" + saved = json.loads(marker.read_text()) + if saved["state"] == "activated": + return saved["before"] + if saved["state"] != "staged": + raise AgentError("Interrupted activation requires explicit rollback") + before = [] + for item in saved["files"]: + content = managed_read(directory / "app", item["path"]) + actual_sha = None if content is None else digest(content) + if item.get("checkBefore") and actual_sha != item.get("expectedBeforeSha"): + raise Rejected("Target changed after campaign review") + before.append({"id": item.get("id"), "path": item["path"], "contentBase64": None if content is None else base64.b64encode(content).decode()}) + # Snapshot preparation may take time. Occupancy is rechecked at the + # local activation boundary, after all file I/O and integrity checks. + self.safe_to_disrupt(instance_id, generation) + saved.update(state="activating", before=before) + atomic_json(marker, saved) + try: + for item in saved["files"]: + managed_write(directory / "app", item["path"], None if item["delete"] else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + saved["state"] = "activated" + atomic_json(marker, saved) + except Exception: + self.restore_files(directory / "app", before) + saved["state"] = "rolled-back" + atomic_json(marker, saved) + raise + return before + + def restore_files(self, app, before): + for item in before: + managed_write(app, item["path"], None if item["contentBase64"] is None else self.decode_file(item), int(self.config.get("runtimeUid", 1000)), int(self.config.get("runtimeGid", 1000))) + + def file_operation(self, kind, instance_id, generation, payload, operation_id): + directory = self.instance_dir(instance_id, generation) + app = directory / "app" + if kind == "files.stage": + return self.stage(instance_id, generation, payload, operation_id) + if kind in ("files.activate", "files.rollback"): + self.safe_to_disrupt(instance_id, generation) + stage_id = identifier(payload["stageId"]) + health = self.inspect(instance_id, generation) + self.checkpoint({"phase": "activating" if kind == "files.activate" else "rolling-back", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")}) + if kind == "files.activate": + before = self.activate(instance_id, generation, stage_id) + result = {"activated": True, "before": before} + else: + marker = directory / "stages" / stage_id / "manifest.json" + saved = json.loads(marker.read_text()) + if saved["state"] not in ("activated", "activating", "rolled-back"): + raise AgentError("Stage has no activation to roll back") + if saved["state"] != "rolled-back": + self.restore_files(app, saved["before"]) + saved["state"] = "rolled-back" + atomic_json(marker, saved) + result = {"rolledBack": True} + if payload.get("restart", True) and self.inspect(instance_id, generation)["exists"]: + self.safe_to_disrupt(instance_id, generation) + self.docker.call("container", "restart", "--time", "30", self.container_name(instance_id, generation)) + return result + if kind == "logs.read": + return self.read_log(instance_id, generation, payload) + path = scoped(app, payload.get("path", ""), allow_root=kind in ("files.list", "files.stat")) + if kind == "files.read": + content = managed_read(app, payload["path"]) + if content is None: + return {"contentBase64": None} + return {"contentBase64": base64.b64encode(content).decode(), "sha256": digest(content), "size": len(content)} + if kind == "files.stat": + if not path.exists(): + return None + stat = path.stat() + return {"size": stat.st_size, "mtime": stat.st_mtime, "isDirectory": path.is_dir(), "isFile": path.is_file()} + if kind == "files.list": + if not path.exists(): + return {"entries": []} + return {"entries": [{"name": child.name, "path": child.relative_to(app).as_posix(), "isDirectory": child.is_dir(), "size": child.stat().st_size} for child in sorted(path.iterdir()) if not child.is_symlink()][:10000]} + raise AgentError("Unsupported file operation") + + def read_log(self, instance_id, generation, payload): + app = self.instance_dir(instance_id, generation) / "app" + value = payload["path"] + count = min(2000, max(1, int(payload.get("lines", 200)))) + # Validate the original relative spelling independently of symlinks. + scoped(self.root / "agent" / "log-validation", value) + try: + resolved = (app / value).resolve(strict=True) + relative = resolved.relative_to(app.resolve(strict=True)).as_posix() + except FileNotFoundError: + if value != "logs/console.log" or not self.inspect(instance_id, generation)["exists"]: + return None + text = self.docker.call("container", "logs", "--tail", str(count), "--timestamps", self.container_name(instance_id, generation), timeout=30, merge_stderr=True) + lines = text.splitlines() + return {"lines": lines[-count:], "size": len(text.encode()), "mtime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "truncated": len(lines) >= count, "source": "container"} + except ValueError: + raise AgentError("Log symlink resolves outside the managed instance") from None + # Reopen the resolved relative target through no-follow directory FDs; + # swapping either the original link or a target component cannot escape. + with managed_open(app, relative) as stream: + stat = os.fstat(stream.fileno()) + start = max(0, stat.st_size - 256 * 1024) + stream.seek(start) + lines = stream.read(256 * 1024).decode("utf-8", "replace").splitlines() + if start: + lines = lines[1:] + return {"lines": lines[-count:], "size": stat.st_size, "mtime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(stat.st_mtime)), "truncated": start > 0} + + def library_dir(self, game, library_id): + if game not in ("t5", "t6"): + raise AgentError("Unsupported library game") + return self.root / "libraries" / game / identifier(library_id) + + def verify_library(self, directory): + marker = directory / "manifest.json" + if not marker.exists(): + raise AgentError("Required game library is not ready") + manifest = json.loads(marker.read_text()) + for item in manifest["files"]: + path = scoped(directory / "content", item["path"]) + if not path.is_file() or path.stat().st_size != item["size"] or file_digest(path) != item["sha256"]: + raise AgentError("Game library integrity verification failed") + return manifest + + def artifact_url(self, value): + controller = self.config["controller"].rstrip("/") + url = urllib.parse.urljoin(controller + "/", value) + parsed = urllib.parse.urlsplit(url) + origin = urllib.parse.urlsplit(controller) + if (parsed.scheme, parsed.netloc) != (origin.scheme, origin.netloc) or parsed.username or parsed.password: + raise AgentError("Library transport must remain on the authenticated controller origin") + if parsed.scheme != "https" and not (self.config.get("allowInsecureLocalhost") and parsed.hostname in ("127.0.0.1", "localhost", "::1")): + raise AgentError("Library transport must use HTTPS") + return url + + def artifact_open(self, method, path, data=None, extra_headers=None): + headers = {"Authorization": "Bearer " + self.config["credential"], **(extra_headers or {})} + req = urllib.request.Request(self.artifact_url(path), data=data, method=method, headers=headers) + # Redirects cannot forward host credentials to another origin. + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + raise AgentError("Artifact redirects are not allowed") + return urllib.request.build_opener(NoRedirect()).open(req, timeout=60) + + def library_download(self, payload): + library_id = identifier(payload.get("libraryVersionId", payload.get("libraryId"))) + with self.artifact_open("GET", payload["manifestUrl"]) as response: + manifest = json.load(response) + if manifest.get("id") != library_id or not isinstance(manifest.get("files"), list) or not manifest["files"]: + raise AgentError("Library manifest identity mismatch") + files = manifest["files"] + canonical = json.dumps([{ "path": item["path"], "size": item["size"], "sha256": item["sha256"] } for item in files], separators=(",", ":"), ensure_ascii=False) + manifest_sha = manifest.get("manifestSha256") + if not isinstance(manifest_sha, str) or not re.fullmatch("[0-9a-f]{64}", manifest_sha): + raise AgentError("Library manifest checksum is missing") + if digest(canonical.encode("utf-8")) != manifest_sha: + raise AgentError("Library manifest checksum mismatch") + if payload.get("manifestSha256") and payload["manifestSha256"] != manifest_sha: + raise AgentError("Library manifest differs from the requested immutable version") + directory = self.library_dir(manifest["game"], library_id) + if directory.exists(): + saved = self.verify_library(directory) + if saved.get("manifestSha256") != manifest_sha: + raise AgentError("Published library changed unexpectedly") + return {"libraryId": library_id, "state": "ready", "size": saved["size"]} + staging = directory.with_name("." + library_id + ".partial") + staging.mkdir(parents=True, exist_ok=True) + source_marker = staging / "source.json" + if source_marker.exists() and json.loads(source_marker.read_text()).get("manifestSha256") != manifest_sha: + raise Rejected("Partial library download belongs to a different manifest") + atomic_json(source_marker, {"libraryId": library_id, "manifestSha256": manifest_sha}) + size = sum(item["size"] for item in files) + downloaded = 0 + paths = set() + progress = {"libraryVersionId": library_id, "status": "downloading", "bytesDownloaded": 0, "bytesTotal": size, "manifestSha256": manifest_sha} + self.library_progress[library_id] = progress + try: + if shutil.disk_usage(staging).free < max(0, size - sum(path.stat().st_size for path in staging.rglob("*") if path.is_file())): + raise AgentError("Insufficient library storage") + for item in files: + if item["path"] in paths or not isinstance(item["size"], int) or item["size"] < 0 or not re.fullmatch("[0-9a-f]{64}", item["sha256"]): + raise AgentError("Invalid library file manifest") + paths.add(item["path"]) + target = scoped(staging / "content", item["path"]) + target.parent.mkdir(parents=True, exist_ok=True) + offset = target.stat().st_size if target.exists() else 0 + if offset > item["size"]: + target.unlink() + offset = 0 + if offset == item["size"] and target.exists() and file_digest(target) != item["sha256"]: + target.unlink() + offset = 0 + if offset < item["size"] or not target.exists(): + url = payload["filesBaseUrl"].rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/") + with self.artifact_open("GET", url, extra_headers={"Range": f"bytes={offset}-"} if offset else {}) as response: + if offset and response.status == 206: + content_range = response.headers.get("Content-Range", "") + if not content_range.startswith(f"bytes {offset}-"): + raise AgentError("Invalid resumed download response") + elif offset: + offset = 0 + with target.open("ab" if offset else "wb") as output: + received = offset + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + received += len(chunk) + if received > item["size"]: + raise AgentError("Library download exceeded its manifest size") + output.write(chunk) + progress["bytesDownloaded"] = downloaded + received + output.flush() + os.fsync(output.fileno()) + progress["status"] = "verifying" + if target.stat().st_size != item["size"] or file_digest(target) != item["sha256"]: + raise AgentError("Downloaded library file failed integrity verification") + downloaded += item["size"] + progress.update(status="downloading", bytesDownloaded=downloaded) + saved = {"libraryId": library_id, "game": manifest["game"], "size": size, "files": files, "manifestSha256": manifest_sha} + atomic_json(staging / "manifest.json", saved) + for path in (staging / "content").rglob("*"): + os.chmod(path, 0o555 if path.is_dir() else 0o444) + os.chmod(staging / "content", 0o555) + os.replace(staging, directory) + self.library_progress.pop(library_id, None) + return {"libraryId": library_id, "state": "ready", "size": size, "manifestSha256": manifest_sha} + except Exception: + progress.update(status="failed", error="Library download or integrity verification failed") + raise + + def library_import(self, payload): + game = payload.get("game") + volume = payload.get("sourceVolume") + if game not in ("t5", "t6") or volume != game + "_game_files": + raise AgentError("Only the explicitly approved game-file Docker volumes can be imported") + raw = self.docker.call("volume", "inspect", volume) + source = Path(json.loads(raw)[0]["Mountpoint"]) + if not source.is_dir() or source.is_symlink(): + raise AgentError("Import volume is unavailable") + files = [] + for path in sorted(source.rglob("*")): + if path.is_symlink(): + raise AgentError("Import source contains a symlink") + if path.is_file(): + files.append({"path": path.relative_to(source).as_posix(), "size": path.stat().st_size, "sha256": file_digest(path)}) + elif not path.is_dir(): + raise AgentError("Import source contains a special file") + if not files: + raise AgentError("Import volume is empty") + with self.artifact_open("PUT", payload["manifestUrl"], json.dumps({"files": files}).encode(), {"Content-Type": "application/json"}) as response: + response.read() + for item in files: + path = scoped(source, item["path"]) + if path.stat().st_size != item["size"] or file_digest(path) != item["sha256"]: + raise AgentError("Import source changed while publishing") + url = payload["uploadBaseUrl"].rstrip("/") + "/" + urllib.parse.quote(item["path"], safe="/") + with path.open("rb") as stream, self.artifact_open("PUT", url, stream, {"Content-Type": "application/octet-stream", "Content-Length": str(item["size"])}) as response: + response.read() + self.checkpoint({"phase": "publishing-import"}) + return self.complete_import(payload, start=True) + + def complete_import(self, payload, start=False): + method = "POST" if start else "GET" + while not self.stopping.is_set(): + with self.artifact_open(method, payload["completeUrl"], b"{}" if method == "POST" else None, {"Content-Type": "application/json"}) as response: + result = json.load(response) + if result.get("status") == "published": + return result + if result.get("status") == "failed": + raise Rejected("Controller rejected the game-file import after integrity verification") + if result.get("status") not in ("pending", "verifying"): + raise AgentError("Controller returned an invalid import publication state") + method = "POST" if result["status"] == "pending" else "GET" + self.stopping.wait(3) + raise AgentError("Agent stopped while the library import was being published") + + def dispatch(self, operation): + kind, payload = operation.get("kind"), operation.get("payload", {}) + if not isinstance(kind, str) or not isinstance(payload, dict): + raise AgentError("Invalid operation") + if kind == "credentials.rotate": + pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode() + self.config["pendingCredential"] = pending + atomic_json(self.config_path, self.config) + self.checkpoint({"phase": "rotating"}) + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + if not self.recover_credential(): + raise + self.config["credential"] = pending + self.config.pop("pendingCredential", None) + atomic_json(self.config_path, self.config) + return {"rotated": True} + if kind == "runtime.pull": + image = payload.get("image") + if not isinstance(image, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", image): + raise AgentError("Invalid runtime image") + self.docker.call("image", "pull", image, timeout=3600) + image_id = self.docker.call("image", "inspect", "--format", "{{.Id}}", image).strip() + aliases_file = self.root / "agent" / "runtime-images.json" + aliases = json.loads(aliases_file.read_text()) if aliases_file.exists() else {} + aliases[image] = image_id + atomic_json(aliases_file, aliases) + return {"image": image, "ready": True} + if kind == "host.inspect": + return self.inventory() + if kind == "library.download": + return self.library_download(payload) + if kind == "library.import": + return self.library_import(payload) + if kind == "library.inspect": + return self.verify_library(self.library_dir(payload["game"], payload["libraryId"])) + instance_id, generation = instance_identity(operation.get("instanceId"), operation.get("generation")) + current_marker = self.instance_dir(instance_id, generation).parent / "current.json" + if current_marker.exists(): + current = json.loads(current_marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"): + raise AgentError("Instance generation has been superseded or removed") + if kind.startswith("instance."): + return self.lifecycle(kind, instance_id, generation, payload) + if kind.startswith("files.") or kind == "logs.read": + return self.file_operation(kind, instance_id, generation, payload, operation["id"]) + if kind == "rcon.execute": + if payload["command"] not in ("status", "getstatus", "version"): + self.checkpoint({"phase": "rcon-command"}) + return self.rcon(instance_id, generation, payload["command"]) + raise AgentError("Unsupported operation kind") + + def execute(self, operation, report_running=False): + row, _ = self.journal.accept(operation) + if row["state"] == "unknown": + return self.reconcile(operation, row) + if row["state"] != "accepted": + return row + running = self.journal.update(operation["id"], "running") + if report_running: + try: + self.report(running) + except AgentError: + return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"}) + self.executing_id = operation["id"] + try: + result = self.dispatch(operation) + except Held as err: + return self.journal.update(operation["id"], "accepted", {"blocked": "occupied" if "Players" in str(err) else "unknown", "message": str(err)}) + except Exception as err: + # The command may already have changed Docker or files. Never infer + # failure means nothing happened, and never retry without inspection. + message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state" + checkpoint = self.journal.get(operation["id"]).get("checkpoint") + return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message) + finally: + self.executing_id = None + return self.journal.update(operation["id"], "completed", result) + + def recover_credential(self): + pending = self.config.get("pendingCredential") + if not pending: + return False + previous = self.config["credential"] + self.config["credential"] = pending + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + self.config["credential"] = previous + try: + self.request("POST", "/api/agent/credentials", {"credential": pending}) + except AgentError: + return False + self.config["credential"] = pending + self.config.pop("pendingCredential", None) + atomic_json(self.config_path, self.config) + return True + + def reconcile(self, operation, row): + """Prove desired state with immutable ownership; never resubmit a command.""" + kind, payload = operation["kind"], operation.get("payload", {}) + instance_id, generation = operation.get("instanceId"), operation.get("generation") + checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {} + try: + if kind == "credentials.rotate": + if not self.config.get("pendingCredential") or self.recover_credential(): + return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True}) + return row + if kind == "library.import": + try: + with self.artifact_open("GET", payload["completeUrl"]) as response: + status = json.load(response) + if status.get("status") == "published": + return self.journal.update(operation["id"], "completed", status) + if checkpoint.get("phase") == "publishing-import" or status.get("status") == "verifying": + return self.journal.update(operation["id"], "completed", self.complete_import(payload)) + # Import uploads use an immutable import ID and per-file + # checksums. Reconcile their staged files after interruption. + self.journal.update(operation["id"], "running") + self.executing_id = operation["id"] + return self.journal.update(operation["id"], "completed", self.library_import(payload)) + except Rejected as err: + return self.journal.update(operation["id"], "failed", error=str(err)) + except Exception: + return self.journal.update(operation["id"], "unknown", error="Import reconciliation is waiting for controller contact") + finally: + self.executing_id = None + if kind == "library.download": + library_id = payload.get("libraryVersionId", payload.get("libraryId")) + for game in ("t5", "t6"): + directory = self.library_dir(game, library_id) + if directory.exists(): + saved = self.verify_library(directory) + return self.journal.update(operation["id"], "completed", {"libraryId": library_id, "state": "ready", "size": saved["size"], "manifestSha256": saved.get("manifestSha256"), "reconciled": True}) + # This is a resumable immutable transfer, not a repeated + # container command: verified completed files remain untouched. + self.journal.update(operation["id"], "running") + try: + return self.journal.update(operation["id"], "completed", self.library_download(payload)) + except Exception: + return self.journal.update(operation["id"], "failed", error="Interrupted library download could not be resumed") + if kind == "files.stage": + return self.journal.update(operation["id"], "completed", self.stage(instance_id, generation, payload, operation["id"])) + if kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "logs.read") or kind == "rcon.execute" and payload.get("command") in ("status", "getstatus", "version"): + return self.journal.update(operation["id"], "completed", self.dispatch(operation)) + if kind in ("files.activate", "files.rollback") and checkpoint: + stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json" + saved = json.loads(stage.read_text()) + expected = "activated" if kind == "files.activate" else "rolled-back" + if saved["state"] == expected: + health = self.inspect(instance_id, generation) + restarted = health["running"] and health.get("containerId") == checkpoint.get("containerId") and bool(checkpoint.get("startedAt")) and health.get("startedAt") != checkpoint["startedAt"] + if not payload.get("restart", True) or restarted: + return self.journal.update(operation["id"], "completed", {"activated": expected == "activated", "rolledBack": expected == "rolled-back", "before": saved["before"], "reconciled": True}) + return row + if not kind.startswith("instance.") or not checkpoint: + return row + observed_generation = payload.get("nextGeneration", payload.get("newGeneration")) if kind == "instance.rebuild" else generation + health = self.inspect(instance_id, observed_generation) + before = [] + if payload.get("stageId"): + stage = self.instance_dir(instance_id, generation) / "stages" / identifier(payload["stageId"]) / "manifest.json" + saved = json.loads(stage.read_text()) + if saved["state"] != "activated": + return row + before = saved["before"] + proven = False + if kind in ("instance.create", "instance.rebuild"): + proven = health["running"] and health.get("creationOperation") == operation["id"] and health.get("specSha256") == checkpoint.get("specSha256") + if proven: + directory = self.instance_dir(instance_id, observed_generation) + atomic_json(directory.parent / "current.json", {"generation": observed_generation}) + if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: + old = self.instance_dir(instance_id, generation) + if old.exists(): + shutil.rmtree(old) + elif kind in ("instance.start", "instance.stop"): + proven = health.get("containerId") == checkpoint.get("containerId") and (health["running"] if kind == "instance.start" else health["state"] in ("stopped", "exited", "created", "dead")) + if kind == "instance.start": + proven = proven and checkpoint.get("phase") == "starting" + if checkpoint.get("restartRequired"): + proven = proven and bool(checkpoint.get("startedAt")) and bool(health.get("startedAt")) and health["startedAt"] != checkpoint["startedAt"] + elif kind == "instance.restart": + proven = health["running"] and health.get("containerId") == checkpoint.get("containerId") and bool(checkpoint.get("startedAt")) and bool(health.get("startedAt")) and health["startedAt"] != checkpoint["startedAt"] + elif kind == "instance.remove": + proven = not health["exists"] + if proven: + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + self.mark_removed(instance_id, generation) + if proven: + return self.journal.update(operation["id"], "completed", {**health, "before": before, "reconciled": True}) + except (AgentError, OSError, ValueError, KeyError): + pass + return row + + def report(self, row): + payload = {"state": row["state"]} + if row.get("result"): + payload["result"] = json.loads(row["result"]) + if row["state"] == "accepted" and isinstance(payload["result"], dict) and payload["result"].get("blocked"): + # The controller sees a durable running/waiting operation; + # locally accepted means its mutation gate can safely retry. + payload["state"] = "running" + if row.get("error"): + payload["error"] = row["error"] + return self.request("POST", "/api/agent/operations/" + urllib.parse.quote(row["id"], safe=""), payload) + + def heartbeat(self): + response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation}) + host = response.get("host", {}) + if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int): + self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]} + + def heartbeat_loop(self): + while not self.stopping.wait(15): + try: + self.heartbeat() + except AgentError: + pass + + def run(self): + if self.config.get("pendingCredential"): + self.recover_credential() + # Obtain the configured host port range before accepting any deployment. + # Enrollment's default values must never race the first heartbeat. + while not self.stopping.is_set(): + try: + self.heartbeat() + break + except AgentError: + self.stopping.wait(3) + heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True) + heartbeats.start() + while not self.stopping.is_set(): + try: + if self.config.get("pendingCredential"): + self.recover_credential() + response = self.request("GET", "/api/agent/operations") + for operation in response.get("operations", []): + if self.stopping.is_set(): + break + self.current_operation = operation["id"] + accepted, _ = self.journal.accept(operation) + self.report(accepted) + self.report(self.execute(operation, report_running=True)) + self.current_operation = None + except (AgentError, OSError, ValueError, KeyError): + # Logs deliberately omit URLs, tokens, payloads and raw errors. + print("Host controller contact unavailable; existing containers continue running", file=sys.stderr) + self.stopping.wait(3) + heartbeats.join(timeout=2) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", default="/etc/electric-cherry/agent.json") + parser.add_argument("--enroll", action="store_true") + parser.add_argument("--controller") + parser.add_argument("--token-file") + parser.add_argument("--root", default="/var/lib/electric-cherry") + parser.add_argument("--force-enroll", action="store_true") + parser.add_argument("--version", action="version", version=VERSION) + args = parser.parse_args() + if args.enroll: + if not args.controller or not args.token_file: + parser.error("Enrollment needs --controller and --token-file") + if Path(args.config).exists() and not args.force_enroll: + existing = json.loads(Path(args.config).read_text()) + if existing.get("credential"): + if existing.get("controller", "").rstrip("/") != args.controller.rstrip("/") or Path(existing.get("root", "")).resolve() != Path(args.root).resolve(): + raise AgentError("Existing enrollment uses a different controller or data directory") + print("Host is already enrolled; retaining existing credentials") + return + agent = Agent({"controller": args.controller.rstrip("/"), "root": args.root}) + token = Path(args.token_file).read_text().strip() + enrolled = agent.request("POST", "/api/agent/enroll", {"token": token, "inventory": agent.inventory()}, credential=False) + if not enrolled.get("hostId") or not enrolled.get("credential"): + raise AgentError("Controller returned invalid enrollment credentials") + atomic_json(args.config, {"controller": args.controller.rstrip("/"), "root": args.root, "hostId": enrolled["hostId"], "credential": enrolled["credential"], "portRange": enrolled.get("portRange", {"min": 28960, "max": 29060})}) + agent.journal.close() + print("Host enrollment completed") + return + config = json.loads(Path(args.config).read_text()) + config["configPath"] = args.config + lock_root = Path(config.get("root", "/var/lib/electric-cherry")) / "agent" + lock_root.mkdir(parents=True, exist_ok=True) + lock = open(lock_root / "process.lock", "a") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise AgentError("Another host agent already owns this data directory") from None + agent = Agent(config) + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: agent.stopping.set()) + try: + agent.run() + finally: + agent.journal.close() + lock.close() + + +if __name__ == "__main__": + try: + main() + except (AgentError, OSError, ValueError, KeyError): + print("Electric Cherry host agent failed; check configuration and controller availability", file=sys.stderr) + sys.exit(1) diff --git a/static/agent/install.sh b/static/agent/install.sh new file mode 100644 index 0000000..d96601d --- /dev/null +++ b/static/agent/install.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Electric Cherry host installer. Downloaded release integrity is checked before +# execution. Existing Docker data and unrelated containers are never changed. +set -euo pipefail +umask 077 + +controller='' +enrollment_token='' +data_dir='/var/lib/electric-cherry' +reenroll='' +while [ "$#" -gt 0 ]; do + case "$1" in + --controller) controller="${2:?Missing controller URL}"; shift 2 ;; + --token) enrollment_token="${2:?Missing enrollment token}"; shift 2 ;; + --data-dir) data_dir="${2:?Missing data directory}"; shift 2 ;; + --reenroll) reenroll='--force-enroll'; shift ;; + *) echo "Unknown installer option" >&2; exit 2 ;; + esac +done +[ "$(id -u)" = 0 ] || { echo 'Run the installer as root.' >&2; exit 1; } +case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac +[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; } +case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac +case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac +[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; } +. /etc/os-release +[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; } +case "${VERSION_ID:-}" in 12|13) ;; *) echo 'Supported platform: Debian 12 or 13.' >&2; exit 1 ;; esac +[ "$(dpkg --print-architecture)" = amd64 ] || { echo 'Supported architecture: amd64.' >&2; exit 1; } +command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; } + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends ca-certificates curl python3 +python3 - "$controller" "$data_dir" <<'PY' +import json, pathlib, sys +config = pathlib.Path('/etc/electric-cherry/agent.json') +if config.exists(): + saved = json.loads(config.read_text()) + if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve(): + raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.') +PY +if ! command -v docker >/dev/null; then + if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then + echo 'Existing Docker data/configuration detected without a Docker CLI. Repair Docker, then rerun.' >&2 + exit 1 + fi + apt-get install -y --no-install-recommends docker.io + systemctl enable --now docker +fi +docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; } +install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances" +install -d -m 0700 "$data_dir/agent" /etc/electric-cherry +available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}') +[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; } +temporary_dir=$(mktemp -d) +trap 'rm -rf "$temporary_dir"' EXIT +controller="${controller%/}" +curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json" +curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py" +python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY' +import hashlib, json, pathlib, sys +manifest = json.loads(pathlib.Path(sys.argv[1]).read_text()) +actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest() +if actual != manifest.get('sha256'): + raise SystemExit('Agent release checksum verification failed.') +if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1: + raise SystemExit('Unsupported agent release manifest.') +PY +# Stop only the agent while replacing its own executable/journal owner. Existing +# game containers keep running under Docker throughout installation and updates. +if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then + systemctl stop electric-cherry-agent.service +fi +install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || { + install -d -m 0755 /usr/local/libexec + install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py +} +printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token" +unset enrollment_token +python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"} +cat > /etc/systemd/system/electric-cherry-agent.service <&2; exit 1; } +echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.' diff --git a/static/agent/manifest.json b/static/agent/manifest.json new file mode 100644 index 0000000..3623cb0 --- /dev/null +++ b/static/agent/manifest.json @@ -0,0 +1,6 @@ +{ + "version": "1.0.0", + "protocolVersion": 1, + "platform": "debian-amd64", + "sha256": "2b6e6c2cc4c949e8f980c147aa6b8725844b7c320d7627e0328570d23c985a43" +} diff --git a/svelte.config.js b/svelte.config.js index 40dc908..ad3b27e 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -10,15 +10,10 @@ const config = { $ui: 'src/lib/ui', '$ui/*': 'src/lib/ui/*' }, - // Electric Cherry is a private-network fleet manager reached via - // whatever hostname/port the operator's proxy exposes it on. Origin - // headers get rewritten en route (reverse proxies, LAN IPs, .local - // hostnames), so SvelteKit's default cross-site POST guard blocks - // legitimate multipart uploads from our own UI. There is no - // cross-origin surface to defend against here — the app only accepts - // traffic from inside the LAN — so turn the guard off. If this ever - // gets exposed to the public internet, add per-route origin checks in - // hooks.server.ts instead of re-enabling this blanket rule. + // The authentication hook requires exact same-origin browser mutations, + // including multipart uploads. Machine routes use their own credentials + // and cannot be subjected to the browser-origin rule. Set ORIGIN to the + // externally visible HTTPS URL when running behind a reverse proxy. csrf: { checkOrigin: false } diff --git a/tests/helpers/telemetry.ts b/tests/helpers/telemetry.ts index 2d16b0a..6d24a64 100644 --- a/tests/helpers/telemetry.ts +++ b/tests/helpers/telemetry.ts @@ -1,5 +1,11 @@ import type { GameEvent, TelemetryPlayerStats } from '../../src/lib/telemetry.js'; +export const TEST_TELEMETRY_TOKEN = 'ec_test_telemetry_0123456789abcdef0123456789abcdef'; +export function telemetryAuthHeaders(): Record { + process.env.EC_TELEMETRY_TOKEN = TEST_TELEMETRY_TOKEN; + return { Authorization: `Bearer ${TEST_TELEMETRY_TOKEN}` }; +} + export const roster = (count = 4): TelemetryPlayerStats[] => Array.from({ length: count }, (_, i) => ({ name: `Player ${i + 1}`, guid: `1234567890123456${i}`, kills: 100 + i, headshots: 50, downs: 1, revives: 2, score: 5000 @@ -12,6 +18,6 @@ export function gameEvent(event: GameEvent['event'], round = 25, players = 4, se export async function postTelemetry(body: unknown, contentType = 'application/json') { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); return POST({ request: new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': contentType }, body: JSON.stringify(body) + method: 'POST', headers: { 'Content-Type': contentType, ...telemetryAuthHeaders() }, body: JSON.stringify(body) }) }); } diff --git a/tests/unit/auth-access.test.ts b/tests/unit/auth-access.test.ts new file mode 100644 index 0000000..efb8625 --- /dev/null +++ b/tests/unit/auth-access.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from 'vitest'; +import type { UserRole } from '../../src/lib/auth.js'; +import { roleAllows, routePermission, type RoutePermission } from '../../src/lib/server/auth/access.js'; + +const roles: UserRole[] = ['viewer', 'operator', 'admin']; +const allows = (role: UserRole, route: string | null, method = 'GET') => roleAllows(role, routePermission(route, method)); + +describe('management route permissions', () => { + it.each([ + '/', '/api/fleet', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts', + '/api/metrics', '/metrics', '/api/matches', '/matches/[id]', '/api/leaderboard', + '/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]', + '/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events', + '/api/betas/[id]', '/api/scripts/[id]', '/api/configs', '/api/plugins', '/api/hosts/[id]', + '/api/game-files/[id]', '/api/maintenance', '/api/events/servers', '/api/events/pipeline' + ])('lets viewers read the safe summary route %s', route => { + expect(routePermission(route, 'GET')).toBe('viewer'); + expect(allows('viewer', route)).toBe(true); + expect(allows('viewer', route, 'HEAD')).toBe(true); + }); + + it.each([ + ['/api/servers', 'POST'], ['/api/servers/[name]', 'DELETE'], ['/api/provisioning', 'POST'], + ['/api/provisioning/[id]/cancel', 'POST'], ['/api/provisioning/[id]/resume', 'POST'], + ['/api/deployment-collections', 'POST'], ['/api/deployment-collections/preview', 'POST'], + ['/api/blueprints', 'POST'], ['/api/blueprints/[id]', 'PUT'], ['/api/blueprints/[id]', 'DELETE'], + ['/api/betas', 'POST'], ['/api/betas/[id]/promote', 'POST'], ['/api/betas/[id]/rollback', 'POST'], + ['/api/campaigns/[id]/rollback', 'POST'], ['/api/deployments/[id]/rollback', 'POST'], + ['/api/scripts', 'POST'], ['/api/scripts/[id]/versions', 'POST'], ['/api/scripts/[id]/versions/[versionId]', 'DELETE'], + ['/api/configs', 'POST'], ['/api/plugins/github/import', 'POST'], + ['/api/maintenance/[id]', 'POST'], ['/api/watchdog/[id]', 'POST'], ['/api/servers/[name]/probe-rcon', 'POST'] + ])('requires an operator for %s %s', (route, method) => { + expect(routePermission(route, method)).toBe('operator'); + expect(allows('viewer', route, method)).toBe(false); + expect(allows('operator', route, method)).toBe(true); + expect(allows('admin', route, method)).toBe(true); + }); + + it.each(['/api/users', '/api/users/[id]', '/api/users/[id]/password', '/settings', '/settings/users', '/server-keys', '/keys', '/api/server-keys'])('reserves all methods on %s for administrators', route => { + for (const method of ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']) { + expect(routePermission(route, method)).toBe('admin'); + expect(allows('viewer', route, method)).toBe(false); + expect(allows('operator', route, method)).toBe(false); + expect(allows('admin', route, method)).toBe(true); + } + }); + + it.each([ + ['/api/hosts', 'POST'], ['/api/hosts/[id]', 'PATCH'], ['/api/hosts/[id]/enrollment', 'POST'], + ['/api/hosts/[id]/credentials', 'DELETE'], ['/api/hosts/[id]/credentials', 'POST'], + ['/api/hosts/[id]/operations', 'POST'], ['/api/settings', 'PUT'], ['/api/game-files', 'POST'], + ['/api/game-files/[id]/download', 'POST'], ['/api/maintenance', 'POST'], ['/api/watchdog', 'POST'], ['/api/plutonium', 'POST'] + ])('protects host, game-file, and global policy mutations at %s', (route, method) => { + expect(routePermission(route, method)).toBe('admin'); + expect(allows('operator', route, method)).toBe(false); + expect(allows('admin', route, method)).toBe(true); + }); + + it.each([ + '/configs/[id]', '/api/configs/[id]', '/api/configs/inspect', + '/api/campaigns/[id]/file/[fileId]', '/api/deployments/[id]/file/[fileId]', + '/api/servers/[name]/logs/[file]', '/api/servers/[name]/telemetry-requests' + ])('prevents viewer access to secret-bearing raw content at %s', route => { + for (const method of ['GET', 'HEAD']) { + expect(routePermission(route, method)).toBe('operator'); + expect(allows('viewer', route, method)).toBe(false); + expect(allows('operator', route, method)).toBe(true); + } + }); +}); + +describe('authentication and machine boundaries', () => { + it('makes only the intended login/bootstrap methods public', () => { + for (const route of ['/login', '/setup']) { + expect(routePermission(route, 'GET')).toBe('public'); + expect(routePermission(route, 'HEAD')).toBe('public'); + expect(routePermission(route, 'POST')).not.toBe('public'); + } + for (const route of ['/api/auth/login', '/api/auth/setup']) { + expect(routePermission(route, 'POST')).toBe('public'); + expect(routePermission(route, 'GET')).not.toBe('public'); + expect(routePermission(route, 'DELETE')).not.toBe('public'); + } + expect(routePermission('/api/auth/unknown', 'POST')).toBe('admin'); + }); + it.each(['/account', '/api/auth/me', '/api/auth/password', '/api/auth/logout'])('permits every authenticated role to manage their own account at %s', route => { + expect(routePermission(route, 'GET')).toBe('authenticated'); + expect(routePermission(route, 'POST')).toBe('authenticated'); + for (const role of roles) expect(allows(role, route, 'POST')).toBe(true); + }); + it.each([ + ['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/agent/operations', 'GET'], + ['/api/agent/operations/[id]', 'POST'], ['/api/agent/libraries/[id]/files/[...path]', 'GET'], + ['/api/agent/imports/[id]/files/[...path]', 'PUT'], ['/api/agent/imports/[id]/complete', 'POST'], + ['/api/agent/install.sh', 'GET'], ['/api/agent/release/[file]', 'GET'], ['/api/telemetry', 'POST'] + ])('preserves endpoint-specific machine authentication for %s', (route, method) => { + expect(routePermission(route, method)).toBe('machine'); + for (const role of roles) expect(allows(role, route, method)).toBe(false); + }); + it('does not grant machine exemptions to similar prefixes or unsupported telemetry methods', () => { + for (const route of ['/api/agents', '/api/agentish/operations', '/api/telemetry-extra', '/api/telemetry/anything']) { + expect(routePermission(route, 'POST')).toBe('operator'); + } + expect(routePermission('/api/telemetry', 'GET')).toBe('viewer'); + expect(routePermission('/api/telemetry', 'PUT')).toBe('operator'); + }); + it('protects unknown routes and does not infer public assets from URL suffixes', () => { + for (const route of [null, '/not-found', '/unrecognized.json', '/anything.css', '/__data.json']) { + expect(routePermission(route, 'GET')).toBe('viewer'); + expect(routePermission(route, 'POST')).toBe('operator'); + expect(allows('viewer', route, 'POST')).toBe(false); + } + }); + it.each(['/api/agent/../server-keys', '/api/%61gent/operations', '/api/agent/%2e%2e/settings', '/api/agent\\operations', '//api/agent/operations', '/api/agent/operations?bypass=1'])('fails closed if a caller supplies noncanonical route identity %s', route => { + expect(routePermission(route, 'GET')).toBe('admin'); + expect(routePermission(route, 'POST')).toBe('admin'); + }); + it('rejects invalid roles even when a permission would normally allow all users', () => { + for (const permission of ['public', 'authenticated', 'viewer', 'operator', 'admin', 'machine'] as RoutePermission[]) { + expect(roleAllows('unexpected' as UserRole, permission)).toBe(false); + } + }); +}); diff --git a/tests/unit/auth-guard.test.ts b/tests/unit/auth-guard.test.ts new file mode 100644 index 0000000..59161f9 --- /dev/null +++ b/tests/unit/auth-guard.test.ts @@ -0,0 +1,268 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Handle, RequestEvent } from '@sveltejs/kit'; +import type { AuthUser, UserRole } from '../../src/lib/auth.js'; + +type Session = { user: AuthUser; tokenHash: string; expiresAt: number }; +const auth = vi.hoisted(() => ({ + session: null as Session | null, current: null as Session | null, setup: false, + readSession: vi.fn(), readSessionHash: vi.fn(), recordAudit: vi.fn(), setupRequired: vi.fn() +})); +vi.mock('../../src/lib/server/auth/index.js', () => ({ + SESSION_COOKIE: 'ec_session', + AuthError: class AuthError extends Error { constructor(message: string, public status = 400) { super(message); } }, + readSession: auth.readSession, readSessionHash: auth.readSessionHash, recordAudit: auth.recordAudit, setupRequired: auth.setupRequired +})); +import { authenticationHandle, protectEventStream } from '../../src/lib/server/auth/guard.js'; + +const token = 'x'.repeat(43); +function signIn(role: UserRole = 'viewer', mustChangePassword = false): Session { + const user: AuthUser = { id: 'user-1', username: 'tester', displayName: 'Test User', role, disabled: false, + mustChangePassword, createdAt: '2026-01-01T00:00:00Z', updatedAt: '2026-01-01T00:00:00Z', lastLoginAt: null }; + auth.session = { user, tokenHash: 'hash-of-session', expiresAt: Date.now() + 3600_000 }; + auth.current = auth.session; + return auth.session; +} +type Resolve = Parameters[0]['resolve']; +function requestFor(routeId: string | null, options: { + method?: string; path?: string; base?: string; origin?: string | null; headers?: HeadersInit; + body?: unknown; rawBody?: BodyInit; signal?: AbortSignal; resolve?: Resolve; +} = {}) { + const url = new URL(options.path ?? routeId ?? '/not-found', options.base ?? 'https://ec.test'); + const method = options.method ?? 'GET'; const headers = new Headers(options.headers); + if (!['GET', 'HEAD'].includes(method) && options.origin !== null) headers.set('Origin', options.origin ?? url.origin); + if (options.body !== undefined && !headers.has('Content-Type')) headers.set('Content-Type', 'application/json'); + const request = new Request(url, { method, headers, body: options.rawBody ?? (options.body === undefined ? undefined : JSON.stringify(options.body)), signal: options.signal }); + const cookies = { get: vi.fn(() => token) }; + const event = { request, url, route: { id: routeId }, locals: { user: null, sessionHash: null }, cookies } as unknown as RequestEvent; + const resolve = vi.fn(options.resolve ?? (async () => new Response('resolved', { headers: { 'Content-Type': 'text/plain' } }))); + return { event, resolve, cookies, invoke: () => authenticationHandle({ event, resolve } as Parameters[0]) }; +} +beforeEach(() => { + vi.clearAllMocks(); auth.session = null; auth.current = null; auth.setup = false; + auth.readSession.mockImplementation(() => auth.session); + auth.readSessionHash.mockImplementation(() => auth.current); + auth.setupRequired.mockImplementation(() => auth.setup); + vi.stubEnv('EC_AUTH_ALLOW_HTTP', 'false'); +}); +afterEach(() => { vi.useRealTimers(); vi.unstubAllEnvs(); }); + +describe('browser authentication guard', () => { + it('redirects anonymous pages to login while APIs return JSON 401 without resolving', async () => { + const page = requestFor('/metrics', { path: '/metrics?days=7' }); + await expect(page.invoke()).rejects.toMatchObject({ status: 303, location: '/login?next=%2Fmetrics%3Fdays%3D7' }); + expect(page.resolve).not.toHaveBeenCalled(); + const api = requestFor('/api/metrics'); const response = await api.invoke(); + expect(response.status).toBe(401); expect(await response.json()).toMatchObject({ setupRequired: false }); + expect(response.headers.get('cache-control')).toBe('no-store'); expect(api.resolve).not.toHaveBeenCalled(); + }); + it('directs first-run browsers to setup and tells APIs setup is required', async () => { + auth.setup = true; + await expect(requestFor('/').invoke()).rejects.toMatchObject({ status: 303, location: '/setup' }); + const response = await requestFor('/api/fleet').invoke(); + expect(response.status).toBe(401); expect(await response.json()).toMatchObject({ setupRequired: true }); + expect((await requestFor('/setup').invoke()).status).toBe(200); + }); + it('handles login/setup redirects without exposing management pages', async () => { + await expect(requestFor('/setup').invoke()).rejects.toMatchObject({ location: '/login' }); + auth.setup = true; + await expect(requestFor('/login').invoke()).rejects.toMatchObject({ location: '/setup' }); + auth.setup = false; signIn('viewer'); + await expect(requestFor('/login').invoke()).rejects.toMatchObject({ location: '/' }); + signIn('operator', true); + await expect(requestFor('/login').invoke()).rejects.toMatchObject({ location: '/account' }); + }); + it.each([ + ['viewer', '/api/fleet', 'GET', 200], ['viewer', '/api/servers', 'POST', 403], + ['viewer', '/api/configs/[id]', 'GET', 403], ['operator', '/api/configs/[id]', 'GET', 200], + ['operator', '/api/provisioning', 'POST', 200], ['operator', '/api/hosts/[id]', 'PATCH', 403], + ['operator', '/api/users', 'GET', 403], ['admin', '/api/hosts/[id]', 'PATCH', 200], ['admin', '/api/users', 'GET', 200] + ] as const)('enforces %s role for %s %s', async (role, routeId, method, expectedStatus) => { + const session = signIn(role); const call = requestFor(routeId, { method, body: method === 'GET' ? undefined : {} }); + const response = await call.invoke(); expect(response.status).toBe(expectedStatus); + if (expectedStatus === 200) { expect(call.resolve).toHaveBeenCalledOnce(); expect(call.event.locals.user).toEqual(session.user); } + else expect(call.resolve).not.toHaveBeenCalled(); + }); + it('requires temporary-password users to change it even if their role is administrator', async () => { + signIn('admin', true); + await expect(requestFor('/settings').invoke()).rejects.toMatchObject({ status: 303, location: '/account' }); + const blocked = await requestFor('/api/users').invoke(); + expect(blocked.status).toBe(403); expect(await blocked.json()).toMatchObject({ mustChangePassword: true }); + for (const [route, method] of [['/account', 'GET'], ['/api/auth/me', 'GET'], ['/api/auth/password', 'POST'], ['/api/auth/logout', 'POST']]) { + expect((await requestFor(route, { method, body: method === 'POST' ? {} : undefined }).invoke()).status).toBe(200); + } + }); + it('adds private response headers and audits successful writes without recording request secrets', async () => { + signIn('admin'); + const call = requestFor('/api/servers', { method: 'POST', body: { plutoKey: 'secret-server-key' } }); + const response = await call.invoke(); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect(response.headers.get('vary')).toContain('Cookie'); + expect(response.headers.get('x-frame-options')).toBe('DENY'); + expect(response.headers.get('x-content-type-options')).toBe('nosniff'); + expect(auth.recordAudit).toHaveBeenCalledWith('user-1', 'request', '/api/servers', { method: 'POST', status: 200, path: '/api/servers' }); + expect(JSON.stringify(auth.recordAudit.mock.calls)).not.toContain('secret-server-key'); + }); + it('does not audit denied or unsuccessful writes as successful actions', async () => { + signIn('viewer'); await requestFor('/api/servers', { method: 'POST', body: {} }).invoke(); + expect(auth.recordAudit).not.toHaveBeenCalled(); + signIn('admin'); await requestFor('/api/servers', { method: 'POST', body: {}, resolve: async () => new Response('bad', { status: 400 }) }).invoke(); + expect(auth.recordAudit).not.toHaveBeenCalled(); + }); +}); + +describe('origin and transport enforcement', () => { + it.each(['/api/auth/login', '/api/auth/setup', '/api/servers'])('requires exact Origin for %s, including public auth requests', async route => { + signIn('admin'); + for (const origin of [null, 'null', 'https://attacker.test', 'https://ec.test.attacker.test', 'http://ec.test', 'https://ec.test:8443']) { + const call = requestFor(route, { method: 'POST', origin, body: {} }); + expect((await call.invoke()).status).toBe(403); expect(call.resolve).not.toHaveBeenCalled(); + } + expect((await requestFor(route, { method: 'POST', body: {} }).invoke()).status).toBe(200); + }); + it('rejects explicitly cross-site requests even if their Origin header matches', async () => { + signIn('admin'); + const call = requestFor('/api/auth/login', { method: 'POST', body: {}, headers: { 'Sec-Fetch-Site': 'cross-site' } }); + expect((await call.invoke()).status).toBe(403); expect(call.resolve).not.toHaveBeenCalled(); + }); + it('applies origin validation to multipart artifact uploads without consuming their body', async () => { + signIn('operator'); + const form = new FormData(); form.set('name', 'script'); form.set('content', new Blob(['script bytes']), 'test.gsc'); + const denied = requestFor('/api/scripts', { method: 'POST', origin: null, rawBody: form }); + expect((await denied.invoke()).status).toBe(403); + const allowed = requestFor('/api/scripts', { method: 'POST', rawBody: form, resolve: async event => { + expect((await event.request.formData()).get('name')).toBe('script'); return new Response('uploaded', { status: 201 }); + } }); + expect((await allowed.invoke()).status).toBe(201); + }); + it('requires HTTPS by default even on loopback', async () => { + signIn('admin'); + for (const base of ['http://localhost', 'http://127.0.0.1', 'http://[::1]', 'http://private.lan']) { + const call = requestFor('/api/fleet', { base }); + expect((await call.invoke()).status).toBe(426); expect(call.resolve).not.toHaveBeenCalled(); + } + }); + it('permits insecure developer sessions only with the explicit flag and a literal loopback hostname', async () => { + signIn('admin'); vi.stubEnv('EC_AUTH_ALLOW_HTTP', 'true'); + for (const base of ['http://localhost:3000', 'http://127.0.0.1:3000', 'http://[::1]:3000']) { + const call = requestFor('/api/fleet', { base }); expect((await call.invoke()).status).toBe(200); + expect(call.cookies.get).toHaveBeenCalledWith('ec_session_dev'); + } + for (const base of ['http://private.lan', 'http://localhost.attacker.test', 'http://192.168.1.2']) { + expect((await requestFor('/api/fleet', { base }).invoke()).status).toBe(426); + } + const secure = requestFor('/api/fleet'); await secure.invoke(); expect(secure.cookies.get).toHaveBeenCalledWith('ec_session'); + }); +}); + +describe('machine and route identity boundaries', () => { + it.each([['/api/agent/operations', 'GET'], ['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/telemetry', 'POST']] as const)('passes %s to its machine authentication without consulting browser sessions', async (routeId, method) => { + auth.readSession.mockImplementation(() => { throw new Error('Browser session must not be consulted'); }); + auth.setupRequired.mockImplementation(() => { throw new Error('Browser setup must not be consulted'); }); + const call = requestFor(routeId, { method, origin: null, base: 'http://host.lan', body: method === 'POST' ? {} : undefined, + resolve: async () => new Response('machine credential rejected', { status: 401 }) }); + const response = await call.invoke(); expect(response.status).toBe(401); expect(call.resolve).toHaveBeenCalledOnce(); + expect(auth.readSession).not.toHaveBeenCalled(); expect(auth.setupRequired).not.toHaveBeenCalled(); + expect(call.event.locals.user).toBeNull(); + }); + it('does not allow host/enrollment bearer credentials to substitute for operator access even with a browser cookie', async () => { + signIn('admin'); + for (const authorization of ['Bearer ec_host_secret', 'Bearer ec_enroll_secret']) { + const call = requestFor('/api/fleet', { headers: { Authorization: authorization } }); + expect((await call.invoke()).status).toBe(403); expect(call.resolve).not.toHaveBeenCalled(); + } + const machine = requestFor('/api/agent/operations', { headers: { Authorization: 'Bearer ec_host_secret' } }); + expect((await machine.invoke()).status).toBe(200); expect(machine.resolve).toHaveBeenCalledOnce(); + }); + it('protects encoded management paths and Svelte page data using the resolved route ID', async () => { + signIn('viewer'); + for (const path of ['/api/%75sers', '/api/%61gent/../users', '/_app/pretend-public.js']) { + const call = requestFor('/api/users', { path }); + expect((await call.invoke()).status).toBe(403); expect(call.resolve).not.toHaveBeenCalled(); + } + const data = requestFor('/settings/users', { path: '/settings/users/__data.json' }); + await expect(data.invoke()).rejects.toMatchObject({ status: 403 }); expect(data.resolve).not.toHaveBeenCalled(); + const rawConfig = requestFor('/api/configs/[id]', { path: '/api/configs/123?x=/_app/public.css' }); + expect((await rawConfig.invoke()).status).toBe(403); + }); + it('exempts only static assets on unresolved routes and does not use arbitrary file extensions as public access', async () => { + for (const path of ['/_app/immutable/start.js', '/favicon.svg', '/agent/ec-agent.py', '/agent/install.sh', '/agent/manifest.json']) { + const call = requestFor(null, { path }); expect((await call.invoke()).status).toBe(200); expect(call.resolve).toHaveBeenCalledOnce(); + } + for (const path of ['/private.json', '/secrets.css', '/settings/users/__data.json']) { + const call = requestFor(null, { path }); await expect(call.invoke()).rejects.toMatchObject({ status: 303 }); expect(call.resolve).not.toHaveBeenCalled(); + } + const write = requestFor(null, { path: '/_app/anything.js', method: 'POST', body: {} }); + await expect(write.invoke()).rejects.toMatchObject({ status: 303 }); expect(write.resolve).not.toHaveBeenCalled(); + }); +}); + +describe('operator launch credential restrictions', () => { + it.each([ + ['/api/servers', { plutoKey: 'new-key' }], + ['/api/deployment-collections', { keys: { t5_zombie_theater_1: 'new-key' } }] + ])('rejects supplied administrator-managed keys at %s', async (route, body) => { + signIn('operator'); const call = requestFor(route as string, { method: 'POST', body }); + expect((await call.invoke()).status).toBe(403); expect(call.resolve).not.toHaveBeenCalled(); + }); + it.each([ + ['/api/servers', { previewId: 'reviewed', plutoKey: '' }], + ['/api/servers', { previewId: 'reviewed' }], + ['/api/deployment-collections', { previewId: 'reviewed', keys: { t5_zombie_theater_1: '' } }], + ['/api/deployment-collections', { previewId: 'reviewed', keys: {} }] + ])('allows saved-key launches at %s and preserves the body for the handler', async (route, body) => { + signIn('operator'); + const call = requestFor(route as string, { method: 'POST', body, resolve: async event => { + expect(await event.request.json()).toEqual(body); return new Response('queued', { status: 202 }); + } }); + expect((await call.invoke()).status).toBe(202); expect(call.resolve).toHaveBeenCalledOnce(); + }); + it('allows administrators to supply keys and rejects oversized operator launch bodies before handler execution', async () => { + signIn('admin'); expect((await requestFor('/api/servers', { method: 'POST', body: { plutoKey: 'new-key' } }).invoke()).status).toBe(200); + signIn('operator'); + const tooLarge = requestFor('/api/servers', { method: 'POST', body: { plutoKey: 'x'.repeat(1024 * 1024 + 1) } }); + expect((await tooLarge.invoke()).status).toBe(413); expect(tooLarge.resolve).not.toHaveBeenCalled(); + }); +}); + +describe('authenticated event streams', () => { + function streaming() { + let source!: ReadableStreamDefaultController; + const cancel = vi.fn(); + const response = new Response(new ReadableStream({ start(controller) { source = controller; }, cancel }), { headers: { 'Content-Type': 'text/event-stream' } }); + const abort = new AbortController(); + return { response, source, cancel, abort }; + } + it.each(['revoked', 'password-reset', 'role-demoted', 'storage-failure'] as const)('closes and cancels an established event stream when %s', async change => { + vi.useFakeTimers(); signIn('operator'); const source = streaming(); + const protectedResponse = protectEventStream(source.response, 'hash-of-session', 'operator', source.abort.signal); + const reader = protectedResponse.body!.getReader(); + source.source.enqueue(new TextEncoder().encode('event: test\n\n')); + expect((await reader.read()).done).toBe(false); + if (change === 'revoked') auth.current = null; + if (change === 'password-reset') auth.current = { ...auth.current!, user: { ...auth.current!.user, mustChangePassword: true } }; + if (change === 'role-demoted') auth.current = { ...auth.current!, user: { ...auth.current!.user, role: 'viewer' } }; + if (change === 'storage-failure') auth.readSessionHash.mockImplementation(() => { throw new Error('Storage unavailable'); }); + const ended = reader.read(); await vi.advanceTimersByTimeAsync(15_000); + expect(await ended).toEqual({ done: true, value: undefined }); expect(source.cancel).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + it('rechecks valid streams without refreshing idle sessions and cancels upstream when the request aborts', async () => { + vi.useFakeTimers(); signIn('viewer'); const source = streaming(); + const response = protectEventStream(source.response, 'hash-of-session', 'viewer', source.abort.signal); + const reader = response.body!.getReader(); const ended = reader.read(); + await vi.advanceTimersByTimeAsync(15_000); + expect(auth.readSessionHash).toHaveBeenCalledWith('hash-of-session'); expect(source.cancel).not.toHaveBeenCalled(); + source.abort.abort(); + expect(await ended).toEqual({ done: true, value: undefined }); expect(source.cancel).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + it('wraps event-stream responses at the guard while preserving content type and denying anonymous subscriptions', async () => { + const anonymous = requestFor('/api/events/servers'); expect((await anonymous.invoke()).status).toBe(401); expect(anonymous.resolve).not.toHaveBeenCalled(); + vi.useFakeTimers(); signIn('viewer'); const source = streaming(); + const call = requestFor('/api/events/servers', { signal: source.abort.signal, resolve: async () => source.response }); + const response = await call.invoke(); expect(response.headers.get('content-type')).toBe('text/event-stream'); + const reader = response.body!.getReader(); const ended = reader.read(); + auth.current = null; await vi.advanceTimersByTimeAsync(15_000); + expect(await ended).toEqual({ done: true, value: undefined }); expect(source.cancel).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/unit/auth-users.test.ts b/tests/unit/auth-users.test.ts new file mode 100644 index 0000000..e140464 --- /dev/null +++ b/tests/unit/auth-users.test.ts @@ -0,0 +1,369 @@ +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +// These tests intentionally use production-strength scrypt while the complete +// suite may also be saturating the CPU. Do not weaken password parameters. +vi.setConfig({ testTimeout: 20_000, hookTimeout: 20_000 }); +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import type { UserRole } from '../../src/lib/auth.js'; + +const password = 'correct horse battery staple'; +const replacement = 'another long secret passphrase'; +const setupToken = 'ec_setup_test_0123456789abcdef0123456789abcdef'; +let passwordHash: string; +let replacementHash: string; +let fixture: Fixture; +let auth: typeof import('../../src/lib/server/auth/index.js'); +let database: typeof import('../../src/lib/server/db/index.js'); + +// Real scrypt hashes are shared only as fixture data. Security transitions use +// the production API; deterministic gates below exercise its async race checks. +beforeAll(async () => { + const passwords = await import('../../src/lib/server/auth/passwords.js'); + passwordHash = await passwords.hashPassword(password); + replacementHash = await passwords.hashPassword(replacement); +}, 15_000); +beforeEach(async () => { + vi.resetModules(); + vi.stubEnv('EC_SETUP_TOKEN', setupToken); + fixture = await newFixture([]); + auth = await import('../../src/lib/server/auth/index.js'); + database = await import('../../src/lib/server/db/index.js'); + auth.setupRequired(); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + vi.unstubAllEnvs(); + await fixture.cleanup(); +}); + +function seedUser(username = 'administrator', role: UserRole = 'admin', options: { disabled?: boolean; mustChangePassword?: boolean } = {}) { + const id = randomUUID(), now = Date.now(); + database.db().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,disabled,must_change_password,created_at,updated_at) + VALUES(?,?,?,?,?,?,?,?,?)`).run(id, username, username, passwordHash, role, Number(options.disabled ?? false), Number(options.mustChangePassword ?? false), now, now); + return auth.getUser(id)!; +} +function storedUser(id: string) { + return database.db().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as { password_hash: string; role: UserRole; disabled: number; must_change_password: number }; +} +function gate() { + let release!: (value: T) => void; + const promise = new Promise(resolve => { release = resolve; }); + return { promise, release }; +} + +describe('initial administrator setup', () => { + it('requires an owner token and never creates an account from missing or wrong credentials', async () => { + const input = { username: 'Admin', password, token: setupToken }; + vi.stubEnv('EC_SETUP_TOKEN', undefined); + await expect(auth.bootstrapAdmin(input, '127.0.0.1')).rejects.toMatchObject({ status: 503 }); + vi.stubEnv('EC_SETUP_TOKEN', 'short'); + await expect(auth.bootstrapAdmin(input, '127.0.0.1')).rejects.toMatchObject({ status: 503 }); + vi.stubEnv('EC_SETUP_TOKEN', setupToken); + await expect(auth.bootstrapAdmin({ ...input, token: 'wrong' }, '127.0.0.1')).rejects.toMatchObject({ status: 401 }); + expect(auth.setupRequired()).toBe(true); + expect(auth.listUsers()).toEqual([]); + }); + + it('allows exactly one competing bootstrap and atomically establishes its authenticated session', async () => { + const results = await Promise.allSettled([ + auth.bootstrapAdmin({ username: 'FirstAdmin', displayName: ' First Admin ', password, token: setupToken, role: 'viewer' }, '192.0.2.1'), + auth.bootstrapAdmin({ username: 'SecondAdmin', password, token: setupToken }, '192.0.2.2') + ]); + const successes = results.filter(result => result.status === 'fulfilled'); + const failures = results.filter(result => result.status === 'rejected'); + expect(successes).toHaveLength(1); + expect(failures).toHaveLength(1); + if (failures[0].status === 'rejected') expect(failures[0].reason).toMatchObject({ status: 409 }); + if (successes[0].status !== 'fulfilled') throw new Error('Bootstrap did not succeed'); + const { user, session } = successes[0].value; + expect(user).toMatchObject({ role: 'admin', disabled: false, mustChangePassword: false }); + expect(user.username).toBe(user.username.toLowerCase()); + expect(auth.readSession(session.token)?.user.id).toBe(user.id); + expect(auth.listUsers()).toHaveLength(1); + expect(auth.setupRequired()).toBe(false); + await expect(auth.bootstrapAdmin({ username: 'ThirdAdmin', password, token: setupToken }, '192.0.2.3')).rejects.toMatchObject({ status: 409 }); + const persisted = JSON.stringify(database.db().prepare('SELECT * FROM auth_users').all()) + JSON.stringify(auth.listAuthAudit()); + expect(persisted).not.toContain(password); + expect(persisted).not.toContain(setupToken); + expect(JSON.stringify(user)).not.toContain('password_hash'); + }); +}); + +describe('password storage and verification', () => { + it('uses independently salted scrypt hashes and rejects wrong passwords or malformed stored hashes', async () => { + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const anotherHash = await passwords.hashPassword(password); + expect(anotherHash).toMatch(/^scrypt\$131072\$8\$1\$[a-f0-9]{64}\$[a-f0-9]{128}$/); + expect(anotherHash).not.toBe(passwordHash); + expect(anotherHash).not.toContain(password); + expect(await passwords.verifyPassword(password, anotherHash)).toBe(true); + expect(await passwords.verifyPassword(replacement, anotherHash)).toBe(false); + expect(await passwords.verifyPassword(password, 'scrypt$1$8$1$weak$bad')).toBe(false); + expect(await passwords.verifyPassword(undefined, anotherHash)).toBe(false); + }); + + it('bounds password length by Unicode characters and rejects invalid inputs before hashing', async () => { + const { validatePassword, hashPassword, verifyPassword } = await import('../../src/lib/server/auth/passwords.js'); + for (const value of [undefined, null, 123, '', 'short password', 'a'.repeat(129)]) { + await expect(hashPassword(value)).rejects.toThrow('15 and 128'); + } + expect(validatePassword('🦊'.repeat(15))).toBe('🦊'.repeat(15)); + expect(validatePassword('a'.repeat(128))).toHaveLength(128); + expect(await verifyPassword('a'.repeat(129), passwordHash)).toBe(false); + }); + + it('bounds concurrent scrypt work and releases capacity when hashes finish', async () => { + const { hashPassword } = await import('../../src/lib/server/auth/passwords.js'); + const first = hashPassword(password), second = hashPassword(replacement); + await expect(hashPassword('third sufficiently long passphrase')).rejects.toThrow('busy'); + await expect(Promise.all([first, second])).resolves.toHaveLength(2); + await expect(hashPassword('subsequent sufficiently long passphrase')).resolves.toMatch(/^scrypt\$/); + }); + + it('returns the same login error for unknown, disabled and incorrect-password accounts', async () => { + seedUser('enabled'); + seedUser('disabled', 'viewer', { disabled: true }); + for (const [username, supplied] of [['missing', password], ['enabled', replacement], ['disabled', password]]) { + await expect(auth.authenticatePassword({ username, password: supplied }, '192.0.2.5')).rejects.toMatchObject({ status: 401, message: 'Invalid username or password.' }); + } + expect(database.db().prepare('SELECT COUNT(*) AS count FROM auth_sessions').get()).toEqual({ count: 0 }); + expect(JSON.stringify(auth.listAuthAudit())).not.toContain(password); + expect(auth.listAuthAudit()).toHaveLength(3); + }); +}); + +describe('administrator user management', () => { + it('creates each supported role with a forced initial password change and rejects case-insensitive duplicate names', async () => { + const admin = seedUser(); + for (const role of ['admin', 'operator', 'viewer'] as const) { + const user = await auth.createUser(admin.id, { username: role + 'User', displayName: ' ' + role + ' user ', role, password }); + expect(user).toMatchObject({ username: (role + 'user'), displayName: role + ' user', role, mustChangePassword: true, disabled: false }); + expect(storedUser(user.id).password_hash).toMatch(/^scrypt\$/); + expect(JSON.stringify(user)).not.toContain('password_hash'); + } + await expect(auth.createUser(admin.id, { username: 'OPERATORUSER', role: 'viewer', password })).rejects.toMatchObject({ status: 409 }); + expect(auth.listUsers()).toHaveLength(4); + expect(JSON.stringify(auth.listAuthAudit())).not.toContain(password); + }); + + it('rejects administrator mutations by operators, viewers, disabled administrators and forced-change administrators', async () => { + const target = seedUser('target', 'viewer'); + const actors = [seedUser('operator', 'operator'), seedUser('viewer', 'viewer'), seedUser('disabledadmin', 'admin', { disabled: true }), seedUser('temporaryadmin', 'admin', { mustChangePassword: true })]; + const session = auth.createSession(target.id); + for (const actor of actors) { + await expect(auth.createUser(actor.id, { username: 'intruder', role: 'admin', password })).rejects.toMatchObject({ status: 403 }); + await expect(auth.updateUser(actor.id, target.id, { role: 'admin' })).rejects.toMatchObject({ status: 403 }); + expect(() => auth.revokeUserSessions(actor.id, target.id)).toThrow('Administrator'); + } + expect(auth.getUser(target.id)?.role).toBe('viewer'); + expect(auth.readSession(session.token)).not.toBeNull(); + expect(auth.listUsers().some(user => user.username === 'intruder')).toBe(false); + }); + + it('preserves at least one enabled administrator and permits demotion only when another remains', async () => { + const first = seedUser(); + const session = auth.createSession(first.id); + await expect(auth.updateUser(first.id, first.id, { role: 'operator' })).rejects.toMatchObject({ status: 409 }); + await expect(auth.updateUser(first.id, first.id, { disabled: true })).rejects.toMatchObject({ status: 409 }); + expect(auth.readSession(session.token)?.user.role).toBe('admin'); + const second = seedUser('secondadmin'); + await auth.updateUser(first.id, first.id, { role: 'viewer' }); + expect(auth.readSession(session.token)).toBeNull(); + await expect(auth.updateUser(second.id, second.id, { disabled: true })).rejects.toMatchObject({ status: 409 }); + expect(auth.getUser(second.id)?.role).toBe('admin'); + }); + + it('revokes every target session on role, enabled-state and password changes while keeping other users signed in', async () => { + const admin = seedUser(), target = seedUser('target', 'viewer'); + const administratorSession = auth.createSession(admin.id); + const first = auth.createSession(target.id), second = auth.createSession(target.id); + await auth.updateUser(admin.id, target.id, { role: 'operator' }); + expect(auth.readSession(first.token)).toBeNull(); + expect(auth.readSession(second.token)).toBeNull(); + const third = auth.createSession(target.id); + await auth.updateUser(admin.id, target.id, { disabled: true }); + expect(auth.readSession(third.token)).toBeNull(); + expect(() => auth.createSession(target.id)).toThrow('unavailable'); + await auth.updateUser(admin.id, target.id, { disabled: false }); + expect(auth.readSession(third.token)).toBeNull(); + const fourth = auth.createSession(target.id); + const reset = await auth.updateUser(admin.id, target.id, { password: replacement }); + expect(reset.mustChangePassword).toBe(true); + expect(auth.readSession(fourth.token)).toBeNull(); + expect(auth.readSession(administratorSession.token)?.user.id).toBe(admin.id); + expect(storedUser(target.id).password_hash).not.toBe(passwordHash); + await expect(auth.updateUser(admin.id, admin.id, { password: replacement })).rejects.toThrow('current password'); + }); + + it('keeps sessions for display-only edits and supports explicit per-user and individual revocation', async () => { + const admin = seedUser(), target = seedUser('target', 'viewer'); + const one = auth.createSession(target.id), two = auth.createSession(target.id), adminSession = auth.createSession(admin.id); + await auth.updateUser(admin.id, target.id, { displayName: 'New display name' }); + expect(auth.readSession(one.token)?.user.displayName).toBe('New display name'); + auth.revokeSession(one.token); + expect(auth.readSession(one.token)).toBeNull(); + expect(auth.readSession(two.token)).not.toBeNull(); + auth.revokeUserSessions(admin.id, target.id); + expect(auth.readSession(two.token)).toBeNull(); + expect(auth.readSession(adminSession.token)).not.toBeNull(); + }); +}); + +describe('session secrecy and expiration', () => { + it('stores only opaque session hashes, never raw cookies or password hashes in public identities', () => { + const user = seedUser(); + const first = auth.createSession(user.id), second = auth.createSession(user.id); + expect(first.token).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(first.token).not.toBe(second.token); + const sessions = JSON.stringify(database.db().prepare('SELECT * FROM auth_sessions').all()); + expect(sessions).not.toContain(first.token); + expect(sessions).not.toContain(second.token); + expect(auth.readSession(first.token)?.user).not.toHaveProperty('passwordHash'); + for (const token of [undefined, '', 'x', '!', 'x'.repeat(44)]) expect(auth.readSession(token)).toBeNull(); + expect(auth.readSession('x'.repeat(43))).toBeNull(); + }); + + it('throttles activity touches and expires sessions at the idle boundary', () => { + let now = 2_000_000_000_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const user = seedUser(), session = auth.createSession(user.id); + now += 59_000; + expect(auth.readSession(session.token)).not.toBeNull(); + expect(database.db().prepare('SELECT last_seen_at FROM auth_sessions').get()).toEqual({ last_seen_at: now - 59_000 }); + now += 2_000; + expect(auth.readSession(session.token)).not.toBeNull(); + expect(database.db().prepare('SELECT last_seen_at FROM auth_sessions').get()).toEqual({ last_seen_at: now }); + const lastSeen = now; + now = lastSeen + auth.SESSION_IDLE_MS - 1; + expect(auth.readSession(session.token, false)).not.toBeNull(); + now++; + expect(auth.readSession(session.token)).toBeNull(); + auth.createSession(user.id); + expect(database.db().prepare('SELECT COUNT(*) AS count FROM auth_sessions').get()).toEqual({ count: 1 }); + }); + + it('enforces absolute expiry even while the user remains recently active', () => { + let now = 2_000_000_000_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const session = auth.createSession(seedUser().id); + now = session.expiresAt - 1; + database.db().prepare('UPDATE auth_sessions SET last_seen_at=?').run(now - 1000); + expect(auth.readSession(session.token)).not.toBeNull(); + now++; + expect(auth.readSession(session.token)).toBeNull(); + }); +}); + +describe('forced password changes and asynchronous authorization races', () => { + it('requires the current password, replaces old sessions atomically and clears the forced-change state', async () => { + const user = seedUser('newuser', 'viewer', { mustChangePassword: true }); + const login = await auth.authenticatePassword({ username: 'NEWUSER', password }, '192.0.2.10'); + expect(login.user.mustChangePassword).toBe(true); + expect(auth.readSession(login.session.token)?.user.mustChangePassword).toBe(true); + await expect(auth.changePassword(user.id, replacement, 'a different long passphrase')).rejects.toThrow('Current password'); + await expect(auth.changePassword(user.id, password, password)).rejects.toThrow('different password'); + expect(auth.readSession(login.session.token)).not.toBeNull(); + const changed = await auth.changePassword(user.id, password, replacement); + expect(changed.user).toMatchObject({ id: user.id, role: 'viewer', mustChangePassword: false }); + expect(auth.readSession(login.session.token)).toBeNull(); + expect(auth.readSession(changed.session.token)?.user.mustChangePassword).toBe(false); + await expect(auth.authenticatePassword({ username: user.username, password }, '192.0.2.11')).rejects.toMatchObject({ status: 401 }); + }); + + it('does not mint a session when the verified password is reset during login', async () => { + const admin = seedUser(), target = seedUser('target', 'viewer'); + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const verifying = gate(); + vi.spyOn(passwords, 'verifyPassword').mockImplementationOnce(() => verifying.promise); + vi.spyOn(passwords, 'hashPassword').mockResolvedValueOnce(replacementHash); + const pending = auth.authenticatePassword({ username: target.username, password }, '192.0.2.20'); + await auth.updateUser(admin.id, target.id, { password: replacement }); + verifying.release(true); + await expect(pending).rejects.toMatchObject({ status: 401 }); + expect(database.db().prepare('SELECT COUNT(*) AS count FROM auth_sessions').get()).toEqual({ count: 0 }); + }); + + it('rechecks a disabled account and current role after asynchronous login verification', async () => { + const admin = seedUser(), target = seedUser('target', 'operator'); + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const verifying = gate(); + vi.spyOn(passwords, 'verifyPassword').mockImplementationOnce(() => verifying.promise); + const pending = auth.authenticatePassword({ username: target.username, password }, '192.0.2.21'); + await auth.updateUser(admin.id, target.id, { disabled: true }); + verifying.release(true); + await expect(pending).rejects.toMatchObject({ status: 401 }); + await auth.updateUser(admin.id, target.id, { disabled: false }); + const second = gate(); + vi.mocked(passwords.verifyPassword).mockImplementationOnce(() => second.promise); + const nextLogin = auth.authenticatePassword({ username: target.username, password }, '192.0.2.22'); + await auth.updateUser(admin.id, target.id, { role: 'viewer' }); + second.release(true); + const loggedIn = await nextLogin; + expect(loggedIn.user.role).toBe('viewer'); + expect(auth.readSession(loggedIn.session.token)?.user.role).toBe('viewer'); + }); + + it('rechecks the administrator before committing a user creation that was hashing during demotion', async () => { + const actor = seedUser(), other = seedUser('otheradmin'); + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const hashing = gate(); + vi.spyOn(passwords, 'hashPassword').mockImplementationOnce(() => hashing.promise); + const pending = auth.createUser(actor.id, { username: 'intruder', password, role: 'admin' }); + await auth.updateUser(other.id, actor.id, { role: 'viewer' }); + hashing.release(passwordHash); + await expect(pending).rejects.toMatchObject({ status: 403 }); + expect(auth.listUsers()).toHaveLength(2); + }); + + it('does not commit a password reset if its administrator lost access while hashing', async () => { + const actor = seedUser(), other = seedUser('otheradmin'), target = seedUser('target', 'viewer'); + const session = auth.createSession(target.id); + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const hashing = gate(); + vi.spyOn(passwords, 'hashPassword').mockImplementationOnce(() => hashing.promise); + const pending = auth.updateUser(actor.id, target.id, { password: replacement }); + await auth.updateUser(other.id, actor.id, { role: 'viewer' }); + hashing.release(replacementHash); + await expect(pending).rejects.toMatchObject({ status: 403 }); + expect(storedUser(target.id).password_hash).toBe(passwordHash); + expect(auth.readSession(session.token)).not.toBeNull(); + }); + + it('rejects a self password change whose original password was concurrently reset', async () => { + const admin = seedUser(), target = seedUser('target', 'viewer'); + const passwords = await import('../../src/lib/server/auth/passwords.js'); + const verifying = gate(); + vi.spyOn(passwords, 'verifyPassword').mockImplementationOnce(() => verifying.promise); + vi.spyOn(passwords, 'hashPassword').mockResolvedValue(replacementHash); + const pending = auth.changePassword(target.id, password, 'self selected new passphrase'); + await auth.updateUser(admin.id, target.id, { password: replacement }); + verifying.release(true); + await expect(pending).rejects.toMatchObject({ status: 409 }); + expect(storedUser(target.id)).toMatchObject({ password_hash: replacementHash, must_change_password: 1 }); + expect(database.db().prepare('SELECT COUNT(*) AS count FROM auth_sessions').get()).toEqual({ count: 0 }); + }); +}); + +describe('persistent authentication throttling', () => { + it('retains case-insensitive per-account limits across controller database reopen and resets after the window', () => { + let now = 2_000_000_000_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + for (let attempt = 0; attempt < 10; attempt++) auth.consumeAuthAttempt('192.0.2.30', attempt % 2 ? 'ADMIN' : 'admin'); + database.closeDb(); + expect(() => auth.consumeAuthAttempt('192.0.2.31', 'Admin')).toThrow('Too many attempts'); + const persisted = JSON.stringify(database.db().prepare('SELECT * FROM auth_rate_limits').all()); + expect(persisted).not.toContain('192.0.2.30'); + expect(persisted).not.toContain('admin'); + now += 15 * 60_000; + expect(() => auth.consumeAuthAttempt('192.0.2.31', 'Admin')).not.toThrow(); + }); + + it('limits attempts from one address across many accounts and separates setup from login windows', () => { + for (let account = 0; account < 30; account++) auth.consumeAuthAttempt('192.0.2.40', 'account-' + account); + expect(() => auth.consumeAuthAttempt('192.0.2.40', 'new-account')).toThrow('Too many attempts'); + expect(() => auth.consumeAuthAttempt('192.0.2.41', 'new-account')).not.toThrow(); + expect(() => auth.consumeAuthAttempt('192.0.2.40', 'bootstrap', 'setup')).not.toThrow(); + }); +}); diff --git a/tests/unit/game-file-library.test.ts b/tests/unit/game-file-library.test.ts new file mode 100644 index 0000000..2fdf49c --- /dev/null +++ b/tests/unit/game-file-library.test.ts @@ -0,0 +1,104 @@ +import Database from 'better-sqlite3'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +const state = vi.hoisted(() => ({ handle: null as Database.Database | null })); +vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle })); +import { createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js'; +import { createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, listLibraries, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js'; +import { serveGameFile } from '../../src/lib/server/game-files/http.js'; +import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js'; + +let directory: string; let hostId: string; let credential: string; +const data = 'real game bytes'; +const file = { path: 'zone/game.ff', size: Buffer.byteLength(data), sha256: createHash('sha256').update(data).digest('hex') }; +beforeEach(async () => { + state.handle = new Database(':memory:'); state.handle.pragma('foreign_keys=ON'); + directory = await mkdtemp(join(tmpdir(), 'ec-libraries-')); vi.stubEnv('GAME_FILES_ROOT', directory); + const host = createHost({ name: 'Debian host' }); hostId = host.id; + const enrollment = createEnrollment(host.id); + credential = enrollHost(enrollment.token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000 }).credential; +}); +afterEach(async () => { state.handle?.close(); await rm(directory, { recursive: true, force: true }); vi.unstubAllEnvs(); }); +function begin(version = '1') { return createLibraryImport({ hostId, game: 't6', name: 'Black Ops II', version, description: 'Complete files', dlc: ['Origins'] }); } +async function published() { + const version = begin(); setImportManifest(hostId, version.id, [file]); + await uploadImportFile(hostId, version.id, file.path, new Response(data).body); + return publishImport(hostId, version.id); +} +describe('game-file import lifecycle', () => { + it('publishes verified filesystem content and permanently pins manifest metadata', async () => { + const initial = begin(); + expect(getHostOperation(initial.operationId!)?.payload.sourceVolume).toBe('t6_game_files'); + setImportManifest(hostId, initial.id, [file]); + await expect(publishImport(hostId, initial.id)).rejects.toThrow(); + expect(getLibraryVersion(initial.id)?.state).toBe('importing'); + await uploadImportFile(hostId, initial.id, file.path, new Response(data).body); + const ready = await publishImport(hostId, initial.id); + expect(ready.state).toBe('ready'); expect(ready.sizeBytes).toBe(data.length); + expect(await readFile(join(ready.storageLocation, file.path), 'utf8')).toBe(data); + expect((await publishImport(hostId, ready.id)).id).toBe(ready.id); + await expect(uploadImportFile(hostId, ready.id, file.path, new Response(data).body)).rejects.toThrow(/not writable/); + expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/); + expect(() => begin()).toThrow(/already exists/); + expect(begin('2').id).not.toBe(ready.id); + }); + it('restricts an import to its source host and rejects manifest mutation', () => { + const version = begin(); + expect(() => setImportManifest('another-host', version.id, [file])).toThrow(/not found/); + const first = setImportManifest(hostId, version.id, [file]); + expect(setImportManifest(hostId, version.id, [file]).manifestSha256).toBe(first.manifestSha256); + expect(() => setImportManifest(hostId, version.id, [{ ...file, path: 'changed.ff' }])).toThrow(/cannot change/); + }); + it('surfaces durable agent failures while preserving historical versions', () => { + const version = begin(); + reportHostOperation(hostId, version.operationId!, { state: 'failed', error: 'Source volume not found' }); + expect(listLibraries()[0].versions[0]).toMatchObject({ state: 'failed', error: 'Source volume not found' }); + }); + it('returns a pollable publication status while large files are verified asynchronously', async () => { + const version = begin(); setImportManifest(hostId, version.id, [file]); + await uploadImportFile(hostId, version.id, file.path, new Response(data).body); + expect(importPublicationStatus(hostId, version.id).status).toBe('pending'); + expect(requestImportPublication(hostId, version.id).status).toBe('verifying'); + expect(requestImportPublication(hostId, version.id).status).toBe('verifying'); + await publishImport(hostId, version.id); + expect(importPublicationStatus(hostId, version.id)).toMatchObject({ status: 'published', version: { id: version.id, manifestSha256: expect.any(String) } }); + }); +}); +describe('verified host distribution', () => { + it('deduplicates active downloads and blocks readiness without exact manifest verification', async () => { + const version = await published(); + const operation = queueLibraryDownload(hostId, version.id); + expect(queueLibraryDownload(hostId, version.id).id).toBe(operation.id); + expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }]); + expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed'); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); + expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'outdated', manifestSha256: version.manifestSha256 }]); + expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', bytesDownloaded: 1, manifestSha256: version.manifestSha256 }]); + expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed'); + updateHostLibraryCaches(hostId, [], { snapshot: true }); + expect(getHostLibraryCache(hostId, version.id)?.status).toBe('missing'); + expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/); + }); + it('streams resumable byte ranges and guards the immutable manifest endpoint with revocable host credentials', async () => { + const version = await published(); + const request = (headers: HeadersInit = {}) => new Request('https://controller/api/agent/libraries/' + version.id + '/manifest', { headers }); + expect(manifestRoute({ request: request(), params: { id: version.id } }).status).toBe(401); + const authorized = request({ Authorization: 'Bearer ' + credential }); + expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(200); + const partial = await serveGameFile(request({ Range: 'bytes=5-' }), version.id, file.path); + expect(partial.status).toBe(206); expect(partial.headers.get('content-range')).toBe(`bytes 5-${data.length - 1}/${data.length}`); + expect(await partial.text()).toBe(data.slice(5)); + const invalid = await serveGameFile(request({ Range: 'bytes=999-' }), version.id, file.path); + expect(invalid.status).toBe(416); + const resumed = await serveGameFile(request({ Range: 'bytes=5-', 'If-Range': '"different"' }), version.id, file.path); + expect(resumed.status).toBe(200); expect(await resumed.text()).toBe(data); + revokeCredentials(hostId); + expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(401); + }); +}); diff --git a/tests/unit/game-file-storage.test.ts b/tests/unit/game-file-storage.test.ts new file mode 100644 index 0000000..80922fd --- /dev/null +++ b/tests/unit/game-file-storage.test.ts @@ -0,0 +1,61 @@ +import { createHash } from 'node:crypto'; +import { mkdtemp, mkdir, readFile, readdir, rm, symlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { assertRegularPath, manifestChecksum, parseByteRange, receiveGameFile, validateManifest, validateGamePath, verifyGameFile } from '../../src/lib/server/game-files/storage.js'; + +const sha = (data: string) => createHash('sha256').update(data).digest('hex'); +const entry = (path = 'zone/file.ff', data = 'game data') => ({ path, size: Buffer.byteLength(data), sha256: sha(data) }); +const stream = (value: string) => new Response(value).body!; +let directory: string; +afterEach(async () => { if (directory) await rm(directory, { recursive: true, force: true }); }); + +describe('game-file manifests', () => { + it.each(['../escape', '/absolute', 'a/../b', 'a//b', 'a\\b', 'C:/file', './file', 'a\0b', 'a/file.', 'a/file '])('rejects unsafe or ambiguous path %s', path => { + expect(() => validateGamePath(path)).toThrow(); + }); + it('canonicalizes file order, rejects duplicates, and hashes the canonical manifest', () => { + const files = validateManifest([entry('z.ff'), entry('a.ff')]); + expect(files.map(file => file.path)).toEqual(['a.ff', 'z.ff']); + expect(manifestChecksum(files)).toBe(sha(JSON.stringify(files))); + expect(() => validateManifest([entry('A.ff'), entry('a.ff')])).toThrow(/Duplicate/); + expect(() => validateManifest([entry('zone'), entry('zone/file.ff')])).toThrow(/file and a directory/); + expect(() => validateManifest([{ ...entry(), size: -1 }])).toThrow(); + expect(() => validateManifest([{ ...entry(), sha256: 'bad' }])).toThrow(); + }); +}); +describe('streaming storage integrity', () => { + it('atomically accepts only complete verified streams and cleans partial uploads', async () => { + directory = await mkdtemp(join(tmpdir(), 'ec-game-storage-')); + const file = entry(); + await expect(receiveGameFile(directory, file, stream('bad'))).rejects.toThrow(/mismatch/); + expect(await readdir(join(directory, 'zone'))).toEqual([]); + await expect(receiveGameFile(directory, file, stream('game data too long'))).rejects.toThrow(/exceeds/); + await receiveGameFile(directory, file, stream('game data')); + expect(await readFile(join(directory, file.path), 'utf8')).toBe('game data'); + await verifyGameFile(directory, file); + await expect(verifyGameFile(directory, { ...file, sha256: sha('something') })).rejects.toThrow(/Checksum/); + }); + it('refuses symlinks before writing or reading shared files', async () => { + directory = await mkdtemp(join(tmpdir(), 'ec-game-symlink-')); + const root = join(directory, 'root'); await mkdir(root); + const outside = join(directory, 'outside'); await mkdir(outside); + await symlink(outside, join(root, 'zone')); + await expect(receiveGameFile(root, entry(), stream('game data'))).rejects.toThrow(/non-regular/); + await expect(assertRegularPath(root, 'zone/file.ff', true)).rejects.toThrow(/non-regular/); + expect(await readdir(outside)).toEqual([]); + }); +}); +describe('HTTP download resume ranges', () => { + it('supports open, bounded, suffix, and clipped ranges', () => { + expect(parseByteRange(null, 10)).toBeNull(); + expect(parseByteRange('bytes=3-', 10)).toEqual({ start: 3, end: 9 }); + expect(parseByteRange('bytes=3-5', 10)).toEqual({ start: 3, end: 5 }); + expect(parseByteRange('bytes=-3', 10)).toEqual({ start: 7, end: 9 }); + expect(parseByteRange('bytes=3-99', 10)).toEqual({ start: 3, end: 9 }); + }); + it.each(['bytes=10-', 'bytes=-0', 'bytes=5-3', 'bytes=0-1,3-4', 'bytes=-', 'not-a-range'])('rejects unsatisfiable %s', range => { + expect(() => parseByteRange(range, 10)).toThrow(); + }); +}); diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts new file mode 100644 index 0000000..5055e95 --- /dev/null +++ b/tests/unit/native-hosts.test.ts @@ -0,0 +1,132 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { randomBytes } from 'node:crypto'; + +let fixture: Fixture; +const inventory = { hostname: 'games-1', agentVersion: '1.0.0', protocolVersion: 1, os: 'Debian 13', architecture: 'amd64', dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }] }; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { vi.useRealTimers(); vi.unstubAllEnvs(); await fixture.cleanup(); }); + +async function enrolled() { + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Game host' }); + const enrollment = hosts.createEnrollment(host.id); + return { ...hosts, host, enrollment, credentials: hosts.enrollHost(enrollment.token, inventory) }; +} + +describe('native host enrollment and scoped authentication', () => { + it('consumes an enrollment exactly once and stores only credential hashes', async () => { + const api = await enrolled(); + expect(api.authenticateCredential(api.credentials.credential)?.id).toBe(api.host.id); + expect(api.authenticateCredential(api.enrollment.token)).toBeNull(); + expect(() => api.enrollHost(api.enrollment.token, inventory)).toThrow('invalid or expired'); + const { db } = await import('../../src/lib/server/db/index.js'); + const stored = JSON.stringify(db().prepare('SELECT * FROM ec_host_credentials').all()) + JSON.stringify(db().prepare('SELECT * FROM ec_host_enrollments').all()); + expect(stored).not.toContain(api.credentials.credential); expect(stored).not.toContain(api.enrollment.token); + }); + it('expires enrollments and invalidates earlier unused enrollment tokens', async () => { + const api = await enrolled(); + const first = api.createEnrollment(api.host.id); const next = api.createEnrollment(api.host.id, 10); + expect(() => api.enrollHost(first.token, inventory)).toThrow('invalid or expired'); + vi.useFakeTimers(); vi.setSystemTime(Date.now() + 20); + expect(() => api.enrollHost(next.token, inventory)).toThrow('invalid or expired'); + }); + it('rotates credentials independently and immediately revokes access', async () => { + const api = await enrolled(); const other = api.createHost({ name: 'Other' }); const second = api.enrollHost(api.createEnrollment(other.id).token, inventory); + const rotated = api.rotateCredential(api.host.id); + expect(api.authenticateCredential(api.credentials.credential)).toBeNull(); expect(api.authenticateCredential(rotated)?.id).toBe(api.host.id); + api.revokeCredentials(api.host.id); + expect(api.authenticateCredential(rotated)).toBeNull(); expect(api.authenticateCredential(second.credential)?.id).toBe(other.id); + }); + it('requires HTTPS and blocks cross-origin management and agent credentials on management APIs', async () => { + const api = await enrolled(); const { authenticateAgent, requireManagementOrigin } = await import('../../src/lib/server/hosts/http.js'); + const headers = { authorization: `Bearer ${api.credentials.credential}` }; + expect(authenticateAgent(new Request('https://ec.test/api/agent/operations', { headers })).id).toBe(api.host.id); + expect(() => authenticateAgent(new Request('http://ec.test/api/agent/operations', { headers }))).toThrow(); + expect(() => authenticateAgent(new Request('https://ec.test/api/agent/operations', { headers: { authorization: `Bearer ${api.enrollment.token}` } }))).toThrow(); + expect(() => requireManagementOrigin(new Request('https://ec.test/api/hosts', { headers }))).toThrow(); + expect(() => requireManagementOrigin(new Request('https://ec.test/api/hosts', { headers: { Origin: 'https://attacker.test' } }))).toThrow(); + }); + it('recovers credential rotation when acknowledgement is lost using the agent-persisted replacement', async () => { + const api = await enrolled(); const next = 'ec_host_' + randomBytes(32).toString('base64url'); + expect(api.rotateCredential(api.host.id, next)).toBe(next); + expect(api.authenticateCredential(api.credentials.credential)).toBeNull(); + expect(api.authenticateCredential(next)?.id).toBe(api.host.id); + expect(api.rotateCredential(api.host.id, next)).toBe(next); + api.revokeCredentials(api.host.id); expect(() => api.rotateCredential(api.host.id, next)).toThrow('cannot be reused'); + }); + it('bounds chunked request bodies before consuming the complete upload', async () => { + const { objectBody } = await import('../../src/lib/server/hosts/http.js'); + let cancelled = false; + const body = new ReadableStream({ pull(controller) { controller.enqueue(new TextEncoder().encode('x'.repeat(100))); }, cancel() { cancelled = true; } }); + const request = new Request('https://ec.test/api/agent/operations/x', { method: 'POST', body, duplex: 'half' } as RequestInit); + await expect(objectBody(request, 50)).rejects.toMatchObject({ status: 413 }); + expect(cancelled).toBe(true); + }); +}); + +describe('native host readiness and reservations', () => { + it('separates a connected agent from ready Docker, storage, runtime and library state', async () => { + const api = await enrolled(); + expect(api.requireReadyHost(api.host.id).readiness).toBe('ready'); + expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime'); + expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library'); + expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('preparing'); + const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false }); + expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded'); + expect(() => api.requireReadyHost(host.id)).toThrow('Docker'); + api.heartbeat(host.id, inventory); + vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000); + expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale'); + }); + it('blocks new deployments on a draining or disabled host without altering existing operations', async () => { + const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 }); + api.updateHost(api.host.id, { mode: 'draining' }); + expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('draining'); + expect(api.getHostOperation(existing.id)?.state).toBe('pending'); + api.updateHost(api.host.id, { mode: 'disabled' }); expect(api.pendingHostOperations(api.host.id)).toEqual([]); + }); + it('allocates per host, bind address and protocol while protecting wildcard bindings', async () => { + const api = await enrolled(); const second = api.createHost({ name: 'Other' }); + expect(api.reserveHostPort(api.host.id, 'a', { port: 28960 })).toBe(28960); + expect(api.reserveHostPort(second.id, 'b', { port: 28960 })).toBe(28960); + expect(api.reserveHostPort(api.host.id, 'c', { port: 28960, protocol: 'tcp' })).toBe(28960); + expect(() => api.reserveHostPort(api.host.id, 'd', { port: 28960, bindAddress: '127.0.0.1' })).toThrow('available'); + expect(api.reserveHostPort(api.host.id, 'a', { port: 28960 })).toBe(28960); + api.releaseHostPorts('a'); + expect(api.reserveHostPort(api.host.id, 'e', { port: 28960, bindAddress: '127.0.0.1' })).toBe(28960); + expect(api.reserveHostPort(api.host.id, 'f', { port: 28960, bindAddress: '127.0.0.2' })).toBe(28960); + expect(() => api.reserveHostPort(api.host.id, 'g', { port: 28960, bindAddress: '0.0.0.0' })).toThrow('available'); + expect(() => api.reserveHostPort(api.host.id, 'h', { port: 28000 })).toThrow('available'); + }); +}); + +describe('durable host operations', () => { + it('rotates unresolved operation delivery so an old batch cannot starve new work', async () => { + const api = await enrolled(); + const operations = Array.from({ length: 30 }, () => api.queueHostOperation(api.host.id, 'instance.inspect', {})); + operations.forEach(operation => api.reportHostOperation(api.host.id, operation.id, { state: 'unknown' })); + const first = api.pendingHostOperations(api.host.id).map(operation => operation.id); + const second = api.pendingHostOperations(api.host.id).map(operation => operation.id); + expect(new Set([...first, ...second]).size).toBe(30); + expect(operations.every(operation => api.getHostOperation(operation.id)?.state === 'unknown')).toBe(true); + }); + it('reconciles the same operation ID after loss of acknowledgement and prevents duplicate side effects', async () => { + const api = await enrolled(); const options = { instanceId: 'immutable-id', generation: 4, idempotencyKey: 'provisioning:123' }; + const operation = api.queueHostOperation(api.host.id, 'instance.restart', {}, options); + api.reportHostOperation(api.host.id, operation.id, { state: 'accepted' }); api.reportHostOperation(api.host.id, operation.id, { state: 'running' }); + expect(api.queueHostOperation(api.host.id, 'instance.restart', {}, options).id).toBe(operation.id); + expect(api.pendingHostOperations(api.host.id).map(value => value.id)).toEqual([operation.id]); + expect(() => api.queueHostOperation(api.host.id, 'instance.stop', {}, options)).toThrow('different request'); + api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: { restarted: true } }); + expect(api.reportHostOperation(api.host.id, operation.id, { state: 'running' }).state).toBe('completed'); + expect(api.pendingHostOperations(api.host.id)).toEqual([]); + expect((await api.waitHostOperation(operation.id)).result).toEqual({ restarted: true }); + }); + it('does not allow another host to report a result, and timeouts retain the original operation', async () => { + const api = await enrolled(); const other = api.createHost({ name: 'Other' }); const operation = api.queueHostOperation(api.host.id, 'instance.stop', {}); + expect(() => api.reportHostOperation(other.id, operation.id, { state: 'completed' })).toThrow('this host'); + await expect(api.waitHostOperation(operation.id, 0)).rejects.toThrow(operation.id); + expect(api.getHostOperation(operation.id)?.state).toBe('pending'); + }); +}); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts new file mode 100644 index 0000000..2396d47 --- /dev/null +++ b/tests/unit/native-provisioning.test.ts @@ -0,0 +1,284 @@ +import { createHash } from 'node:crypto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { join } from 'node:path'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } })), + rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) +})); +vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +let fixture: Fixture; let hostId: string; let libraryId: string; +const image = 'example.test/plutainer:test'; +const inventory = () => ({ protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' }] : [] }); +beforeEach(async () => { + vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; + fixture = await newFixture([]); + vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); + vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id; + enrollHost(createEnrollment(hostId).token, inventory()); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { + if (kind !== 'instance.inspect') throw new Error(`Unexpected operation: ${kind}`); + return { healthy: remote.state === 'running', state: remote.state, exists: true } as never; + }); +}); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); + +async function prepareLibrary(game: 't5' | 't6' = 't5') { + const libs = await import('../../src/lib/server/game-files/index.js'); + const version = libs.createLibraryImport({ hostId, game, name: game, version: '1' }); + const bytes = Buffer.from('verified base game'); + const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; + libs.setImportManifest(hostId, version.id, [file]); + await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body); + const ready = await libs.publishImport(hostId, version.id); libraryId = ready.id; + libs.updateHostLibraryCaches(hostId, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); + return ready; +} +async function queue(game: 't5' | 't6' = 't5') { + await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit'; + const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); + const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', hostId, libraryId, image, plutoKey: 'saved-key' }); + if (!result.ok) throw new Error(result.message); + return result; +} +async function operationFor(deploymentId: string) { + const { db } = await import('../../src/lib/server/db/index.js'); + const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(deploymentId) as { operation_id: string | null }; + const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); + return row?.operation_id ? getHostOperation(row.operation_id)! : null; +} +async function advance() { + const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); + await processNativeProvisioning(() => null); +} +async function acknowledge(deploymentId: string) { + const operation = await operationFor(deploymentId); if (!operation) throw new Error('Missing durable native operation'); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: remote.state, healthy: remote.state === 'running' } }); + await advance(); + return operation; +} +async function detail(id: string) { + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + return getDeployment(id)!.deployment; +} + +describe('native instance provisioning', () => { + it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => { + const job = await queue(game); await advance(); + const operation = await operationFor(job.deploymentId); + expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: job.port } }); + expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })])); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(operation!.instanceId)).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); + }); + it('reconciles the same operation after controller restart and an ambiguous remote result', async () => { + const job = await queue(); await advance(); + const original = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, original.id, { state: 'unknown', error: 'Reply lost after execution' }); + const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb(); + const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments(); + await advance(); await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'").get()).toEqual({ count: 1 }); + expect((await detail(job.deploymentId)).state).toBe('running'); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + }); + it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => { + const job = await queue(); await advance(); + const original = (await operationFor(job.deploymentId))!; + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', hostId); + await advance(); + expect(db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)).toEqual({ container_state: 'unknown' }); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + }); + it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + remote.players = 2; await advance(); + expect(await operationFor(rebuild.deploymentId)).toBeNull(); + remote.players = 0; await advance(); + const operation = (await operationFor(rebuild.deploymentId))!; + expect(operation).toMatchObject({ kind: 'instance.rebuild', generation: 1, payload: { nextGeneration: 2, libraryId } }); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(1); + await acknowledge(rebuild.deploymentId); + expect(nativeInstance(job.name)?.generation).toBe(2); + expect((await detail(rebuild.deploymentId)).state).toBe('done'); + await advance(); + expect(nativeInstance(job.name)?.generation).toBe(2); + }); + it('blocks new admission on draining hosts and retires existing instances only when empty', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' }); + const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); + expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/); + remote.players = 2; await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + const retirement = db().prepare("SELECT id FROM server_deployments WHERE action='destroy' AND server=?").get(job.name) as { id: string }; + expect(retirement).toBeTruthy(); expect(await operationFor(retirement.id)).toBeNull(); + remote.players = 0; await advance(); + const remove = await acknowledge(retirement.id); + expect(remove.kind).toBe('instance.remove'); + expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined(); + expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(remove.instanceId)).toEqual([]); + const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js'); + expect(getLibraryVersion(libraryId)?.state).toBe('ready'); + }); + it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { db } = await import('../../src/lib/server/db/index.js'); + const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes'); + db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString()); + db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name); + const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after) VALUES('campaign',?,'updated','hud.gsc',?)").run(job.name, after).lastInsertRowid); + const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); + queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId }); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' }); + remote.players = 2; await advance(); + const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + expect(staged.stage_operation_id).toBeTruthy(); + const { getHostOperation, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const stage = getHostOperation(staged.stage_operation_id)!; + expect(stage.kind).toBe('files.stage'); expect(await operationFor(restart.id)).toBeNull(); + expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending'); + reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } }); + await advance(); expect(await operationFor(restart.id)).toBeNull(); + remote.players = 0; await advance(); + const activation = (await operationFor(restart.id))!; + expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } }); + expect(db().prepare('SELECT content_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null }); + const files = stage.payload.files as Array<{ id: number; path: string }>; + reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } }); + await advance(); + expect((await detail(restart.id)).state).toBe('done'); + expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied'); + expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex') }); + const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); + expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex')); + }); + it('atomically launches mixed-game collections with per-host port reservations', async () => { + const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6'); + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] }); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image }); + expect(preview.ready).toBe(true); + expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962]); + const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }); + expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId); + const { db } = await import('../../src/lib/server/db/index.js'); + const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all(); + expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]); + const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js'); + const secondHost = createHost({ name: 'Second Debian' }); enrollHost(createEnrollment(secondHost.id).token, inventory()); + const { updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + updateHostLibraryCaches(secondHost.id, [{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }]); + const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); + const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' }); + expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message); + expect(second.port).toBe(28960); + expect(hostPortAvailable(hostId, 28960)).toBe(false); + expect(hostPortAvailable(secondHost.id, 28960)).toBe(false); + expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 }); + }); + it('reads remote instance files through the agent including missing files and epoch timestamps', async () => { + const job = await queue(); + const backend = await import('../../src/lib/server/backends/native.js'); + const remoteRead = vi.mocked(backend.nativeRequest); + remoteRead.mockImplementation(async (_instance, kind, payload) => { + if (payload?.path === 'missing') return (kind === 'files.read' ? { contentBase64: null } : null) as never; + if (kind === 'files.stat') return { size: 15, mtime: 1_790_000_000, isFile: true, isDirectory: false } as never; + if (kind === 'files.read') return { contentBase64: Buffer.from('agent file bytes').toString('base64') } as never; + if (kind === 'files.list') return { entries: [{ name: 'configs', isDirectory: true, isFile: false }] } as never; + throw new Error('Unexpected file operation'); + }); + const files = await import('../../src/lib/server/backends/files.js'); + const root = join(fixture.t5, job.name); + expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe('agent file bytes'); + const info = await files.stat(join(root, 'configs/server.cfg')); + expect(info.mtime.getTime()).toBe(1_790_000_000_000); expect(info.isFile()).toBe(true); + expect(await files.exists(join(root, 'missing'))).toBe(false); + await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ code: 'ENOENT' }); + const entries = await files.readdir(root, { withFileTypes: true }); + expect(entries[0].name).toBe('configs'); expect(entries[0].isDirectory()).toBe(true); + expect(remoteRead).toHaveBeenCalledWith(expect.objectContaining({ server: job.name }), 'files.read', { path: 'configs/server.cfg' }); + const { stat } = await import('node:fs/promises'); + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const removal = destroyServer(job.name); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockClear(); + remote.state = 'missing'; await advance(); + expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove'); + expect(rconStatusResult).not.toHaveBeenCalled(); + await acknowledge(removal.deploymentId); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)).toBeNull(); + expect((await detail(removal.deploymentId)).state).toBe('done'); + }); + it('releases a failed activation slot without recording fictitious file changes', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)").run(new Date().toISOString()); + db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')").run(job.name); + const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')").run(job.name).lastInsertRowid); + const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); + queueFileChange({ sourceKind: 'campaign', sourceId: 'preflight-failed', server: job.name, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('new bytes'), inventorySource: 'test', deploymentFileId: fileId }); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' }); + await advance(); + const stage = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, stage.stage_operation_id, { state: 'completed', result: { state: 'staged' } }); + await advance(); + const activation = (await operationFor(restart.id))!; + expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying'); + reportHostOperation(hostId, activation.id, { state: 'failed', error: 'Target changed after campaign review; no activation occurred' }); + await advance(); + expect((await detail(restart.id)).state).toBe('failed'); + expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending'); + expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 }); + expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null }); + const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); + expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined(); + }); + it('cancels a queued native launch locally and releases both ports without submitting a create', async () => { + const job = await queue(); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + const instance = nativeInstance(job.name)!; + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true); + await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect(nativeInstance(job.name)).toBeNull(); + expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(instance.id)).toEqual([]); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 }); + const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js'); + expect(getLibraryVersion(libraryId)?.state).toBe('ready'); + const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); + expect(getServerKey(job.name)).toBe('saved-key'); + }); +}); diff --git a/tests/unit/native-safety.test.ts b/tests/unit/native-safety.test.ts new file mode 100644 index 0000000..b90207a --- /dev/null +++ b/tests/unit/native-safety.test.ts @@ -0,0 +1,121 @@ +import { randomUUID } from 'node:crypto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })), + rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +let fixture: Fixture; let hostId: string; let instanceId: string; +const server = 't5_zombie_theater_1'; +const healthy = { exists: true, healthy: true, state: 'running' }; +beforeEach(async () => { + vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); + vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + hostId = createHost({ name: 'Safety test host' }).id; + enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, + storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }] }); + const { db } = await import('../../src/lib/server/db/index.js'); + instanceId = randomUUID(); + db().prepare("INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'','t5','zombie_theater','zombie','theater',28960,'running')").run(server); + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`).run(instanceId, server, hostId, new Date().toISOString()); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { + if (kind !== 'instance.inspect') throw new Error(`Unexpected remote mutation: ${kind}`); + return healthy as never; + }); +}); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); + +async function advance() { + const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); + await processNativeProvisioning(() => null); +} +async function operationId(id: string) { + const { db } = await import('../../src/lib/server/db/index.js'); + return (db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(id) as { operation_id: string | null }).operation_id; +} +async function waitingRestart() { + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const result = requestLifecycle(server, 'restart'); if (!result.ok) throw new Error(result.error); + return result.deploymentId; +} +async function suspendInspection() { + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + let release!: () => void; + vi.mocked(nativeRequest).mockImplementationOnce(async () => { await new Promise(resolve => { release = resolve; }); return healthy as never; }); + const running = advance(); + await vi.waitFor(() => expect(release).toBeTypeOf('function')); + return { running, release }; +} +async function stageCampaign() { + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign-race','promote',?)").run(new Date().toISOString()); + db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')").run(server); + const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); + queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('reviewed campaign bytes'), inventorySource: 'campaign:test' }); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const restart = requestRestart(server, { kind: 'campaign', id: 'campaign-race' }); + await advance(); + const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + expect(staged.stage_operation_id).toBeTruthy(); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed', result: { state: 'staged' } }); + return restart.id; +} + +describe('native disruption safety at dispatch', () => { + it('lets fresh occupied telemetry veto an empty RCON result and resumes after that evidence expires', async () => { + const id = await waitingRestart(); + const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Still playing', guid: 'player-1' }], receivedAt: new Date().toISOString() }); + await advance(); + expect(await operationId(id)).toBeNull(); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); + db().prepare('UPDATE server_telemetry SET received_at=? WHERE server=?').run('2000-01-01T00:00:00Z', server); + await advance(); + expect(await operationId(id)).toBeTypeOf('string'); + }); + + it('holds destruction if a beta becomes active while remote inspection is pending', async () => { + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const removal = destroyServer(server); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued.'); + const pending = await suspendInspection(); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { script, version } = createScript({ name: 'Race beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('main() {}'), scopeType: 'game', scopeValue: 't5' }); + const { db } = await import('../../src/lib/server/db/index.js'); + const betaId = db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id, version.id).lastInsertRowid; + db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId, server); + pending.release(); await pending.running; + expect(await operationId(removal.deploymentId)).toBeNull(); + expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server }); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.remove'").get()).toEqual({ count: 0 }); + }); + + it.each(['cancel', 'change bytes'] as const)('does not activate staged files after a %s during remote inspection', async change => { + const id = await stageCampaign(); + const pending = await suspendInspection(); + const { cancelStagedFiles, queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); + if (change === 'cancel') cancelStagedFiles('campaign', 'campaign-race', server); + else queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('different unreviewed bytes'), inventorySource: 'campaign:test' }); + pending.release(); await pending.running; + expect(await operationId(id)).toBeNull(); + expect(stagedFileChanges('campaign', 'campaign-race')[0].state).toBe(change === 'cancel' ? 'cancelled' : 'pending'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); + }); + + it('does not dispatch an old generation if the instance changes during inspection', async () => { + const id = await waitingRestart(); + const pending = await suspendInspection(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); + pending.release(); await pending.running; + expect(await operationId(id)).toBeNull(); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); + }); +}); diff --git a/tests/unit/system-status.test.ts b/tests/unit/system-status.test.ts index 8f05a74..292b079 100644 --- a/tests/unit/system-status.test.ts +++ b/tests/unit/system-status.test.ts @@ -13,14 +13,15 @@ beforeEach(async () => { }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); -it('distinguishes missing setup from an unverifiable host volume without probing', async () => { +it('directs a fresh controller to native hosts and verified libraries without probing Komodo', async () => { const probe = vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe'); const { GET } = await import('../../src/routes/api/system-status/+server.js'); const response = await GET({ url: new URL('http://localhost/api/system-status') }); const body = await response.json(); expect(response.headers.get('cache-control')).toBe('no-store'); - expect(body.checks.find((check: { id: string }) => check.id === 'komodo').state).toBe('unconfigured'); - expect(body.checks.find((check: { id: string }) => check.id === 'game-files').state).toBe('unverified'); + expect(body.checks.find((check: { id: string }) => check.id === 'hosts').state).toBe('unconfigured'); + expect(body.checks.find((check: { id: string }) => check.id === 'game-files').state).toBe('attention'); + expect(body.checks.some((check: { id: string }) => check.id === 'komodo')).toBe(false); expect(body.readyServers).toBe(0); expect(probe).not.toHaveBeenCalled(); }); diff --git a/tests/unit/telemetry-auth.test.ts b/tests/unit/telemetry-auth.test.ts new file mode 100644 index 0000000..7d8f6a7 --- /dev/null +++ b/tests/unit/telemetry-auth.test.ts @@ -0,0 +1,112 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { authenticateTelemetry } from '../../src/lib/server/auth/telemetry.js'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { gameEvent } from '../helpers/telemetry.js'; + +const token = 'ec_telemetry_0123456789abcdef0123456789abcdef'; +const otherToken = 'ec_telemetry_fedcba9876543210fedcba9876543210'; +const request = (values: Record = {}, url = 'http://localhost/api/telemetry') => { + const headers = new Headers({ 'Content-Type': 'application/json' }); + for (const [key, value] of Object.entries(values)) if (value !== undefined) headers.set(key, value); + return new Request(url, { method: 'POST', headers, body: JSON.stringify(gameEvent('heartbeat')) }); +}; + +beforeEach(() => { vi.stubEnv('EC_TELEMETRY_TOKEN', token); }); +afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); }); + +describe('machine telemetry credentials', () => { + it.each([ + { Authorization: `Bearer ${token}` }, + { authorization: `bearer ${token}` }, + { 'X-EC-Telemetry-Token': token }, + { Authorization: `Bearer ${token}`, 'X-EC-Telemetry-Token': token } + ])('accepts an explicit telemetry secret without a browser session: %j', headers => { + expect(authenticateTelemetry(request(headers))).toEqual({ ok: true }); + }); + + it.each([undefined, '', 'too-short', 'x'.repeat(513), 'x'.repeat(40) + ' not-header-safe'])('fails closed when configuration is unusable: %s', configured => { + vi.stubEnv('EC_TELEMETRY_TOKEN', configured); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toMatchObject({ ok: false, status: 503 }); + }); + + it.each([ + {}, + { Cookie: `ec_session=${token}` }, + { Authorization: `Bearer ${otherToken}` }, + { 'X-EC-Telemetry-Token': otherToken }, + { Authorization: `Basic ${token}`, 'X-EC-Telemetry-Token': token }, + { Authorization: `Bearer ${token}` }, + { Authorization: `Bearer ${token}, Bearer ${token}` }, + { Authorization: `Bearer ${token}`, 'X-EC-Telemetry-Token': otherToken }, + { Authorization: `Bearer ${otherToken}`, 'X-EC-Telemetry-Token': token }, + { Authorization: 'Bearer short', 'X-EC-Telemetry-Token': token }, + { Authorization: `Bearer ec_host_${'1'.repeat(43)}` }, + { Authorization: `Bearer ec_enroll_${'1'.repeat(43)}` } + ])('rejects missing, wrong, ambiguous or unrelated credentials: %j', headers => { + const authentication = authenticateTelemetry(request(headers)); + expect(authentication).toMatchObject({ ok: false, status: 401 }); + expect(JSON.stringify(authentication)).not.toContain(token); + expect(JSON.stringify(authentication)).not.toContain(otherToken); + }); + + it('does not accept tokens placed in query parameters', () => { + expect(authenticateTelemetry(request({}, `http://localhost/api/telemetry?token=${token}`))).toMatchObject({ ok: false, status: 401 }); + }); + + it('retains existing transport behavior for authenticated local machine relays', () => { + expect(authenticateTelemetry(request({ 'X-EC-Telemetry-Token': token }, 'http://controller.internal/api/telemetry'))).toEqual({ ok: true }); + }); + + it('uses the current configured token after rotation and safely rejects differing token lengths', () => { + expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toEqual({ ok: true }); + vi.stubEnv('EC_TELEMETRY_TOKEN', otherToken); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toMatchObject({ ok: false, status: 401 }); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${otherToken}` }))).toEqual({ ok: true }); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${'x'.repeat(32)}` }))).toMatchObject({ ok: false, status: 401 }); + }); +}); + +describe('telemetry endpoint authentication boundary', () => { + let fixture: Fixture; + beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture(); + const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); + await seedServerRegistry(); + }); + afterEach(async () => { await fixture.cleanup(); }); + + it.each([401, 503])('returns %s before parsing or storing untrusted evidence', async status => { + if (status === 503) vi.stubEnv('EC_TELEMETRY_TOKEN', undefined); + const { POST } = await import('../../src/routes/api/telemetry/+server.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const incoming = request({ Cookie: 'ec_session=browser-user' }); + const parse = vi.spyOn(incoming, 'json'); + const listener = vi.fn(); + const unsubscribe = subscribe(listener); + const response = await POST({ request: incoming }); + unsubscribe(); + expect(response.status).toBe(status); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect(parse).not.toHaveBeenCalled(); + expect(incoming.bodyUsed).toBe(false); + expect(listTelemetry()).toEqual([]); + expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 0 }); + expect(listener).not.toHaveBeenCalled(); + }); + + it('accepts authenticated automation with the unchanged event payload and excludes headers from diagnostics', async () => { + const { POST } = await import('../../src/routes/api/telemetry/+server.js'); + const { listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js'); + const payload = gameEvent('heartbeat'); + const response = await POST({ request: request({ 'X-EC-Telemetry-Token': token }) }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ ok: true, telemetry: payload }); + const history = listTelemetryRequests(payload.server); + expect(history).toHaveLength(1); + expect(JSON.stringify(history)).not.toContain(token); + expect(JSON.parse(history[0].payload!)).toEqual(payload); + }); +}); diff --git a/tests/unit/telemetry-diagnostics.test.ts b/tests/unit/telemetry-diagnostics.test.ts index 940238e..ee437fe 100644 --- a/tests/unit/telemetry-diagnostics.test.ts +++ b/tests/unit/telemetry-diagnostics.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { gameEvent, postTelemetry as post } from '../helpers/telemetry.js'; +import { gameEvent, postTelemetry as post, telemetryAuthHeaders } from '../helpers/telemetry.js'; const server = 't5_zombie_theater_1'; const otherServer = 't5_zombie_theater_2'; @@ -76,7 +76,7 @@ it('records identifiable JSON sent with an unsupported or missing Content-Type', const { POST } = await import('../../src/routes/api/telemetry/+server.js'); // A byte body avoids Request automatically assigning text/plain to a string body. const request = new Request('http://localhost/api/telemetry', { - method: 'POST', body: new TextEncoder().encode(JSON.stringify(payload)) + method: 'POST', headers: telemetryAuthHeaders(), body: new TextEncoder().encode(JSON.stringify(payload)) }); expect(request.headers.get('content-type')).toBeNull(); expect((await POST({ request })).status).toBe(415); @@ -124,7 +124,7 @@ it('keeps unknown and destroyed server rejections and matches the submitted serv it('does not attribute malformed JSON or unusable server names to a server', async () => { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); const malformed = new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: `{"server":"${server}",` + method: 'POST', headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, body: `{"server":"${server}",` }); expect((await POST({ request: malformed })).status).toBe(400); for (const value of [undefined, null, '', ' ', 123, 's'.repeat(256)]) { diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 3123c75..f9ea8f9 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { gameEvent, postTelemetry as post, roster } from '../helpers/telemetry.js'; +import { gameEvent, postTelemetry as post, roster, telemetryAuthHeaders } from '../helpers/telemetry.js'; let fixture: Fixture; beforeEach(async () => { @@ -171,7 +171,7 @@ it('rejects unknown and destroyed servers and non-JSON requests', async () => { it('rejects malformed JSON', async () => { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); expect((await POST({ request: new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{' + method: 'POST', headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, body: '{' }) })).status).toBe(400); });