diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..91415e8 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,5 @@ +# Prettier also respects .gitignore. +package-lock.json + +# Generated by agent/package_release.py, including its checksum manifest. +static/agent/ diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..f121071 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,14 @@ +{ + "useTabs": false, + "tabWidth": 4, + "singleQuote": true, + "trailingComma": "none", + "printWidth": 100, + "plugins": ["prettier-plugin-svelte"], + "overrides": [ + { + "files": "*.svelte", + "options": { "parser": "svelte" } + } + ] +} diff --git a/README.md b/README.md index aca2f9e..f15a674 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ a small Python agent. ![Fleet overview showing sample T5 and T6 lobbies, player counts, and match rounds](docs/assets/fleet.png) -*Fleet overview with sample data.* +_Fleet overview with sample data._ ## Features @@ -58,7 +58,7 @@ doesn't need a Docker socket or direct access to your game-server files. ## Development The controller uses SvelteKit, TypeScript, and SQLite. The host agent uses the -Python 3 standard library. For local development, install Node.js 22.12+ and Python 3: +Python 3 standard library. For local development, install Node.js 22.13+ and Python 3: ```sh npm ci @@ -72,8 +72,11 @@ database defaults to `./data/cherry.db`; keep it separate from production data. `npm run dev` packages the downloadable agent before starting the server. ```sh +npm run lint npm run check npm test npm run test:agent npm run build ``` + +Run `npm run format` to apply the repository's Prettier formatting. diff --git a/docker-compose.yml b/docker-compose.yml index 125a55f..3e01490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,29 +1,29 @@ services: - electric-cherry: - build: - context: . - image: electric-cherry:latest - container_name: electric-cherry - restart: unless-stopped - ports: - - "3000:3000" - volumes: - # Preserve the existing controller database by default. Set EC_DATA_DIR - # to a dedicated persistent directory for a new controller installation. - - ${EC_DATA_DIR:-/root/zombies/.electric-cherry}:/data:rw - environment: - ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000} - EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-} - EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-} - EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false} - # Large game files are streamed with manifest size and checksum limits. - BODY_SIZE_LIMIT: "Infinity" - DB_PATH: /data/cherry.db - GAME_FILES_ROOT: /data/game-files - BETA_ARCHIVE_DIR: /data/beta-archive - # These are logical legacy paths; native instance files live on agents. - SERVERS_ROOT: /data/legacy/t5 - T6_SERVERS_ROOT: /data/legacy/t6 - PLUTAINER_IMAGE: ${PLUTAINER_IMAGE:-ghcr.io/ayymoss/plutainer:latest} - # Optional. Native instances otherwise receive individual random secrets. - RCON_PASSWORD: ${RCON_PASSWORD:-} + electric-cherry: + build: + context: . + image: electric-cherry:latest + container_name: electric-cherry + restart: unless-stopped + ports: + - '3000:3000' + volumes: + # Preserve the existing controller database by default. Set EC_DATA_DIR + # to a dedicated persistent directory for a new controller installation. + - ${EC_DATA_DIR:-/root/zombies/.electric-cherry}:/data:rw + environment: + ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000} + EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-} + EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-} + EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false} + # Large game files are streamed with manifest size and checksum limits. + BODY_SIZE_LIMIT: 'Infinity' + DB_PATH: /data/cherry.db + GAME_FILES_ROOT: /data/game-files + BETA_ARCHIVE_DIR: /data/beta-archive + # These are logical legacy paths; native instance files live on agents. + SERVERS_ROOT: /data/legacy/t5 + T6_SERVERS_ROOT: /data/legacy/t6 + PLUTAINER_IMAGE: ${PLUTAINER_IMAGE:-ghcr.io/ayymoss/plutainer:latest} + # Optional. Native instances otherwise receive individual random secrets. + RCON_PASSWORD: ${RCON_PASSWORD:-} diff --git a/docs/development/README.md b/docs/development/README.md index bcb7f22..f91182b 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -10,7 +10,7 @@ does not require a Docker socket or direct access to host filesystems. ## Run locally -Use Node.js 22.12+ (the container uses Node.js 22), npm and Python 3. +Use Node.js 22.13+ (the container uses Node.js 22), npm and Python 3. ```sh npm ci @@ -76,12 +76,27 @@ remove the generated fixture directory. ## Checks and builds ```sh +npm run lint npm run check npm test npm run test:agent npm run build ``` +Run `npm run format` to apply Prettier formatting before committing. It uses +four-space indentation and a 100-column target for supported source, config and +documentation files, including Svelte components. Python, shell and game scripts +are outside Prettier's supported languages. Generated agent downloads and the npm +lockfile keep their generator's formatting. + +`npm run lint` checks formatting without changing files, then runs the recommended +JavaScript, TypeScript and Svelte ESLint rules. Generated build output and local +runtime data are excluded. Type checking remains part of `npm run check`. +The ESLint config documents exceptions for existing Svelte navigation, list and +reactivity patterns. Individual suppressions explain intentional validation and +test fixtures. Node.js 22.13 is the minimum supported 22.x version for the lint +toolchain. + The `package.json` overrides keep SvelteKit 2 on patched `devalue` 5.x (5.9.4 or newer) and `cookie` 0.7.x releases until its dependency ranges include those fixes. These dependencies ship in the server bundle even though SvelteKit is a @@ -103,17 +118,17 @@ T5/T6 server validation use the [host acceptance checklist](../guides/native-hos Paths below are relative to the repository root. -| Path | Responsibility | -| --- | --- | -| `src/routes` | Pages and API endpoints | -| `src/lib/ui` | Shared components, stores and [UI patterns](ui-patterns.md) | -| `src/lib/server/hosts` | Enrollment, readiness and command transport | -| `src/lib/server/backends` | Instance identity and remote file access | -| `src/lib/server/provisioning/native.ts` | Lifecycle and campaign queues connected to agents | -| `src/lib/server/game-files` | Immutable uploads and streaming distribution | -| `src/lib/server/db` | SQLite schema, migrations and shared artifact storage | -| `src/lib/server/reports` | Metrics, leaderboard and capacity workers | -| `agent` | Python service, installer, packaging and tests | +| Path | Responsibility | +| --------------------------------------- | ----------------------------------------------------------- | +| `src/routes` | Pages and API endpoints | +| `src/lib/ui` | Shared components, stores and [UI patterns](ui-patterns.md) | +| `src/lib/server/hosts` | Enrollment, readiness and command transport | +| `src/lib/server/backends` | Instance identity and remote file access | +| `src/lib/server/provisioning/native.ts` | Lifecycle and campaign queues connected to agents | +| `src/lib/server/game-files` | Immutable uploads and streaming distribution | +| `src/lib/server/db` | SQLite schema, migrations and shared artifact storage | +| `src/lib/server/reports` | Metrics, leaderboard and capacity workers | +| `agent` | Python service, installer, packaging and tests | ## Performance diagnostics diff --git a/docs/development/ui-patterns.md b/docs/development/ui-patterns.md index dc77758..b0e0b4d 100644 --- a/docs/development/ui-patterns.md +++ b/docs/development/ui-patterns.md @@ -74,16 +74,16 @@ logs. Static values use text or state colors; the accent marks links and control ### Product language -| Concept | Visible label | -| --- | --- | -| A managed game instance | Server; use the friendly map and lobby name | -| Creating a server | Create server | -| Removing a server | Remove server | -| Favourite server | Pin | -| Game does not answer | Not responding | -| Games | Black Ops I / Black Ops II; BO1 / BO2 in compact labels | -| Releasing file changes | Campaigns | -| Player completion evidence | Present in final report / Not present in final report | +| Concept | Visible label | +| -------------------------- | ------------------------------------------------------- | +| A managed game instance | Server; use the friendly map and lobby name | +| Creating a server | Create server | +| Removing a server | Remove server | +| Favourite server | Pin | +| Game does not answer | Not responding | +| Games | Black Ops I / Black Ops II; BO1 / BO2 in compact labels | +| Releasing file changes | Campaigns | +| Player completion evidence | Present in final report / Not present in final report | Keep internal values such as `t5`, `t6`, `destroy`, instance IDs, and filenames in API fields and technical details. User-facing actions use the labels above. diff --git a/docs/guides/authentication.md b/docs/guides/authentication.md index 3acd5de..6de6b4a 100644 --- a/docs/guides/authentication.md +++ b/docs/guides/authentication.md @@ -31,16 +31,16 @@ guessable token. An unset setup token holds setup closed. ## Roles -| Capability | Viewer | Operator | Administrator | -| --- | --- | --- | --- | -| Fleet, metrics, matches and operation summaries | Yes | Yes | Yes | -| Raw configs, file diffs, logs, RCON and telemetry diagnostics | No | Yes | Yes | -| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes | -| Scripts, plugins, configs, campaigns, betas and collections | Read summaries | Manage | Manage | -| Host setup, deletion, enrollment, credentials, runtime preparation and game-file upload/cache | Read status | Read status | Manage | -| Server keys and global settings/maintenance policies | No | No | Manage | -| Create users, change roles, disable accounts and reset passwords | No | No | Yes | -| Own password and sign out | Yes | Yes | Yes | +| Capability | Viewer | Operator | Administrator | +| --------------------------------------------------------------------------------------------- | -------------- | ----------- | ------------- | +| Fleet, metrics, matches and operation summaries | Yes | Yes | Yes | +| Raw configs, file diffs, logs, RCON and telemetry diagnostics | No | Yes | Yes | +| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes | +| Scripts, plugins, configs, campaigns, betas and collections | Read summaries | Manage | Manage | +| Host setup, deletion, enrollment, credentials, runtime preparation and game-file upload/cache | Read status | Read status | Manage | +| Server keys and global settings/maintenance policies | No | No | Manage | +| Create users, change roles, disable accounts and reset passwords | No | No | Yes | +| Own password and sign out | Yes | Yes | Yes | Operators can launch using available game-pool keys already saved by an administrator. They cannot save keys indirectly through a launch request. An operator's content diff --git a/docs/guides/autoscaling.md b/docs/guides/autoscaling.md index 6496fd2..a745420 100644 --- a/docs/guides/autoscaling.md +++ b/docs/guides/autoscaling.md @@ -12,10 +12,10 @@ Choose **Add policy**, select the game and scope, and enter an explicit maximum. Choose **Save policy** to apply changes directly. Saving checks the current fleet, policy revision, ownership conflicts, and host readiness. If saving is blocked, the form keeps your settings and shows the reason. Saved policies show lobby counts, key availability, and the current status. An active job appears once with its latest stage and a **View job** link. Use **Lobbies** for per-map spare coverage and occupancy evidence, **Activity** for decision changes grouped by job, and **Metrics** for observed capacity and provisioning measurements. -| Autoscaling | Behavior | -| --- | --- | -| Off | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled. | -| On | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. | +| Autoscaling | Behavior | +| ----------- | ----------------------------------------------------------------------------------------------------- | +| Off | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled. | +| On | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. | Plutonium keys belong to a game pool shared by every map, manual launch, collection, beta, and autoscaling policy for that game. An eligible creation uses the lowest available lobby number from 1–20 and reserves any free game key atomically. Key labels do not select a map or lobby. The policy shows saved and available game keys; policy maxima can exceed pool size, and a policy can be saved before any keys are added. An empty or exhausted pool holds new growth with a clear reason; it does not prevent surplus removal. Queued launches reserve their key, stopped instances retain it, and confirmed cleanup returns it to the pool. Failed or uncertain launches keep their reservation until safely reconciled. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host. diff --git a/docs/guides/beta-test-plans.md b/docs/guides/beta-test-plans.md index f02ccd9..95c94f6 100644 --- a/docs/guides/beta-test-plans.md +++ b/docs/guides/beta-test-plans.md @@ -56,12 +56,12 @@ There are four events. Send them whether or not a beta is active. ### Event fields -| Event | Fields | -| --- | --- | -| `game_start` | `server`, `map`, `players` with each player's `name` and `guid` | +| Event | Fields | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `game_start` | `server`, `map`, `players` with each player's `name` and `guid` | | `round_change` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` | -| `heartbeat` | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats | -| `game_end` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` | +| `heartbeat` | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats | +| `game_end` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` | `server` is the exact registered Electric Cherry server name. `map` is the current map. `players` is the full player array, including an empty array when @@ -89,13 +89,13 @@ Send when the game starts. Round and stats are not required. ```json { - "event": "game_start", - "server": "t6_zombie_town_1", - "map": "zm_transit", - "players": [ - { "name": "Josh", "guid": "12345678901234567" }, - { "name": "Alex", "guid": "23456789012345678" } - ] + "event": "game_start", + "server": "t6_zombie_town_1", + "map": "zm_transit", + "players": [ + { "name": "Josh", "guid": "12345678901234567" }, + { "name": "Alex", "guid": "23456789012345678" } + ] } ``` @@ -105,14 +105,30 @@ Send the new round and each player's current stats. ```json { - "event": "round_change", - "server": "t6_zombie_town_1", - "map": "zm_transit", - "round": 25, - "players": [ - { "name": "Josh", "guid": "12345678901234567", "kills": 420, "headshots": 160, "downs": 1, "revives": 3, "score": 32000 }, - { "name": "Alex", "guid": "23456789012345678", "kills": 380, "headshots": 140, "downs": 3, "revives": 1, "score": 28000 } - ] + "event": "round_change", + "server": "t6_zombie_town_1", + "map": "zm_transit", + "round": 25, + "players": [ + { + "name": "Josh", + "guid": "12345678901234567", + "kills": 420, + "headshots": 160, + "downs": 1, + "revives": 3, + "score": 32000 + }, + { + "name": "Alex", + "guid": "23456789012345678", + "kills": 380, + "headshots": 140, + "downs": 3, + "revives": 1, + "score": 28000 + } + ] } ``` @@ -122,14 +138,30 @@ Send the final round and each player's final stats when the game ends. ```json { - "event": "game_end", - "server": "t6_zombie_town_1", - "map": "zm_transit", - "round": 26, - "players": [ - { "name": "Josh", "guid": "12345678901234567", "kills": 445, "headshots": 170, "downs": 2, "revives": 3, "score": 33500 }, - { "name": "Alex", "guid": "23456789012345678", "kills": 401, "headshots": 146, "downs": 4, "revives": 1, "score": 29750 } - ] + "event": "game_end", + "server": "t6_zombie_town_1", + "map": "zm_transit", + "round": 26, + "players": [ + { + "name": "Josh", + "guid": "12345678901234567", + "kills": 445, + "headshots": 170, + "downs": 2, + "revives": 3, + "score": 33500 + }, + { + "name": "Alex", + "guid": "23456789012345678", + "kills": 401, + "headshots": 146, + "downs": 4, + "revives": 1, + "score": 29750 + } + ] } ``` @@ -213,18 +245,18 @@ an occupied server can still have no scoreboard report. Requirements on a qualifying server are ANDed: -| Requirement | Evidence | -| --- | --- | -| Reach round X | A round change, heartbeat, or game end reports round >= X with the required player count | +| Requirement | Evidence | +| --------------------------- | ------------------------------------------------------------------------------------------------ | +| Reach round X | A round change, heartbeat, or game end reports round >= X with the required player count | | Complete qualifying matches | Game end at round 2 or later, optionally qualified by a higher round, player count, and duration | -| Populated minutes | Time between consecutive game reports with enough players at both ends | -| Player-hours | Populated minutes multiplied by the lower reported player count, divided by 60 | -| Reach player count | Player array length in a game report | -| Sustained population | Consecutive report intervals above the player threshold within one game | -| Continuous healthy minutes | Consecutive fresh container/RCON observations | -| Maximum health outages | Transitions from observed healthy to unhealthy | -| Successful recovery cycles | Repeat the other requirements after each additional empty-server restart | -| Tester checklist | Tester name and observation entered on the beta detail page | +| Populated minutes | Time between consecutive game reports with enough players at both ends | +| Player-hours | Populated minutes multiplied by the lower reported player count, divided by 60 | +| Reach player count | Player array length in a game report | +| Sustained population | Consecutive report intervals above the player threshold within one game | +| Continuous healthy minutes | Consecutive fresh container/RCON observations | +| Maximum health outages | Transitions from observed healthy to unhealthy | +| Successful recovery cycles | Repeat the other requirements after each additional empty-server restart | +| Tester checklist | Tester name and observation entered on the beta detail page | All gameplay requirements use the script reports. Match counts do not read game logs. The reported round and required player count must occur together in the diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 5eb828b..cb2d487 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -26,11 +26,11 @@ openssl rand -hex 32 Set these values in `.env`: -| Variable | Purpose | -| --- | --- | -| `EC_PUBLIC_URL` | The external HTTPS address of the dashboard. | -| `EC_SETUP_TOKEN` | The generated secret, used to create the first administrator. | -| `EC_DATA_DIR` | A persistent directory for the database, game files, and beta archives. | +| Variable | Purpose | +| -------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `EC_PUBLIC_URL` | The external HTTPS address of the dashboard. | +| `EC_SETUP_TOKEN` | The generated secret, used to create the first administrator. | +| `EC_DATA_DIR` | A persistent directory for the database, game files, and beta archives. | | `EC_TELEMETRY_TOKEN` | A separate secret of at least 32 characters for game reporters. Generate it independently with the same command. | For an existing installation, retain its original data directory. Compose diff --git a/docs/guides/leaderboard.md b/docs/guides/leaderboard.md index c15c14b..af2e9dd 100644 --- a/docs/guides/leaderboard.md +++ b/docs/guides/leaderboard.md @@ -87,15 +87,15 @@ of inferring one. GUIDs stay strings. ## Recorded statistics -| Statistic | Meaning | -| --- | --- | -| Highest round | Highest round reported with that player present, including players who joined mid-game. | -| Total kills, headshots, revives, downs | Sum of each player's highest reported counter in each game. Round snapshots are cumulative, so they are never added together within a game. | -| Total rounds | Round progression during observed participation, added across games. A player observed at game start and then round 8 receives seven rounds. A player first seen at round 8 gets a highest-round record of 8, but zero total rounds; reaching round 9 then adds one. | -| Games played | Completed games in which a stats report at round 2 or later contains the player, excluding players who left in the same round they joined. Staying through game end counts even without a round change. Legacy participation with an unknown joining round still counts. Start-only and round 1 participation do not count. | -| Games completed | Games whose `game_end` roster contains the player. Disconnected players retain earlier stats without receiving completion credit. | -| Best game kills | Highest kill count recorded in any one game. | -| Highest reported score | Highest observed score balance. Spending points can lower the score, so this is not cumulative points earned. | +| Statistic | Meaning | +| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Highest round | Highest round reported with that player present, including players who joined mid-game. | +| Total kills, headshots, revives, downs | Sum of each player's highest reported counter in each game. Round snapshots are cumulative, so they are never added together within a game. | +| Total rounds | Round progression during observed participation, added across games. A player observed at game start and then round 8 receives seven rounds. A player first seen at round 8 gets a highest-round record of 8, but zero total rounds; reaching round 9 then adds one. | +| Games played | Completed games in which a stats report at round 2 or later contains the player, excluding players who left in the same round they joined. Staying through game end counts even without a round change. Legacy participation with an unknown joining round still counts. Start-only and round 1 participation do not count. | +| Games completed | Games whose `game_end` roster contains the player. Disconnected players retain earlier stats without receiving completion credit. | +| Best game kills | Highest kill count recorded in any one game. | +| Highest reported score | Highest observed score balance. Spending points can lower the score, so this is not cumulative points earned. | Same-round departures retain their recorded stats, personal bests, and match history; only the games-played count excludes them. This also changes the game @@ -211,15 +211,15 @@ example above works for command-line or server-side clients. Successful response All parameters are optional. Names and enumerated values are case-sensitive. Use URL encoding for map keys and search text. -| Parameter | Default | Accepted values and behavior | -| --- | --- | --- | -| `game` | Both games | Exactly `t5` or `t6`. An empty value is invalid. | -| `map` | All maps | Opaque `maps[].key`, at most 1024 characters. Empty means no map filter. An unknown key or a game/map mismatch returns an empty selection, not an error. | -| `players` | All squad sizes | Exactly `1`, `2`, `3`, or `4`, matching the recorded starting player count. Unknown counts contribute only when this filter is absent. | -| `metric` | `highestRound` | One of the metric keys below. An empty or unknown value is invalid. | -| `q` | No search | At most 100 characters before trimming. Matches a literal substring of the current name or GUID, ignoring ASCII letter case. Surrounding whitespace is trimmed; whitespace-only input means no search. `%` and `_` are literal characters, not wildcards. | -| `limit` | `50` | Integer from 1 to 100. | -| `offset` | `0` | Integer from 0 to 1,000,000; counts matching entries to skip, not ranks or pages. | +| Parameter | Default | Accepted values and behavior | +| --------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `game` | Both games | Exactly `t5` or `t6`. An empty value is invalid. | +| `map` | All maps | Opaque `maps[].key`, at most 1024 characters. Empty means no map filter. An unknown key or a game/map mismatch returns an empty selection, not an error. | +| `players` | All squad sizes | Exactly `1`, `2`, `3`, or `4`, matching the recorded starting player count. Unknown counts contribute only when this filter is absent. | +| `metric` | `highestRound` | One of the metric keys below. An empty or unknown value is invalid. | +| `q` | No search | At most 100 characters before trimming. Matches a literal substring of the current name or GUID, ignoring ASCII letter case. Surrounding whitespace is trimmed; whitespace-only input means no search. `%` and `_` are literal characters, not wildcards. | +| `limit` | `50` | Integer from 1 to 100. | +| `offset` | `0` | Integer from 0 to 1,000,000; counts matching entries to skip, not ranks or pages. | `limit` and `offset` accept decimal digits only. Empty strings, signs, spaces, fractions and exponent notation are rejected rather than clamped. String @@ -235,40 +235,40 @@ Metric keys are `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives HTTP `200` returns the following top-level fields directly, with no `ok` or `data` wrapper: -| Field | Type | Meaning | -| --- | --- | --- | -| `entries` | Entry array | Requested page, ordered by the selected metric descending. | -| `maps` | Map array | Available completed-match map options, filtered only by `game`, independent of `map`, `q`, metric qualification and pagination. | -| `total` | number | Number of matching GUIDs after metric qualification and search, before pagination. | -| `limit`, `offset` | number | Accepted pagination values. | -| `metric` | string | Selected metric key, including the default when omitted. | -| `summary` | object | Totals for all qualifying completed participation in the game/map scope, independent of `q`, metric qualification and pagination. | +| Field | Type | Meaning | +| ----------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `entries` | Entry array | Requested page, ordered by the selected metric descending. | +| `maps` | Map array | Available completed-match map options, filtered only by `game`, independent of `map`, `q`, metric qualification and pagination. | +| `total` | number | Number of matching GUIDs after metric qualification and search, before pagination. | +| `limit`, `offset` | number | Accepted pagination values. | +| `metric` | string | Selected metric key, including the default when omitted. | +| `summary` | object | Totals for all qualifying completed participation in the game/map scope, independent of `q`, metric qualification and pagination. | Each entry combines one GUID's qualifying participation across the selected games/maps. It contains all of these fields regardless of the selected metric: -| Entry field | Type | Meaning | -| --- | --- | --- | -| `guid` | string | Player identity. Preserve it as a string, including leading zeros. | -| `name` | string | Latest reported name across both games, even if that report is outside the selected scope or from an unfinished match. | -| `rank` | number | Rank within the scope after metric qualification, before search/pagination. Ties share rank and leave gaps: 1, 1, 3. | -| `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives`, `downs`, `gamesPlayed`, `gamesCompleted`, `bestKills`, `bestScore` | number | The ten counters defined under [Recorded statistics](#recorded-statistics). | -| `headshotRate`, `killsPerDown`, `averageKills`, `averageRounds`, `completionRate` | number or null | Ratios defined below; `null` if the denominator is zero. | -| `lastSeen` | string | Latest player observation among the qualifying participation in this scope, not the latest global identity update. | -| `records` | object | Personal-best evidence keyed by `highestRound`, `bestKills` and `bestScore`. | +| Entry field | Type | Meaning | +| ------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `guid` | string | Player identity. Preserve it as a string, including leading zeros. | +| `name` | string | Latest reported name across both games, even if that report is outside the selected scope or from an unfinished match. | +| `rank` | number | Rank within the scope after metric qualification, before search/pagination. Ties share rank and leave gaps: 1, 1, 3. | +| `highestRound`, `kills`, `roundsSurvived`, `headshots`, `revives`, `downs`, `gamesPlayed`, `gamesCompleted`, `bestKills`, `bestScore` | number | The ten counters defined under [Recorded statistics](#recorded-statistics). | +| `headshotRate`, `killsPerDown`, `averageKills`, `averageRounds`, `completionRate` | number or null | Ratios defined below; `null` if the denominator is zero. | +| `lastSeen` | string | Latest player observation among the qualifying participation in this scope, not the latest global identity update. | +| `records` | object | Personal-best evidence keyed by `highestRound`, `bestKills` and `bestScore`. | Entries have no `game` field. Equal metric values are displayed in name order (ASCII case-insensitive, then exact name), then GUID order. `q` is a substring search, not an exact-GUID lookup; a tool looking up one GUID must check equality on returned `guid` values and account for pagination. -| Ratio | Calculation | Requirement to rank by this metric | -| --- | --- | --- | -| `headshotRate` | `headshots / kills` | `gamesPlayed >= 5` and `kills >= 100` | -| `killsPerDown` | `kills / downs` | `gamesPlayed >= 5` and `downs >= 5` | -| `averageKills` | `kills / gamesPlayed` | `gamesPlayed >= 5` | -| `averageRounds` | `roundsSurvived / gamesPlayed` | `gamesPlayed >= 5` | -| `completionRate` | `gamesCompleted / gamesPlayed` | `gamesPlayed >= 5` | +| Ratio | Calculation | Requirement to rank by this metric | +| ---------------- | ------------------------------ | ------------------------------------- | +| `headshotRate` | `headshots / kills` | `gamesPlayed >= 5` and `kills >= 100` | +| `killsPerDown` | `kills / downs` | `gamesPlayed >= 5` and `downs >= 5` | +| `averageKills` | `kills / gamesPlayed` | `gamesPlayed >= 5` | +| `averageRounds` | `roundsSurvived / gamesPlayed` | `gamesPlayed >= 5` | +| `completionRate` | `gamesCompleted / gamesPlayed` | `gamesPlayed >= 5` | These are unrounded ratios, not formatted strings. For `headshotRate` and `completionRate`, `0.5` means 50%. Requirements use the selected game/map scope @@ -283,14 +283,14 @@ engine map identifier; `name` is the display name. Pass `key` as the request's completed matches reaching round 2, so an option can exist without any qualifying players. Removed servers' maps remain available. -| Summary field | Type | Meaning | -| --- | --- | --- | -| `players` | number | Unique qualifying GUIDs, combined across games. | -| `matches` | number | Distinct completed matches with qualifying player participation; not the sum of `gamesPlayed`. | -| `kills`, `roundsSurvived` | number | Sums across qualifying players; rounds are player-rounds. | -| `highestRound` | number | Highest qualifying player's recorded round. | -| `firstTrackedAt` | string or null | Earliest match `startedAt` among those matches. | -| `updatedAt` | string or null | Latest match report receipt time among those matches; not a response-generation time or a complete data-change token. | +| Summary field | Type | Meaning | +| ------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------- | +| `players` | number | Unique qualifying GUIDs, combined across games. | +| `matches` | number | Distinct completed matches with qualifying player participation; not the sum of `gamesPlayed`. | +| `kills`, `roundsSurvived` | number | Sums across qualifying players; rounds are player-rounds. | +| `highestRound` | number | Highest qualifying player's recorded round. | +| `firstTrackedAt` | string or null | Earliest match `startedAt` among those matches. | +| `updatedAt` | string or null | Latest match report receipt time among those matches; not a response-generation time or a complete data-change token. | For an empty scope, numeric summary fields are zero and both timestamps are `null`. A search with no results or an offset beyond `total` returns @@ -301,15 +301,15 @@ Each `records` value has the following fields. Evidence uses the same game/map scope as the entry, and ties choose the earliest completion, then lowest match ID. The three records can refer to different matches. -| Evidence field | Type | Meaning | -| --- | --- | --- | -| `matchId`, `value` | number | Completed match ID and that personal-best value. Link to `/matches/{matchId}` for its results page. | -| `game`, `mapKey`, `mapName`, `server` | string | Recorded game, opaque map key, map display name and server name. | -| `startedAt` | string | First recorded match observation; only the game start when `startObserved` is true. | -| `endedAt` | string or null | Recorded completion time; can be unknown in older stored records. | -| `startObserved` | boolean | Whether a `game_start` was received for the match. | -| `presentAtStart` | boolean or null | Whether the player was first observed at the recorded start; `null` for unknown older participation. | -| `firstObservedRound` | number or null | Round of the player's first observation, or unknown. This does not prove continuous survival. | +| Evidence field | Type | Meaning | +| ------------------------------------- | --------------- | ---------------------------------------------------------------------------------------------------- | +| `matchId`, `value` | number | Completed match ID and that personal-best value. Link to `/matches/{matchId}` for its results page. | +| `game`, `mapKey`, `mapName`, `server` | string | Recorded game, opaque map key, map display name and server name. | +| `startedAt` | string | First recorded match observation; only the game start when `startObserved` is true. | +| `endedAt` | string or null | Recorded completion time; can be unknown in older stored records. | +| `startObserved` | boolean | Whether a `game_start` was received for the match. | +| `presentAtStart` | boolean or null | Whether the player was first observed at the recorded start; `null` for unknown older participation. | +| `firstObservedRound` | number or null | Round of the player's first observation, or unknown. This does not prove continuous survival. | Timestamps reflect controller receipt/observation times, normally ISO 8601 UTC strings such as `2026-10-05T12:00:00.000Z`, not reporter gameplay-clock times. diff --git a/docs/guides/metrics.md b/docs/guides/metrics.md index f5e02d2..df312e3 100644 --- a/docs/guides/metrics.md +++ b/docs/guides/metrics.md @@ -63,13 +63,13 @@ represents the current registration rather than every instance in the history. Hosting Rules v1 deliberately offer experiments even with small samples: -| Recorded condition | Suggested experiment | -| --- | --- | -| At least 20% of known capacity time had every lobby full | Trial one additional lobby during busy hours. | -| Capacity unavailable and players present for at least 80% of known server time | Tentatively trial an extra lobby using occupancy as a proxy. | +| Recorded condition | Suggested experiment | +| ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------- | +| At least 20% of known capacity time had every lobby full | Trial one additional lobby during busy hours. | +| Capacity unavailable and players present for at least 80% of known server time | Tentatively trial an extra lobby using occupancy as a proxy. | | Players present for at most 10% of known time, with under 1% measured pressure or unavailable capacity | Review one idle lobby or a shorter hosting window while preserving access to the map. | -| Occupancy unavailable | Check telemetry/RCON reporting and watch for known player counts. | -| Other observations | Check missing capacity or keep the lineup and watch peaks. | +| Occupancy unavailable | Check telemetry/RCON reporting and watch for known player counts. | +| Other observations | Check missing capacity or keep the lineup and watch peaks. | Each suggestion includes evidence and an outcome to watch. Unknown capacity, less than one hour of relevant known evidence, or substantial unknown captured diff --git a/docs/guides/plugin-releases.md b/docs/guides/plugin-releases.md index f1f4b54..514fda4 100644 --- a/docs/guides/plugin-releases.md +++ b/docs/guides/plugin-releases.md @@ -52,7 +52,7 @@ GitHub release is imported from its detail page. - `GET /api/plugins/github/release?url=...` previews a release and its DLL assets. - `POST /api/plugins/github/import` accepts JSON `{ url, assetId, name, - scopeType?, scopeValue? }` for a new plugin, or `{ url, assetId, pluginId }` +scopeType?, scopeValue? }` for a new plugin, or `{ url, assetId, pluginId }` for an existing plugin. Use the preview's exact release URL, not a mutable `latest` URL. Returns plugin/version IDs, checksum and the beta builder URL. - `GET /api/plugins/updates` checks all plugins; `?pluginId=123` checks one. diff --git a/docs/guides/provisioning.md b/docs/guides/provisioning.md index 03a27d3..ec43bb2 100644 --- a/docs/guides/provisioning.md +++ b/docs/guides/provisioning.md @@ -34,14 +34,14 @@ of its explicit removal list, which resets whenever the preview is refreshed. See [Collections](collections.md) for the review details and safe submission retries when a response is lost. -| Action | Behavior | -| --- | --- | -| Create | Create a new instance using the latest promoted files and verify container health, game status, the selected map, and the Plutonium revision. | -| Start | Start the existing container and verify container health and RCON. | -| Restart | Wait until empty, restart the existing container, and verify container health and RCON. | +| Action | Behavior | +| ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Create | Create a new instance using the latest promoted files and verify container health, game status, the selected map, and the Plutonium revision. | +| Start | Start the existing container and verify container health and RCON. | +| Restart | Wait until empty, restart the existing container, and verify container health and RCON. | | Rebuild | Wait until empty, replace the instance through the host agent, preserving its server identity and port while advancing its generation. Verify container health, game status, map, and Plutonium revision. | -| Stop | Wait until empty, stop the container, and verify that the host agent reports it stopped. Keep the instance for a later Start. | -| Remove | Wait until empty, remove the container and its files through the host agent, then retire the instance and registration. | +| Stop | Wait until empty, stop the container, and verify that the host agent reports it stopped. Keep the instance for a later Start. | +| Remove | Wait until empty, remove the container and its files through the host agent, then retire the instance and registration. | ## Safety and verification diff --git a/docs/guides/settings.md b/docs/guides/settings.md index e0c263b..9fde5af 100644 --- a/docs/guides/settings.md +++ b/docs/guides/settings.md @@ -44,28 +44,28 @@ expire unused artifact versions when their age policy is disabled. Game-file gra is a delay: 0 adds no delay. A positive age and version count together preserve the minimum count as well as versions younger than the age threshold. -| History or storage | Default | -| --- | --- | -| Campaigns | 7 days | -| Successful routine operations | 1 day | -| Provisioning workflows | Indefinite | -| Agent actions | Indefinite | -| Explicitly resolved failures | Indefinite | -| Non-update maintenance jobs | 30 days | -| Resolved recovery incidents | Indefinite | -| Host resource history | 7 days | -| Instance resource samples | 1 day | -| Fleet occupancy metrics | 31 days | -| Telemetry diagnostics | 7 days; 200 per server; 10,000 overall | -| Completed/interrupted matches | Indefinite | -| Closed betas and test evidence | Indefinite | -| Campaign file bodies and closed beta snapshots | 30 days | -| Completed launch and command file bodies | 30 days | -| Unused artifact versions | No automatic expiry; no minimum count | -| Unused game files | No grace period; keep latest 1 version per game | -| Incomplete browser uploads | Indefinite | -| Authentication audit events | Indefinite | -| Minimum history count | 0 additional records | +| History or storage | Default | +| ---------------------------------------------- | ----------------------------------------------- | +| Campaigns | 7 days | +| Successful routine operations | 1 day | +| Provisioning workflows | Indefinite | +| Agent actions | Indefinite | +| Explicitly resolved failures | Indefinite | +| Non-update maintenance jobs | 30 days | +| Resolved recovery incidents | Indefinite | +| Host resource history | 7 days | +| Instance resource samples | 1 day | +| Fleet occupancy metrics | 31 days | +| Telemetry diagnostics | 7 days; 200 per server; 10,000 overall | +| Completed/interrupted matches | Indefinite | +| Closed betas and test evidence | Indefinite | +| Campaign file bodies and closed beta snapshots | 30 days | +| Completed launch and command file bodies | 30 days | +| Unused artifact versions | No automatic expiry; no minimum count | +| Unused game files | No grace period; keep latest 1 version per game | +| Incomplete browser uploads | Indefinite | +| Authentication audit events | Indefinite | +| Minimum history count | 0 additional records | Telemetry age and count policies are independent; 0 disables each individual cap. Diagnostic responses remain bounded to 10,000 rows when the per-server retention diff --git a/docs/plans/lobby-autoscaling-plan.md b/docs/plans/lobby-autoscaling-plan.md index d3ca374..ffd69f1 100644 --- a/docs/plans/lobby-autoscaling-plan.md +++ b/docs/plans/lobby-autoscaling-plan.md @@ -14,16 +14,16 @@ Update October 7, 2026: Dry run and Create only have been removed. Current polic ## Existing foundations and required extensions -| Foundation | Reuse and required work | -| --- | --- | -| Background worker | Add a requester to [pipeline/worker.ts](../../src/lib/server/pipeline/worker.ts), which already reconciles background work every 15 seconds independently of Provisioning. Keep host execution inside Provisioning. | -| Occupancy evidence | Reuse [status evidence](../../src/lib/server/fleet/status-evidence.ts) and shared freshness rules. Monitoring runs on a 30-second cadence; accepted evidence currently expires after 90 seconds. Positive occupancy vetoes an empty claim. Unknown is never zero. | -| Safe creation | Use the durable creation intent and placement path in [provisioning/native.ts](../../src/lib/server/provisioning/native.ts). Preserve CPU, memory, port, host eligibility, library, and startup concurrency checks. A blocked create remains queued. | -| Safe removal | Reuse the controller occupancy checks, instance generation checks, host-local check, and cleanup checkpoints. Add autoscaling-specific admission and cancellation behavior before enabling automatic destruction. | +| Foundation | Reuse and required work | +| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Background worker | Add a requester to [pipeline/worker.ts](../../src/lib/server/pipeline/worker.ts), which already reconciles background work every 15 seconds independently of Provisioning. Keep host execution inside Provisioning. | +| Occupancy evidence | Reuse [status evidence](../../src/lib/server/fleet/status-evidence.ts) and shared freshness rules. Monitoring runs on a 30-second cadence; accepted evidence currently expires after 90 seconds. Positive occupancy vetoes an empty claim. Unknown is never zero. | +| Safe creation | Use the durable creation intent and placement path in [provisioning/native.ts](../../src/lib/server/provisioning/native.ts). Preserve CPU, memory, port, host eligibility, library, and startup concurrency checks. A blocked create remains queued. | +| Safe removal | Reuse the controller occupancy checks, instance generation checks, host-local check, and cleanup checkpoints. Add autoscaling-specific admission and cancellation behavior before enabling automatic destruction. | | Requester eligibility | [provisioning/worker.ts](../../src/lib/server/provisioning/worker.ts) currently gives Create and Destroy separate paths that bypass the runtime requester eligibility check. Autoscaling ownership and policy checks must cover both paths, including delayed execution. A source label alone is insufficient. | -| Saved keys | [server-keys.ts](../../src/lib/server/fleet/server-keys.ts) stores keys in game pools. Each live or pending launch reserves a key until cleanup confirms it is safe to reuse. Allocate lobby identities numbered 1–20 independently of the key label. | -| Collections | [collections.ts](../../src/lib/server/fleet/collections.ts) already counts queued creations and launches promoted recipes. Exact matching can remove lobbies across the fleet, including maps omitted from the collection. Add ownership checks to preview and submission. | -| Hosting insights | [metrics-insights.ts](../../src/lib/metrics-insights.ts) provides historical suggestions for hosting trials. Use these to help choose policies; the live evaluator needs current empty-lobby evidence and must not execute historical suggestions directly. | +| Saved keys | [server-keys.ts](../../src/lib/server/fleet/server-keys.ts) stores keys in game pools. Each live or pending launch reserves a key until cleanup confirms it is safe to reuse. Allocate lobby identities numbered 1–20 independently of the key label. | +| Collections | [collections.ts](../../src/lib/server/fleet/collections.ts) already counts queued creations and launches promoted recipes. Exact matching can remove lobbies across the fleet, including maps omitted from the collection. Add ownership checks to preview and submission. | +| Hosting insights | [metrics-insights.ts](../../src/lib/metrics-insights.ts) provides historical suggestions for hosting trials. Use these to help choose policies; the live evaluator needs current empty-lobby evidence and must not execute historical suggestions directly. | Create already uses the promoted launch recipe and capacity-aware placement. Start retains existing files and has recovery concurrency controls, but would need equivalent CPU and memory admission before becoming an autoscaling strategy. The proposed scope follows option 4 in [the idle CPU research](../research/t5-t6-idle-cpu-research.md). @@ -31,15 +31,15 @@ Create already uses the promoted launch recipe and capacity-aware placement. Sta Use one policy per `(game, mapSlug)`, preserving configured variants such as T6 start locations. Do not group ambiguous legacy records by engine map alone. Resolve or exclude ambiguous identities before activation. -| Setting | Proposed initial behavior | -| --- | --- | -| Mode | Off, Dry run, Create only, or Create and destroy. Default Off. | -| Minimum running lobbies | Default 1; a floor for desired ready capacity, subject to available keys and host capacity. | -| Maximum lobbies | Required explicit limit, at most 20. Count occupied lobbies, registered identities, reservations, and unresolved creations toward admission. | -| Empty spare target | Default 1. Count empty ready lobbies, not unfilled player slots. | -| Empty duration before removal | Default 15 minutes; adjustable after the pilot. | -| Action cooldown | Default 5 minutes between new policy actions, measured from durable admission. Cancellation and safety checks do not wait for cooldown. | -| Launch settings | Reuse normal map recipes and automatic placement on eligible active hosts. Use current promoted files when provisioning begins. | +| Setting | Proposed initial behavior | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| Mode | Off, Dry run, Create only, or Create and destroy. Default Off. | +| Minimum running lobbies | Default 1; a floor for desired ready capacity, subject to available keys and host capacity. | +| Maximum lobbies | Required explicit limit, at most 20. Count occupied lobbies, registered identities, reservations, and unresolved creations toward admission. | +| Empty spare target | Default 1. Count empty ready lobbies, not unfilled player slots. | +| Empty duration before removal | Default 15 minutes; adjustable after the pilot. | +| Action cooldown | Default 5 minutes between new policy actions, measured from durable admission. Cancellation and safety checks do not wait for cooldown. | +| Launch settings | Reuse normal map recipes and automatic placement on eligible active hosts. Use current promoted files when provisioning begins. | Validate integer counts with `1 <= minimum <= maximum <= 20` and `1 <= spare target <= maximum`. Show the game pool’s saved and available counts independently of each map’s maximum. Policies can be saved with fewer keys than the maximum, or with no keys yet. Exhaustion holds growth across every map sharing that game, while eligible surplus removal can still return capacity. Game-wide targets are per map, but game keys are shared across all maps and launch paths. @@ -121,20 +121,20 @@ Complete and validate each phase in a separate commit before starting the next. Required behavioral scenarios across these phases: -| Scenario | Expected result | -| --- | --- | -| Last spare fills, including a high-round partially filled session | One create when eligible; free player slots do not count as spares. | -| Create is queued, starting, or awaiting verification | Reservation counts once; it is not advertised as ready and no duplicate is queued. | -| Every eligible host is full or unavailable | Existing job waits; other policies and worker tasks continue. | -| Game key pool is empty or exhausted | Hold growth with a clear reason; other games remain independent and eligible surplus removal can proceed. | -| A launch is queued, stopped, failed, or uncertain | Its key stays reserved until confirmed cleanup; no other launch can use that key concurrently. | -| Occupancy evidence conflicts, expires, or arrives from an old lifecycle | Positive current occupancy wins; invalid evidence cannot establish emptiness. | -| Players join before claim, after dispatch, or while the agent holds removal | Queued work is withdrawn where possible; agent safety prevents occupied destruction; held autoscaling intent cannot execute later without a fresh decision. | -| Process restarts before or after intent submission | Existing operation is recovered by durable identity without replay or duplicate admission. | -| Collection exact match omits a managed map | Preview and submission report the ownership conflict, including stale-preview races. | -| Beta, campaign, maintenance, host drain, or recovery begins after evaluation | Admission rechecks block scaling and existing work retains its lifecycle ownership. | -| Operator pauses, changes limits, or cancels a policy job | Unstarted intent is invalidated; unsettled commands stay visible and keep reservations until reconciled. | -| A name is reused after confirmed deletion | Old observations, cooldown decisions, and commands cannot apply to the replacement generation. | +| Scenario | Expected result | +| ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Last spare fills, including a high-round partially filled session | One create when eligible; free player slots do not count as spares. | +| Create is queued, starting, or awaiting verification | Reservation counts once; it is not advertised as ready and no duplicate is queued. | +| Every eligible host is full or unavailable | Existing job waits; other policies and worker tasks continue. | +| Game key pool is empty or exhausted | Hold growth with a clear reason; other games remain independent and eligible surplus removal can proceed. | +| A launch is queued, stopped, failed, or uncertain | Its key stays reserved until confirmed cleanup; no other launch can use that key concurrently. | +| Occupancy evidence conflicts, expires, or arrives from an old lifecycle | Positive current occupancy wins; invalid evidence cannot establish emptiness. | +| Players join before claim, after dispatch, or while the agent holds removal | Queued work is withdrawn where possible; agent safety prevents occupied destruction; held autoscaling intent cannot execute later without a fresh decision. | +| Process restarts before or after intent submission | Existing operation is recovered by durable identity without replay or duplicate admission. | +| Collection exact match omits a managed map | Preview and submission report the ownership conflict, including stale-preview races. | +| Beta, campaign, maintenance, host drain, or recovery begins after evaluation | Admission rechecks block scaling and existing work retains its lifecycle ownership. | +| Operator pauses, changes limits, or cancels a policy job | Unstarted intent is invalidated; unsettled commands stay visible and keep reservations until reconciled. | +| A name is reused after confirmed deletion | Old observations, cooldown decisions, and commands cannot apply to the replacement generation. | Run targeted Vitest coverage for planner, collections, provisioning, cancellation, telemetry, and routes, plus Python agent tests for the removal contract. Before each implementation phase is committed, run the relevant checks; before live release, run `npm run check`, `npm test`, `npm run test:agent`, and `npm run build`. Verify the UI with keyboard navigation and narrow layouts, including stale evidence, blocked capacity, and partial cancellation states. diff --git a/docs/plans/player-achievements-and-visual-refresh-plan.md b/docs/plans/player-achievements-and-visual-refresh-plan.md index 1704b5f..c26954a 100644 --- a/docs/plans/player-achievements-and-visual-refresh-plan.md +++ b/docs/plans/player-achievements-and-visual-refresh-plan.md @@ -21,14 +21,14 @@ its own. ## Scope and defaults -| Area | First release | -| --- | --- | -| Leaderboard periods | All time, This week, and This month, with previous/next period navigation. Keep All time as the default. | -| Calendar | Weeks start Monday at 00:00 UTC; months start on the first at 00:00 UTC. Show the boundaries explicitly. Personal timezone preferences format timestamps without changing competition membership. | -| Personal bests | Highest round, best game kills, and highest reported score, compared within the same game and map variant. | -| Sharing | A preview and PNG download for a match summary or one player's result, plus an authenticated match link. Native device sharing is an optional enhancement with download/copy fallbacks. | -| Visual direction | Existing dark surfaces, electric-blue accent, typography, and brand mark, extended with original SVG map emblems and consistent game/status treatments. | -| Initial visual coverage | Fleet, server detail, Autoscaling, provisioning and host operation history, player highlights, and match/share cards. | +| Area | First release | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Leaderboard periods | All time, This week, and This month, with previous/next period navigation. Keep All time as the default. | +| Calendar | Weeks start Monday at 00:00 UTC; months start on the first at 00:00 UTC. Show the boundaries explicitly. Personal timezone preferences format timestamps without changing competition membership. | +| Personal bests | Highest round, best game kills, and highest reported score, compared within the same game and map variant. | +| Sharing | A preview and PNG download for a match summary or one player's result, plus an authenticated match link. Native device sharing is an optional enhancement with download/copy fallbacks. | +| Visual direction | Existing dark surfaces, electric-blue accent, typography, and brand mark, extended with original SVG map emblems and consistent game/status treatments. | +| Initial visual coverage | Fleet, server detail, Autoscaling, provisioning and host operation history, player highlights, and match/share cards. | Season rewards, player-account linking, public profile/match pages, notification subscriptions, rank-change tracking, custom themes, and photographic map artwork @@ -212,13 +212,13 @@ styling, and document their use. ### Information order and status consistency -| Surface | Presentation changes | -| --- | --- | -| Fleet cards | Map/lobby/game first; health, player count, and round next; roster and match duration below; lower-priority timing and technical identity in supporting details. Preserve compact empty cards, search, pins, table mode, and the match spotlight. | -| Server detail | Add map/game identity to the shared header with health. Keep player counts, round, and timing in the Live content so Configuration and Diagnostics remain focused on their task. | -| Autoscaling | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Off/On setting, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. | -| Provisioning and host operations | Use consistent labels and status components while preserving action, target, result, elapsed time, and specific wait reasons. Put long IDs, source paths, and low-level metadata behind explicit details/copy controls. | -| Player and match results | Reuse map identity and achievement treatment across profile highlights, match results, and downloaded cards. Distinguish a current record from a record earned at that time. | +| Surface | Presentation changes | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Fleet cards | Map/lobby/game first; health, player count, and round next; roster and match duration below; lower-priority timing and technical identity in supporting details. Preserve compact empty cards, search, pins, table mode, and the match spotlight. | +| Server detail | Add map/game identity to the shared header with health. Keep player counts, round, and timing in the Live content so Configuration and Diagnostics remain focused on their task. | +| Autoscaling | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Off/On setting, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. | +| Provisioning and host operations | Use consistent labels and status components while preserving action, target, result, elapsed time, and specific wait reasons. Put long IDs, source paths, and low-level metadata behind explicit details/copy controls. | +| Player and match results | Reuse map identity and achievement treatment across profile highlights, match results, and downloaded cards. Distinguish a current record from a record earned at that time. | Align status presentation by meaning, while preserving domain-specific labels. For example, `StatusPip` currently colors starting/provisioning amber while shared @@ -238,15 +238,15 @@ Complete, validate, and commit each phase before starting the next. Finish the player-achievement workstream before starting the visual refresh. Commit any unrelated fix separately. -| Phase | Reviewable result | Acceptance gate | -| --- | --- | --- | -| A1 Completion time and period queries | Stable completion field/migration, period parsing, scoped ranking/evidence/profile queries, and cache keys. | Boundary, correction, qualification, and all-time compatibility tests pass; query plans and representative timing are recorded. | -| A2 Period navigation | Week/month controls, history navigation, period metadata, profile context, and stable share URLs. | Direct links, back/forward, rollover, empty periods, and inspection-preserving refresh work on desktop and mobile. | -| A3 Personal best highlights | Derived milestones, profile section, and match annotations. | Strict improvements, ties, first results, variants, corrections, and undated history behave as specified; long player histories remain practical. | -| A4 Result sharing | Accessible preview, PNG download, authenticated copy-link action, and supported native sharing. | Export matches the preview, contains only selected result data, handles long/incomplete data, and works after live-server deletion. Update leaderboard documentation and commit the completed workstream. | -| B1 Shared visual identity | Original emblems, resolver, game labels, achievement tokens, and usage guidance. | Every map selection resolves correctly or falls back; no misleading variant, color-only meaning, or missing-asset layout shift. | -| B2 Fleet and operational presentation | Apply identity/hierarchy/status rules to Fleet, server detail, Autoscaling, provisioning, and host operation history. | Existing actions and critical states remain visible and usable; no new polling or behavior changes. | -| B3 Player presentation and review | Apply the shared visual identity to player highlights, match pages, and export layouts; refresh documentation images. | Complete the visual/accessibility review and update UI guidance, then commit the completed workstream. | +| Phase | Reviewable result | Acceptance gate | +| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A1 Completion time and period queries | Stable completion field/migration, period parsing, scoped ranking/evidence/profile queries, and cache keys. | Boundary, correction, qualification, and all-time compatibility tests pass; query plans and representative timing are recorded. | +| A2 Period navigation | Week/month controls, history navigation, period metadata, profile context, and stable share URLs. | Direct links, back/forward, rollover, empty periods, and inspection-preserving refresh work on desktop and mobile. | +| A3 Personal best highlights | Derived milestones, profile section, and match annotations. | Strict improvements, ties, first results, variants, corrections, and undated history behave as specified; long player histories remain practical. | +| A4 Result sharing | Accessible preview, PNG download, authenticated copy-link action, and supported native sharing. | Export matches the preview, contains only selected result data, handles long/incomplete data, and works after live-server deletion. Update leaderboard documentation and commit the completed workstream. | +| B1 Shared visual identity | Original emblems, resolver, game labels, achievement tokens, and usage guidance. | Every map selection resolves correctly or falls back; no misleading variant, color-only meaning, or missing-asset layout shift. | +| B2 Fleet and operational presentation | Apply identity/hierarchy/status rules to Fleet, server detail, Autoscaling, provisioning, and host operation history. | Existing actions and critical states remain visible and usable; no new polling or behavior changes. | +| B3 Player presentation and review | Apply the shared visual identity to player highlights, match pages, and export layouts; refresh documentation images. | Complete the visual/accessibility review and update UI guidance, then commit the completed workstream. | ## Validation diff --git a/docs/research/leaderboard-query-benchmark.md b/docs/research/leaderboard-query-benchmark.md index 6d05d66..498198a 100644 --- a/docs/research/leaderboard-query-benchmark.md +++ b/docs/research/leaderboard-query-benchmark.md @@ -20,16 +20,16 @@ active, interrupted and nonqualifying records. On the local Intel i5-8279U / Node 25.9.0 / SQLite 3.49.2 test environment, three measured runs after warmup gave these median query times on the large fixture: -| Request | Before | After | -| --- | ---: | ---: | +| Request | Before | After | +| ------------------------ | -------: | -----: | | Highest round, all games | 1,717 ms | 829 ms | -| Total kills, all games | 1,555 ms | 858 ms | -| One game | 899 ms | 491 ms | -| One map | 345 ms | 224 ms | -| Player search | 1,515 ms | 757 ms | -| Deep page | 1,421 ms | 752 ms | -| Headshot percentage | 1,444 ms | 760 ms | -| Exact player rank | 1,524 ms | 554 ms | +| Total kills, all games | 1,555 ms | 858 ms | +| One game | 899 ms | 491 ms | +| One map | 345 ms | 224 ms | +| Player search | 1,515 ms | 757 ms | +| Deep page | 1,421 ms | 752 ms | +| Headshot percentage | 1,444 ms | 760 ms | +| Exact player rank | 1,524 ms | 554 ms | The small fixture's unfiltered request fell from 41 to 30 ms. All 150 additional small-fixture parity cases and 22 timed small/large request cases matched the diff --git a/docs/research/t5-t6-idle-cpu-research.md b/docs/research/t5-t6-idle-cpu-research.md index 9e6bbba..27c274a 100644 --- a/docs/research/t5-t6-idle-cpu-research.md +++ b/docs/research/t5-t6-idle-cpu-research.md @@ -26,16 +26,16 @@ The live host kernel, Wine version, image digest, installed scripts/plugins, CPU ## Options -| Option | Expected effect on idle CPU | Availability and gameplay tradeoff | Evidence and priority | -| --- | --- | --- | --- | -| 1. Enable and verify NTSYNC | Potentially substantial if Wine synchronization dominates | Intended to retain normal joins and gameplay; compatibility testing required | Official Wine/kernel support; Plutonium operator anecdote. First trial | -| 2. Audit busy-wait settings, scripts, and helpers | Small to substantial, depending on the actual cause | Usually compatible with keeping lobbies online | Documented settings plus diagnostic experiments. First trial | -| 3. Apply CPU quotas only when empty | Enforceable ceiling on each empty container's CPU time | Excessive throttling can delay joins, RCON, and heartbeats | Docker-supported mechanism; automatic policy needs development | -| 4. Run fewer spare lobbies | Removes the stopped instances' process CPU use | Fewer immediately available lobbies | Predictable mechanism; best larger saving if acceptable | -| 5. Start on request or on a schedule | Removes process CPU use during stopped periods | Players wait for startup; stopped servers cannot provide live listings | Practical alternative for infrequent maps or private sessions | -| 6. Pause and resume containers | Suspends their process execution while retaining memory | Server cannot answer while paused; resume behavior needs validation | Docker supports pause; Plutonium behavior unproven | -| 7. Wake through a UDP gateway | Could combine stopped backends with automated wakeup | Protocol, listing, session, and timeout problems | Experimental; substantially harder for T5 | -| 8. Change the host/runtime or develop engine idling | Potentially substantial, cause dependent | Migration cost or ongoing engine compatibility work | Conditional alternatives after profiling | +| Option | Expected effect on idle CPU | Availability and gameplay tradeoff | Evidence and priority | +| --------------------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| 1. Enable and verify NTSYNC | Potentially substantial if Wine synchronization dominates | Intended to retain normal joins and gameplay; compatibility testing required | Official Wine/kernel support; Plutonium operator anecdote. First trial | +| 2. Audit busy-wait settings, scripts, and helpers | Small to substantial, depending on the actual cause | Usually compatible with keeping lobbies online | Documented settings plus diagnostic experiments. First trial | +| 3. Apply CPU quotas only when empty | Enforceable ceiling on each empty container's CPU time | Excessive throttling can delay joins, RCON, and heartbeats | Docker-supported mechanism; automatic policy needs development | +| 4. Run fewer spare lobbies | Removes the stopped instances' process CPU use | Fewer immediately available lobbies | Predictable mechanism; best larger saving if acceptable | +| 5. Start on request or on a schedule | Removes process CPU use during stopped periods | Players wait for startup; stopped servers cannot provide live listings | Practical alternative for infrequent maps or private sessions | +| 6. Pause and resume containers | Suspends their process execution while retaining memory | Server cannot answer while paused; resume behavior needs validation | Docker supports pause; Plutonium behavior unproven | +| 7. Wake through a UDP gateway | Could combine stopped backends with automated wakeup | Protocol, listing, session, and timeout problems | Experimental; substantially harder for T5 | +| 8. Change the host/runtime or develop engine idling | Potentially substantial, cause dependent | Migration cost or ongoing engine compatibility work | Conditional alternatives after profiling | ### 1. Test NTSYNC before treating Wine overhead as unavoidable @@ -155,15 +155,15 @@ First record the actual host/VM details, kernel, image digest, Wine version, Plu Use one representative instance from each game, then expand successful changes to other maps. Suggested measurement windows are 10–15 minutes after startup settles, repeated after players join and all leave. These are proposed test windows, not product requirements. -| Test | What it resolves | -| --- | --- | -| Normal deployment, empty immediately after startup | Initial baseline | -| Same instance after the last player disconnects | Whether work accumulates or persists after play | -| Minimal scripts/plugins on the same map | Contribution from custom behavior | -| Verified NTSYNC versus current synchronization path | Wine synchronization contribution | -| One idle quota at a time, with repeated fresh joins | Lowest useful ceiling without degraded access | +| Test | What it resolves | +| ---------------------------------------------------------- | --------------------------------------------------------- | +| Normal deployment, empty immediately after startup | Initial baseline | +| Same instance after the last player disconnects | Whether work accumulates or persists after play | +| Minimal scripts/plugins on the same map | Contribution from custom behavior | +| Verified NTSYNC versus current synchronization path | Wine synchronization contribution | +| One idle quota at a time, with repeated fresh joins | Lowest useful ceiling without degraded access | | Stop/start or pause/resume after brief and long inactivity | Real wake latency, listing recovery, and join reliability | -| Occupied session with ordinary and high-round gameplay | Whether the change preserves active performance | +| Occupied session with ordinary and high-round gameplay | Whether the change preserves active performance | Collect container CPU averages and peaks, process/thread CPU, memory, quota throttling, RCON response times/failures, external browser visibility, first-join success, and time until players can actually play. Measure the whole host as well to catch work shifted into a proxy or management process. Use unprofiled runs for final CPU comparisons because tracing itself adds overhead. diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..2de9fc7 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,64 @@ +import js from '@eslint/js'; +import { defineConfig, globalIgnores } from 'eslint/config'; +import prettier from 'eslint-config-prettier'; +import svelte from 'eslint-plugin-svelte'; +import globals from 'globals'; +import ts from 'typescript-eslint'; +import svelteConfig from './svelte.config.js'; + +export default defineConfig( + globalIgnores([ + '**/node_modules/', + '.svelte-kit/', + 'build/', + 'data/', + 'test-fixtures/', + 'static/agent/', + '.claude/', + '.gstack/' + ]), + js.configs.recommended, + ts.configs.recommended, + svelte.configs.recommended, + prettier, + svelte.configs.prettier, + { + // The controller is deployed at the origin root, including imperative navigation in TS. + rules: { 'svelte/no-navigation-without-resolve': 'off' } + }, + { + languageOptions: { + globals: { ...globals.browser, ...globals.node } + }, + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + ignoreRestSiblings: true + } + ] + } + }, + { + files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js'], + languageOptions: { + parserOptions: { + parser: ts.parser, + svelteConfig + } + }, + rules: { + // Existing lists include intentionally positional rows and repeated primitive values. + 'svelte/require-each-key': 'off', + // Local collection helpers do not need reactive replacements. + 'svelte/prefer-svelte-reactivity': 'off', + // Preserve the existing state-reset behavior during tooling adoption. + 'svelte/prefer-writable-derived': 'off', + // Explicit string interpolation preserves whitespace around inline controls and logs. + 'svelte/no-useless-mustaches': 'off' + } + } +); diff --git a/package-lock.json b/package-lock.json index e2ba0fd..402ec73 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,2836 +1,4401 @@ { - "name": "electric-cherry", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "electric-cherry", - "version": "0.1.0", - "dependencies": { - "@fontsource/alfa-slab-one": "^5.3.0", - "@fontsource/inter": "^5.3.0", - "@fontsource/jetbrains-mono": "^5.3.0", - "better-sqlite3": "^11.3.0" - }, - "devDependencies": { - "@sveltejs/adapter-node": "^5.2.9", - "@sveltejs/kit": "^2.70.3", - "@sveltejs/vite-plugin-svelte": "^6.2.4", - "@types/better-sqlite3": "^7.6.11", - "@types/node": "^22.7.0", - "svelte": "^5.1.0", - "svelte-check": "^4.0.0", - "tsx": "^4.19.0", - "typescript": "^5.6.0", - "vite": "^7.3.7", - "vitest": "^4.1.11" - }, - "engines": { - "node": "^22.12.0 || >=24.0.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", - "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", - "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", - "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", - "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", - "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", - "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", - "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", - "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", - "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", - "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", - "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", - "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", - "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", - "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", - "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", - "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", - "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", - "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", - "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", - "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", - "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", - "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", - "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", - "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", - "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", - "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@fontsource/alfa-slab-one": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@fontsource/alfa-slab-one/-/alfa-slab-one-5.3.0.tgz", - "integrity": "sha512-J/ANFZ7S9CbZClr/QhMjmBtpFmmKipVkEXHj20aurR5uqJPCBChVPJn1/kx6vtrwnfnDY0kQaYSTYasSt4GruA==", - "license": "OFL-1.1", - "funding": { - "url": "https://github.com/sponsors/ayuhito" - } - }, - "node_modules/@fontsource/inter": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@fontsource/inter/-/inter-5.3.0.tgz", - "integrity": "sha512-RofMylZmjlJEfELXeNHFWBRcSs75rGU/6bV2S2jfnvv/3rPXPGe0LgUJTklcHZ9lM4OZmAVFhcJPnACfb91A3g==", - "license": "OFL-1.1", - "funding": { - "url": "https://github.com/sponsors/ayuhito" - } - }, - "node_modules/@fontsource/jetbrains-mono": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@fontsource/jetbrains-mono/-/jetbrains-mono-5.3.0.tgz", - "integrity": "sha512-fqDfB5I9f1p1TV486aUgB9t8zP84P0O1FtQR5Ol9vjwPy+S+EIGlVYm1cvj2W5shcZMTg2nZFdVMoH5wFu8a1A==", - "license": "OFL-1.1", - "funding": { - "url": "https://github.com/sponsors/ayuhito" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", - "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@napi-rs/lzma-linux-x64-gnu": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", - "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.20 || ^24.12 || >=25" - } - }, - "node_modules/@polka/url": { - "version": "1.0.0-next.29", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", - "integrity": "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==", - "dev": true, - "license": "MIT" - }, - "node_modules/@rollup/plugin-commonjs": { - "version": "29.0.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.3.tgz", - "integrity": "sha512-ZaOxZceP7SOUW7Lqw5IRVweSQYWaeIPnXIGLiB690EBA3FGJTO40EEr2L5yZplJWsgTCogILRSpcAe7+U0Otdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "commondir": "^1.0.1", - "estree-walker": "^2.0.2", - "fdir": "^6.2.0", - "is-reference": "1.2.1", - "magic-string": "^0.30.3", - "picomatch": "^4.0.2" - }, - "engines": { - "node": ">=16.0.0 || 14 >= 14.17" - }, - "peerDependencies": { - "rollup": "^2.68.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true + "name": "electric-cherry", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "electric-cherry", + "version": "0.1.0", + "dependencies": { + "@fontsource/alfa-slab-one": "^5.3.0", + "@fontsource/inter": "^5.3.0", + "@fontsource/jetbrains-mono": "^5.3.0", + "better-sqlite3": "^11.3.0" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@sveltejs/adapter-node": "^5.2.9", + "@sveltejs/kit": "^2.70.3", + "@sveltejs/vite-plugin-svelte": "^6.2.4", + "@types/better-sqlite3": "^7.6.11", + "@types/node": "^22.7.0", + "eslint": "^10.12.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-svelte": "^3.23.1", + "globals": "^17.13.0", + "prettier": "^3.9.9", + "prettier-plugin-svelte": "^3.5.2", + "svelte": "^5.1.0", + "svelte-check": "^4.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0", + "typescript-eslint": "^8.71.1", + "vite": "^7.3.7", + "vitest": "^4.1.11" + }, + "engines": { + "node": "^22.13.0 || >=24.0.0" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz", + "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@fontsource/alfa-slab-one": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/alfa-slab-one/-/alfa-slab-one-5.3.0.tgz", + "integrity": "sha512-J/ANFZ7S9CbZClr/QhMjmBtpFmmKipVkEXHj20aurR5uqJPCBChVPJn1/kx6vtrwnfnDY0kQaYSTYasSt4GruA==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-RofMylZmjlJEfELXeNHFWBRcSs75rGU/6bV2S2jfnvv/3rPXPGe0LgUJTklcHZ9lM4OZmAVFhcJPnACfb91A3g==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/jetbrains-mono": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/jetbrains-mono/-/jetbrains-mono-5.3.0.tgz", + "integrity": "sha512-fqDfB5I9f1p1TV486aUgB9t8zP84P0O1FtQR5Ol9vjwPy+S+EIGlVYm1cvj2W5shcZMTg2nZFdVMoH5wFu8a1A==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@polka/url": { + "version": "1.0.0-next.29", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", + "integrity": "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/plugin-commonjs": { + "version": "29.0.3", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.3.tgz", + "integrity": "sha512-ZaOxZceP7SOUW7Lqw5IRVweSQYWaeIPnXIGLiB690EBA3FGJTO40EEr2L5yZplJWsgTCogILRSpcAe7+U0Otdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "commondir": "^1.0.1", + "estree-walker": "^2.0.2", + "fdir": "^6.2.0", + "is-reference": "1.2.1", + "magic-string": "^0.30.3", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=16.0.0 || 14 >= 14.17" + }, + "peerDependencies": { + "rollup": "^2.68.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-json": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-json/-/plugin-json-6.1.0.tgz", + "integrity": "sha512-EGI2te5ENk1coGeADSIwZ7G2Q8CJS2sF120T7jLw4xFw9n7wIOXHo+kIYRAoVpJAN+kmqZSoO3Fp4JtoNF4ReA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^5.1.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-node-resolve": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-16.0.3.tgz", + "integrity": "sha512-lUYM3UBGuM93CnMPG1YocWu7X802BrNF3jW2zny5gQyLQgRFJhV1Sq0Zi74+dh/6NBx1DxFC4b4GXg9wUCG5Qg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "@types/resolve": "1.20.2", + "deepmerge": "^4.2.2", + "is-module": "^1.0.0", + "resolve": "^1.22.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^2.78.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-replace": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-6.0.3.tgz", + "integrity": "sha512-J4RZarRvQAm5IF0/LwUUg+obsm+xZhYnbMXmXROyoSE1ATJe3oXSb9L5MMppdxP2ylNSjv6zFBwKYjcKMucVfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "magic-string": "^0.30.3" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/pluginutils": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.4.0.tgz", + "integrity": "sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.63.1.tgz", + "integrity": "sha512-UZ8sUxPTiHWYX9QNdJedb1kDZSpS1t/VPWBWGSgqHNi9w3Cu6IXvu2mzbhiTiPvtrqgTQJ+zqiAq2iPIPilpaQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.63.1.tgz", + "integrity": "sha512-cQ4nFQABN5cDvDpbvJ7bMStCpnaVxynZrRMfUJYgxcIk9Sh54FIO1vtfkg0B69REjER77ioZ/ov+eAApx/KmLQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.1.tgz", + "integrity": "sha512-FQNqd1lRy/0QhDk3xeRIkSBiCpXCiDnZO3YLVdcDKN1UBiKToNftCzcXYNLshmPDUMlu2TdeS8tGcsU6f3YF1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.63.1.tgz", + "integrity": "sha512-pvD16V939D3CloK0+qikpGaxiPrDUXTe7Y5cWOMkMSy7m1cawa8EGy/kXYi/G/cKAC4HDAbSnzCIk1WmsoOKXg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.63.1.tgz", + "integrity": "sha512-pcFGeL2345VwdTnJhA6zLbew+YgWB0qBG2+dMtXjCicf6+rm6kO6cOoh5VnTe0ZMrMRgRyuHmCJxZWrIdzYuOw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.63.1.tgz", + "integrity": "sha512-mRJlqSRulVzcKq/LKA6ICSIc3K/l4fzlVn/gePn2nXIHy8seRi5z/eeRE0d/XMBxcMldiXtQTSpRj0tkkC3g8Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.63.1.tgz", + "integrity": "sha512-YDUNvVM85TI3g/1OpnqKP1h4NeW/j64DfWMf+G3M809xNk1bJSnpFp4sh83NpmVE5DXnkh8ULor4LTVZKoYLHw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.63.1.tgz", + "integrity": "sha512-7Mcn71p9ZuQFAj+h+dhQXy/yeLePRS2yKRnmW1DijA9thKO5qap0GNOIQK4yQ6iP3SU0Mrb/yWo8h8vgRba8lw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.63.1.tgz", + "integrity": "sha512-4YiLQTX6U4CSl0L9cluep9A9W6UmTfqBDc2/CH6wlu54pl4E7Jn3cOD8oxzvBDEGk/JMKgJ47C8g+radF7mwvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.63.1.tgz", + "integrity": "sha512-2ra8F7w8OquwZN9z2/fKFnli69wa8PLwaVzRMIPGb13ByMJwC28Fbp8YcVGoUhlYMTt7j5j9bNgpysrN2UM+vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.63.1.tgz", + "integrity": "sha512-Sy20ncyhjmBP0Ml+UvQbimjlk6VFgjW5uNP+qqwHB00mTE8Bl2C1TuHTlRwK2YoXeZbee5lP2XevBWVkAQAtSQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.63.1.tgz", + "integrity": "sha512-noITLp8oNjYliPnGWmLyelIHwULGqbHloQHGw1rtxbWhTuWooRpnZarZQJ1y9EUC4szuCusCc+HEpUtxpIwYvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.63.1.tgz", + "integrity": "sha512-hlxxXd+F1mWiAcaFR7Sv9ZQT6m6UfI8+Vy/kFJzztq2pDMU/0wZ9sish0iszNZvsQDo8Gc0i5yuFEOz5dDf6fA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.63.1.tgz", + "integrity": "sha512-EF7OpqQTQ/BvGqLzUi4rEHuagCV9MugAUXSHemwPW5vxZ75RR+jxO/2j95Ph2dalMpFHSVECjRoioHZgA9zOYA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.63.1.tgz", + "integrity": "sha512-wQO3JesW9PRkwlabQ27y7sPfVOOTLRG73I4F2UYHG5PXun3J9U3y+b7ezVKSYbsvSKGQ1k1cq8Qlun4C9kLt3w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.63.1.tgz", + "integrity": "sha512-ouAGwhO6wHRXdnOVCOsB0tRFkA7nhNB2Nwax6oECXN0YiN8EYUTBAOudADOB1PI+yDL61TeNx/u7MVCzksNbkQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.63.1.tgz", + "integrity": "sha512-q2R38Sn+1J8RxhfJ+T54wSWmyKXWec+9jgDfqO2AtArEqHO5R2aeayp5H5OYLr5UYDVGsVaZPEFUooMhYCdz5A==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.63.1.tgz", + "integrity": "sha512-gfI5T24WLLuFfSKw7Go/zDXjAAV0fny0swTaDv+WjK7vqcw4cRhFfdsyKL1n+ukI+ooBxn3bVQnyrn06WpI50w==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.63.1.tgz", + "integrity": "sha512-4h6XqthmB4Hspji84wvgk+ElodTsGj+dbZqHJHHtKxj4mYq0ANSEEPX9ys3moJueqsRjwpaJYH7874Itwnj2ow==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.63.1.tgz", + "integrity": "sha512-dlfCOa87o1VAYegLQ9EKilx2JCeRofiyPGhTCmqnuXZ6bMPiycO1rq1+sKoulAp7pGLIsTIw+1x5R+zgh5LhhA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.63.1.tgz", + "integrity": "sha512-cjkLbOlfcm3QGhMM1J5zaZjsw1GggbN6rw9UTSSRrPrR1KkcXnN7Uq9rPw34xImQ9VOY9GN+6u2Zj80B9ptkcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.63.1.tgz", + "integrity": "sha512-Li1KdUnWGE4N3e1F/B4RTB1ms+nG4WBgjByO46pkeBVX/2UBsY53xf5vK9WygVmnH3RwncIST7lkSdLSY6P9lg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.63.1.tgz", + "integrity": "sha512-t4ZYOSoLTgwhuFMrmTMLx/+i1DQVK7HYqMc6kY46EApwi8X0nIVphzdNoThU3xt6n+N5urG1/gxBdCaKDLavfg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.63.1.tgz", + "integrity": "sha512-RgroPfMmKlD1RzSDxvwgcPiy2HNQKoYV7OmwIXDsk73uKW5t6B/V8KIy27SMv/FNXFo/oSBtWc9J0X7t91ezZg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.63.1.tgz", + "integrity": "sha512-at8QVep6S3h5Y6gSbdGU06bRY5WJkf6WUduM9YtvYMbYhB1MOFfUgc6kehitQXzOtMSaT70q7f9ydPhpqu821w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@sveltejs/adapter-node": { + "version": "5.5.7", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-5.5.7.tgz", + "integrity": "sha512-uOfc9eVlI3A37RRSaKcgrheBYPrfJwC9VMqDp8x/O6tlKdcLLvHThSWD0KNIbjQ/d+7bwLGx3vx6aowAcRfd2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/plugin-commonjs": "^29.0.0", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^16.0.0", + "@rollup/plugin-replace": "^6.0.3", + "rollup": "^4.59.0" + }, + "peerDependencies": { + "@sveltejs/kit": "^2.4.0" + } + }, + "node_modules/@sveltejs/kit": { + "version": "2.70.3", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.70.3.tgz", + "integrity": "sha512-UDvEYuZqAMbfB/oXIoqKvbKcb7YczK5zYrzmsGV1zRJk03jntwp8dXiYoIJotxAndsKvcPFtx9H1GRSKFdSHgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@sveltejs/acorn-typescript": "^1.0.9", + "@types/cookie": "^0.6.0", + "acorn": "^8.16.0", + "cookie": "^0.6.0", + "devalue": "^5.8.1", + "esm-env": "^1.2.2", + "kleur": "^4.1.5", + "magic-string": "^0.30.5", + "mrmime": "^2.0.0", + "set-cookie-parser": "^3.0.0", + "sirv": "^3.0.0" + }, + "bin": { + "svelte-kit": "svelte-kit.js" + }, + "engines": { + "node": ">=18.13" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0 || ^6.0.0-next.0 || ^7.0.0", + "svelte": "^4.0.0 || ^5.0.0-next.0", + "typescript": "^5.3.3 || ^6.0.0", + "vite": "^5.0.3 || ^6.0.0 || ^7.0.0-beta.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/@sveltejs/load-config": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", + "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte": { + "version": "6.2.4", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.4.tgz", + "integrity": "sha512-ou/d51QSdTyN26D7h6dSpusAKaZkAiGM55/AKYi+9AGZw7q85hElbjK3kEyzXHhLSnRISHOYzVge6x0jRZ7DXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", + "deepmerge": "^4.3.1", + "magic-string": "^0.30.21", + "obug": "^2.1.0", + "vitefu": "^1.1.1" + }, + "engines": { + "node": "^20.19 || ^22.12 || >=24" + }, + "peerDependencies": { + "svelte": "^5.0.0", + "vite": "^6.3.0 || ^7.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte-inspector": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-5.0.2.tgz", + "integrity": "sha512-TZzRTcEtZffICSAoZGkPSl6Etsj2torOVrx6Uw0KpXxrec9Gg6jFWQ60Q3+LmNGfZSxHRCZL7vXVZIWmuV50Ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "obug": "^2.1.0" + }, + "engines": { + "node": "^20.19 || ^22.12 || >=24" + }, + "peerDependencies": { + "@sveltejs/vite-plugin-svelte": "^6.0.0-next.0", + "svelte": "^5.0.0", + "vite": "^6.3.0 || ^7.0.0" + } + }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/resolve": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", + "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.71.1.tgz", + "integrity": "sha512-nNlBf6HuotqMgZvSkWD5hNdemiBIxQ2NmrlgfcUIeZRs+BbakjscCQRq/Hb74EFVnFiSWWdL12JruqwBy3ERDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.71.1", + "@typescript-eslint/type-utils": "8.71.1", + "@typescript-eslint/utils": "8.71.1", + "@typescript-eslint/visitor-keys": "8.71.1", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.71.1", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.12", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz", + "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.71.1.tgz", + "integrity": "sha512-oM/5sAqz/l1v/zYK1qTOPrn2+JIrBGC1V6uNYucRO5gh6eT9vWJ/RhPbGE59byvZEUQ4VP6XkLBaS5Jti5jpvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.71.1", + "@typescript-eslint/types": "8.71.1", + "@typescript-eslint/typescript-estree": "8.71.1", + "@typescript-eslint/visitor-keys": "8.71.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.71.1.tgz", + "integrity": "sha512-Oijc9RUsohHjncg6iGi7gCtjnEjYt9FWv9u6ygK8uKcHK5eLjLAkqERQ9BaytbpceOFdk6kifPCS/zAZE+A99g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.71.1", + "@typescript-eslint/types": "^8.71.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.71.1.tgz", + "integrity": "sha512-1YUDZXdTnXLlob03SFHBipdyMBsDVm7zoSz0ytTJ39dk5J1TgsTK03VlR/dLXG/RLLUrPcYbD4rp23BmHxHP0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.71.1", + "@typescript-eslint/visitor-keys": "8.71.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.71.1.tgz", + "integrity": "sha512-oTvkml5SxXhgg+WWKhod5qWQICDXtOB+/Hi9VdPKFAANJ2b7YOgMffgvzXuqog7R6A/JtrzRf128dsxummyWhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.71.1.tgz", + "integrity": "sha512-kH3t3jZCYv2PZSBEWPKTariLrCaoF1CLoPitYFBJOhAGEwWnSRAtGEifrfrpBa4K9JlwWNZotgA5w3/wLSk/FQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.71.1", + "@typescript-eslint/typescript-estree": "8.71.1", + "@typescript-eslint/utils": "8.71.1", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.71.1.tgz", + "integrity": "sha512-ybYnPTUwg3VQQOFEiBPLs8Jxl1ry2vXdn1kVKguomBbvxpetdu3MKTNDqFNUrlG8ylP9MpZosmXMFYlGQHk1oQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.71.1.tgz", + "integrity": "sha512-pSKSEK1JJpHs6KiKVtABCE4iJK5d7FLFzDXQNBQ2oRY7LEd3UwOt3TZemTgu9syTIf56dJWlTn/pdmrO9oKUXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.71.1", + "@typescript-eslint/tsconfig-utils": "8.71.1", + "@typescript-eslint/types": "8.71.1", + "@typescript-eslint/visitor-keys": "8.71.1", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.71.1.tgz", + "integrity": "sha512-CBjT6gfAz3DW2j9Q4moFnqt208Aq+506sVGXfpRcyAeqn0EpayL0mjdWoCO2t3RC7GewNrFxdeehuEaPg6DMzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.71.1", + "@typescript-eslint/types": "8.71.1", + "@typescript-eslint/typescript-estree": "8.71.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.71.1.tgz", + "integrity": "sha512-0GxiUGqMU0qXJt58cBsiG3DP8lUmBaFo2vChtUX+CT7PnR3v/VGLLrzVP4K9r1lNiZXjHYefMBbPunplr09v6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.71.1", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.11", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/mocker/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.11", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner/node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot/node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vitest/spy": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", + "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/chai": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/devalue": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", + "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.12.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.12.0.tgz", + "integrity": "sha512-npHjrHb2o4ehH3mE+YuxABR+0gyb0aVWCIolgJDArwCiHEIsonBm3ZKjel5b0U5+a+MbEvIyRCGCzD3eKKK2NA==", + "dev": true, + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "11.1.5 || >11.1.6 <12", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-config-prettier": { + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", + "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", + "dev": true, + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-svelte": { + "version": "3.23.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-svelte/-/eslint-plugin-svelte-3.23.1.tgz", + "integrity": "sha512-+w86Jlx4IRUuuvtqD6j4njql+akx+81JnpvbN5de3AV5iq2SexvST/Jr51CiTlKeY0i+vhoKjPO/ywFKIP75sA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.6.1", + "@jridgewell/sourcemap-codec": "^1.5.0", + "esutils": "^2.0.3", + "globals": "^16.0.0", + "known-css-properties": "^0.37.0", + "postcss": "^8.4.49", + "postcss-load-config": "^3.1.4", + "postcss-safe-parser": "^7.0.0", + "semver": "^7.6.3", + "svelte-eslint-parser": "^1.7.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://github.com/sponsors/ota-meshi" + }, + "peerDependencies": { + "eslint": "^8.57.1 || ^9.0.0 || ^10.0.0", + "svelte": "^3.37.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-svelte/node_modules/globals": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", + "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrap": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.3.7.tgz", + "integrity": "sha512-n2nf7fZR3c9yXf0BPEuHuXqT+KW0SJVj4cN5FMEkpCZ3scLjOQWpiccyCxVzCC2q1wubTghuEGzngJY/7Ah0Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "17.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.13.0.tgz", + "integrity": "sha512-RwMTC61u7hrG3ZJMkZQOK4JLJrKS8QtoTii63EmWvFXHqycY9FObBJQCbsLxSO8kjKhWE5kV1GC+Vb1g3RI0Zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/is-core-module": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-reference": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", + "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/known-css-properties": { + "version": "0.37.0", + "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.37.0.tgz", + "integrity": "sha512-JCDrsP4Z1Sb9JwG0aJ8Eo2r7k4Ou5MwmThS/6lcIe1ICyb7UBJKGRIUUdqc2ASdE/42lgz6zFUnzAIhtXnBVrQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lilconfig": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", + "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/mrmime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", + "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.96.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", + "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-load-config": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-3.1.4.tgz", + "integrity": "sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "lilconfig": "^2.0.5", + "yaml": "^1.10.2" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": ">=8.0.9", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "postcss": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/postcss-load-config/node_modules/yaml": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", + "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-safe-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/postcss-safe-parser/-/postcss-safe-parser-7.1.0.tgz", + "integrity": "sha512-1WzZxRLaAFwEh6Do+zyGpjWV3nGJNxxhuh7Ubu/q1ICImMgZJnLwhoaEpRbG4pJppp2Y1ncL9ffA2f+LhrefQg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss-safe-parser" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-scss": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/postcss-scss/-/postcss-scss-4.0.9.tgz", + "integrity": "sha512-AjKOeiwAitL/MXxQW2DliT28EKukvvbEWx3LBmJIRN8KfBGZbRTxNYW0kSqi1COiTZ57nZ9NW06S6ux//N1c9A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss-scss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "postcss": "^8.4.29" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.6.tgz", + "integrity": "sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prettier": { + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", + "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-plugin-svelte": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/prettier-plugin-svelte/-/prettier-plugin-svelte-3.5.2.tgz", + "integrity": "sha512-ItFouLvzSFE3ulNl4DKoWM3BGcbDCNVpIyy/Y3F2gC3aNiGLxtFUdffVqO5Z5hhYG+DFT5KULWaxmeFFpdbvaQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "prettier": "^3.0.0", + "svelte": "^3.2.0 || ^4.0.0-next.0 || ^5.0.0-next.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/rollup": { + "version": "4.63.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.1.tgz", + "integrity": "sha512-3Df9jsstwhccuEfmAMi9l8XUh/GOkVObmFTU7CCVBysEbcOZLl84jCtaAZMcPiMz2EGKsATzQcU+Xr3n/wU6cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.63.1", + "@rollup/rollup-android-arm64": "4.63.1", + "@rollup/rollup-darwin-arm64": "4.63.1", + "@rollup/rollup-darwin-x64": "4.63.1", + "@rollup/rollup-freebsd-arm64": "4.63.1", + "@rollup/rollup-freebsd-x64": "4.63.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.63.1", + "@rollup/rollup-linux-arm-musleabihf": "4.63.1", + "@rollup/rollup-linux-arm64-gnu": "4.63.1", + "@rollup/rollup-linux-arm64-musl": "4.63.1", + "@rollup/rollup-linux-loong64-gnu": "4.63.1", + "@rollup/rollup-linux-loong64-musl": "4.63.1", + "@rollup/rollup-linux-ppc64-gnu": "4.63.1", + "@rollup/rollup-linux-ppc64-musl": "4.63.1", + "@rollup/rollup-linux-riscv64-gnu": "4.63.1", + "@rollup/rollup-linux-riscv64-musl": "4.63.1", + "@rollup/rollup-linux-s390x-gnu": "4.63.1", + "@rollup/rollup-linux-x64-gnu": "4.63.1", + "@rollup/rollup-linux-x64-musl": "4.63.1", + "@rollup/rollup-openbsd-x64": "4.63.1", + "@rollup/rollup-openharmony-arm64": "4.63.1", + "@rollup/rollup-win32-arm64-msvc": "4.63.1", + "@rollup/rollup-win32-ia32-msvc": "4.63.1", + "@rollup/rollup-win32-x64-gnu": "4.63.1", + "@rollup/rollup-win32-x64-msvc": "4.63.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-cookie-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.2.tgz", + "integrity": "sha512-5/r/lTwbJ3zQ+qwdUFZYeRNqda7P5HD8zQKqlSjdGt1/S0cjLAphHusj4Y58ahDtWn/g32xrIS58/ikOvwl0Lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/sirv": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz", + "integrity": "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@polka/url": "^1.0.0-next.24", + "mrmime": "^2.0.0", + "totalist": "^3.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz", + "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==", + "dev": true, + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/svelte": { + "version": "5.57.0", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.0.tgz", + "integrity": "sha512-NdbDn7fl4be1ViUG0oq/lvG6OZy3oENolV2ONjiqqsfVoeAfzaQAKUcEX3MrQod/Bebv1PgwET9rfXhgn9s4Kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.4", + "@jridgewell/sourcemap-codec": "^1.5.0", + "@sveltejs/acorn-typescript": "^1.0.10", + "@types/estree": "^1.0.5", + "acorn": "^8.12.1", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.8.1", + "esm-env": "^1.2.1", + "esrap": "^2.2.12", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/svelte-check": { + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", + "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "@sveltejs/load-config": "^0.2.3", + "chokidar": "^4.0.1", + "fdir": "^6.2.0", + "picocolors": "^1.0.0", + "sade": "^1.7.4" + }, + "bin": { + "svelte-check": "bin/svelte-check" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0-next.0", + "typescript": "^5.0.0 || ^6.0.0" + } + }, + "node_modules/svelte-eslint-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-1.9.0.tgz", + "integrity": "sha512-aKdKQAZP7MobfVcQNyy5Yk0QjlC9m5T1lTS0qA7+0EIsFzFX3WgbGHfb8267QK+4hI8Az9+2f+hk5sp39op8sg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-scope": "^8.2.0", + "eslint-visitor-keys": "^4.0.0", + "espree": "^10.0.0", + "postcss": "^8.4.49", + "postcss-scss": "^4.0.9", + "postcss-selector-parser": "^7.0.0", + "semver": "^7.7.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0", + "pnpm": "10.34.6" + }, + "funding": { + "url": "https://github.com/sponsors/ota-meshi" + }, + "peerDependencies": { + "svelte": "^3.37.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": true + } + } + }, + "node_modules/svelte/node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.2.0.tgz", + "integrity": "sha512-LgO3D9yZJjApUiuUfl9iFAwrtaX4+lok3wJIqttGoKCHlWUqHqbQpnxCf82L8FjgKsh4iGo78hqJwgL8F6To2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/totalist": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", + "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/tsx": { + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.71.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.71.1.tgz", + "integrity": "sha512-oiNVPC3/NbV1FpaHU07l8O6ynqG9v4PAiW2WvTrAmZdR0f2HI/gE668ElMLCK7Xbsu5k2fK5C4fXNiq05j7/nQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.71.1", + "@typescript-eslint/parser": "8.71.1", + "@typescript-eslint/typescript-estree": "8.71.1", + "@typescript-eslint/utils": "8.71.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/vite": { + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.7.tgz", + "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitefu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", + "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", + "dev": true, + "license": "MIT", + "workspaces": [ + "tests/deps/*", + "tests/projects/*", + "tests/projects/workspace/packages/*" + ], + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "vite": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/vitest/node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "dev": true, + "license": "MIT" } - } - }, - "node_modules/@rollup/plugin-json": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/@rollup/plugin-json/-/plugin-json-6.1.0.tgz", - "integrity": "sha512-EGI2te5ENk1coGeADSIwZ7G2Q8CJS2sF120T7jLw4xFw9n7wIOXHo+kIYRAoVpJAN+kmqZSoO3Fp4JtoNF4ReA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.1.0" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/plugin-node-resolve": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-16.0.3.tgz", - "integrity": "sha512-lUYM3UBGuM93CnMPG1YocWu7X802BrNF3jW2zny5gQyLQgRFJhV1Sq0Zi74+dh/6NBx1DxFC4b4GXg9wUCG5Qg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "@types/resolve": "1.20.2", - "deepmerge": "^4.2.2", - "is-module": "^1.0.0", - "resolve": "^1.22.1" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^2.78.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/plugin-replace": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-6.0.3.tgz", - "integrity": "sha512-J4RZarRvQAm5IF0/LwUUg+obsm+xZhYnbMXmXROyoSE1ATJe3oXSb9L5MMppdxP2ylNSjv6zFBwKYjcKMucVfA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/pluginutils": "^5.0.1", - "magic-string": "^0.30.3" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/pluginutils": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.4.0.tgz", - "integrity": "sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "estree-walker": "^2.0.2", - "picomatch": "^4.0.2" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } - } - }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.63.1.tgz", - "integrity": "sha512-UZ8sUxPTiHWYX9QNdJedb1kDZSpS1t/VPWBWGSgqHNi9w3Cu6IXvu2mzbhiTiPvtrqgTQJ+zqiAq2iPIPilpaQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.63.1.tgz", - "integrity": "sha512-cQ4nFQABN5cDvDpbvJ7bMStCpnaVxynZrRMfUJYgxcIk9Sh54FIO1vtfkg0B69REjER77ioZ/ov+eAApx/KmLQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.1.tgz", - "integrity": "sha512-FQNqd1lRy/0QhDk3xeRIkSBiCpXCiDnZO3YLVdcDKN1UBiKToNftCzcXYNLshmPDUMlu2TdeS8tGcsU6f3YF1Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.63.1.tgz", - "integrity": "sha512-pvD16V939D3CloK0+qikpGaxiPrDUXTe7Y5cWOMkMSy7m1cawa8EGy/kXYi/G/cKAC4HDAbSnzCIk1WmsoOKXg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.63.1.tgz", - "integrity": "sha512-pcFGeL2345VwdTnJhA6zLbew+YgWB0qBG2+dMtXjCicf6+rm6kO6cOoh5VnTe0ZMrMRgRyuHmCJxZWrIdzYuOw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.63.1.tgz", - "integrity": "sha512-mRJlqSRulVzcKq/LKA6ICSIc3K/l4fzlVn/gePn2nXIHy8seRi5z/eeRE0d/XMBxcMldiXtQTSpRj0tkkC3g8Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.63.1.tgz", - "integrity": "sha512-YDUNvVM85TI3g/1OpnqKP1h4NeW/j64DfWMf+G3M809xNk1bJSnpFp4sh83NpmVE5DXnkh8ULor4LTVZKoYLHw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.63.1.tgz", - "integrity": "sha512-7Mcn71p9ZuQFAj+h+dhQXy/yeLePRS2yKRnmW1DijA9thKO5qap0GNOIQK4yQ6iP3SU0Mrb/yWo8h8vgRba8lw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.63.1.tgz", - "integrity": "sha512-4YiLQTX6U4CSl0L9cluep9A9W6UmTfqBDc2/CH6wlu54pl4E7Jn3cOD8oxzvBDEGk/JMKgJ47C8g+radF7mwvg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.63.1.tgz", - "integrity": "sha512-2ra8F7w8OquwZN9z2/fKFnli69wa8PLwaVzRMIPGb13ByMJwC28Fbp8YcVGoUhlYMTt7j5j9bNgpysrN2UM+vw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.63.1.tgz", - "integrity": "sha512-Sy20ncyhjmBP0Ml+UvQbimjlk6VFgjW5uNP+qqwHB00mTE8Bl2C1TuHTlRwK2YoXeZbee5lP2XevBWVkAQAtSQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.63.1.tgz", - "integrity": "sha512-noITLp8oNjYliPnGWmLyelIHwULGqbHloQHGw1rtxbWhTuWooRpnZarZQJ1y9EUC4szuCusCc+HEpUtxpIwYvA==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.63.1.tgz", - "integrity": "sha512-hlxxXd+F1mWiAcaFR7Sv9ZQT6m6UfI8+Vy/kFJzztq2pDMU/0wZ9sish0iszNZvsQDo8Gc0i5yuFEOz5dDf6fA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.63.1.tgz", - "integrity": "sha512-EF7OpqQTQ/BvGqLzUi4rEHuagCV9MugAUXSHemwPW5vxZ75RR+jxO/2j95Ph2dalMpFHSVECjRoioHZgA9zOYA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.63.1.tgz", - "integrity": "sha512-wQO3JesW9PRkwlabQ27y7sPfVOOTLRG73I4F2UYHG5PXun3J9U3y+b7ezVKSYbsvSKGQ1k1cq8Qlun4C9kLt3w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.63.1.tgz", - "integrity": "sha512-ouAGwhO6wHRXdnOVCOsB0tRFkA7nhNB2Nwax6oECXN0YiN8EYUTBAOudADOB1PI+yDL61TeNx/u7MVCzksNbkQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.63.1.tgz", - "integrity": "sha512-q2R38Sn+1J8RxhfJ+T54wSWmyKXWec+9jgDfqO2AtArEqHO5R2aeayp5H5OYLr5UYDVGsVaZPEFUooMhYCdz5A==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.63.1.tgz", - "integrity": "sha512-gfI5T24WLLuFfSKw7Go/zDXjAAV0fny0swTaDv+WjK7vqcw4cRhFfdsyKL1n+ukI+ooBxn3bVQnyrn06WpI50w==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.63.1.tgz", - "integrity": "sha512-4h6XqthmB4Hspji84wvgk+ElodTsGj+dbZqHJHHtKxj4mYq0ANSEEPX9ys3moJueqsRjwpaJYH7874Itwnj2ow==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.63.1.tgz", - "integrity": "sha512-dlfCOa87o1VAYegLQ9EKilx2JCeRofiyPGhTCmqnuXZ6bMPiycO1rq1+sKoulAp7pGLIsTIw+1x5R+zgh5LhhA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] - }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.63.1.tgz", - "integrity": "sha512-cjkLbOlfcm3QGhMM1J5zaZjsw1GggbN6rw9UTSSRrPrR1KkcXnN7Uq9rPw34xImQ9VOY9GN+6u2Zj80B9ptkcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ] - }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.63.1.tgz", - "integrity": "sha512-Li1KdUnWGE4N3e1F/B4RTB1ms+nG4WBgjByO46pkeBVX/2UBsY53xf5vK9WygVmnH3RwncIST7lkSdLSY6P9lg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.63.1.tgz", - "integrity": "sha512-t4ZYOSoLTgwhuFMrmTMLx/+i1DQVK7HYqMc6kY46EApwi8X0nIVphzdNoThU3xt6n+N5urG1/gxBdCaKDLavfg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.63.1.tgz", - "integrity": "sha512-RgroPfMmKlD1RzSDxvwgcPiy2HNQKoYV7OmwIXDsk73uKW5t6B/V8KIy27SMv/FNXFo/oSBtWc9J0X7t91ezZg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.63.1.tgz", - "integrity": "sha512-at8QVep6S3h5Y6gSbdGU06bRY5WJkf6WUduM9YtvYMbYhB1MOFfUgc6kehitQXzOtMSaT70q7f9ydPhpqu821w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sveltejs/acorn-typescript": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", - "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "acorn": "^8.9.0" - } - }, - "node_modules/@sveltejs/adapter-node": { - "version": "5.5.7", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-5.5.7.tgz", - "integrity": "sha512-uOfc9eVlI3A37RRSaKcgrheBYPrfJwC9VMqDp8x/O6tlKdcLLvHThSWD0KNIbjQ/d+7bwLGx3vx6aowAcRfd2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rollup/plugin-commonjs": "^29.0.0", - "@rollup/plugin-json": "^6.1.0", - "@rollup/plugin-node-resolve": "^16.0.0", - "@rollup/plugin-replace": "^6.0.3", - "rollup": "^4.59.0" - }, - "peerDependencies": { - "@sveltejs/kit": "^2.4.0" - } - }, - "node_modules/@sveltejs/kit": { - "version": "2.70.3", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.70.3.tgz", - "integrity": "sha512-UDvEYuZqAMbfB/oXIoqKvbKcb7YczK5zYrzmsGV1zRJk03jntwp8dXiYoIJotxAndsKvcPFtx9H1GRSKFdSHgg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.0.0", - "@sveltejs/acorn-typescript": "^1.0.9", - "@types/cookie": "^0.6.0", - "acorn": "^8.16.0", - "cookie": "^0.6.0", - "devalue": "^5.8.1", - "esm-env": "^1.2.2", - "kleur": "^4.1.5", - "magic-string": "^0.30.5", - "mrmime": "^2.0.0", - "set-cookie-parser": "^3.0.0", - "sirv": "^3.0.0" - }, - "bin": { - "svelte-kit": "svelte-kit.js" - }, - "engines": { - "node": ">=18.13" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0", - "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0 || ^6.0.0-next.0 || ^7.0.0", - "svelte": "^4.0.0 || ^5.0.0-next.0", - "typescript": "^5.3.3 || ^6.0.0", - "vite": "^5.0.3 || ^6.0.0 || ^7.0.0-beta.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "typescript": { - "optional": true - } - } - }, - "node_modules/@sveltejs/load-config": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", - "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 18.0.0" - } - }, - "node_modules/@sveltejs/vite-plugin-svelte": { - "version": "6.2.4", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.4.tgz", - "integrity": "sha512-ou/d51QSdTyN26D7h6dSpusAKaZkAiGM55/AKYi+9AGZw7q85hElbjK3kEyzXHhLSnRISHOYzVge6x0jRZ7DXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", - "deepmerge": "^4.3.1", - "magic-string": "^0.30.21", - "obug": "^2.1.0", - "vitefu": "^1.1.1" - }, - "engines": { - "node": "^20.19 || ^22.12 || >=24" - }, - "peerDependencies": { - "svelte": "^5.0.0", - "vite": "^6.3.0 || ^7.0.0" - } - }, - "node_modules/@sveltejs/vite-plugin-svelte-inspector": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-5.0.2.tgz", - "integrity": "sha512-TZzRTcEtZffICSAoZGkPSl6Etsj2torOVrx6Uw0KpXxrec9Gg6jFWQ60Q3+LmNGfZSxHRCZL7vXVZIWmuV50Ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "obug": "^2.1.0" - }, - "engines": { - "node": "^20.19 || ^22.12 || >=24" - }, - "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^6.0.0-next.0", - "svelte": "^5.0.0", - "vite": "^6.3.0 || ^7.0.0" - } - }, - "node_modules/@types/better-sqlite3": { - "version": "7.6.13", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", - "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/chai": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", - "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/deep-eql": "*", - "assertion-error": "^2.0.1" - } - }, - "node_modules/@types/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/deep-eql": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", - "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", - "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "22.20.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", - "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@types/resolve": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", - "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/@vitest/expect": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", - "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", - "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.11", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/mocker/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", - "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", - "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.11", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@vitest/snapshot": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", - "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.11", - "@vitest/utils": "4.1.11", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/snapshot/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@vitest/spy": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", - "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", - "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.11", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/acorn": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", - "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/aria-query": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", - "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/assertion-error": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", - "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - } - }, - "node_modules/axobject-query": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", - "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/better-sqlite3": { - "version": "11.10.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", - "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "bindings": "^1.5.0", - "prebuild-install": "^7.1.1" - } - }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "license": "MIT", - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/chai": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", - "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" - }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/commondir": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "license": "MIT", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/devalue": { - "version": "5.9.4", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", - "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", - "dev": true, - "license": "MIT" - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", - "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.2", - "@esbuild/android-arm": "0.28.2", - "@esbuild/android-arm64": "0.28.2", - "@esbuild/android-x64": "0.28.2", - "@esbuild/darwin-arm64": "0.28.2", - "@esbuild/darwin-x64": "0.28.2", - "@esbuild/freebsd-arm64": "0.28.2", - "@esbuild/freebsd-x64": "0.28.2", - "@esbuild/linux-arm": "0.28.2", - "@esbuild/linux-arm64": "0.28.2", - "@esbuild/linux-ia32": "0.28.2", - "@esbuild/linux-loong64": "0.28.2", - "@esbuild/linux-mips64el": "0.28.2", - "@esbuild/linux-ppc64": "0.28.2", - "@esbuild/linux-riscv64": "0.28.2", - "@esbuild/linux-s390x": "0.28.2", - "@esbuild/linux-x64": "0.28.2", - "@esbuild/netbsd-arm64": "0.28.2", - "@esbuild/netbsd-x64": "0.28.2", - "@esbuild/openbsd-arm64": "0.28.2", - "@esbuild/openbsd-x64": "0.28.2", - "@esbuild/openharmony-arm64": "0.28.2", - "@esbuild/sunos-x64": "0.28.2", - "@esbuild/win32-arm64": "0.28.2", - "@esbuild/win32-ia32": "0.28.2", - "@esbuild/win32-x64": "0.28.2" - } - }, - "node_modules/esm-env": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", - "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/esrap": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.3.7.tgz", - "integrity": "sha512-n2nf7fZR3c9yXf0BPEuHuXqT+KW0SJVj4cN5FMEkpCZ3scLjOQWpiccyCxVzCC2q1wubTghuEGzngJY/7Ah0Ow==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "peerDependencies": { - "@typescript-eslint/types": "^8.2.0" - }, - "peerDependenciesMeta": { - "@typescript-eslint/types": { - "optional": true - } - } - }, - "node_modules/estree-walker": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", - "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "license": "(MIT OR WTFPL)", - "engines": { - "node": ">=6" - } - }, - "node_modules/expect-type": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", - "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "license": "MIT" - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT" - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "license": "MIT" - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC" - }, - "node_modules/is-core-module": { - "version": "2.16.2", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", - "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-module": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", - "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-reference": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", - "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "*" - } - }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/locate-character": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", - "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", - "dev": true, - "license": "MIT" - }, - "node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" - }, - "node_modules/mri": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", - "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/mrmime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", - "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/nanoid": { - "version": "3.3.19", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", - "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "license": "MIT" - }, - "node_modules/node-abi": { - "version": "3.96.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", - "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", - "license": "MIT", - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/obug": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", - "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", - "dev": true, - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT", - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", - "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/postcss": { - "version": "8.5.28", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", - "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.18", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "license": "MIT", - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pump": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", - "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/resolve": { - "version": "1.22.12", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", - "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/rollup": { - "version": "4.63.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.1.tgz", - "integrity": "sha512-3Df9jsstwhccuEfmAMi9l8XUh/GOkVObmFTU7CCVBysEbcOZLl84jCtaAZMcPiMz2EGKsATzQcU+Xr3n/wU6cg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "1.0.9" - }, - "bin": { - "rollup": "dist/bin/rollup" - }, - "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" - }, - "optionalDependencies": { - "@napi-rs/lzma-linux-x64-gnu": "1.5.1", - "@rollup/rollup-android-arm-eabi": "4.63.1", - "@rollup/rollup-android-arm64": "4.63.1", - "@rollup/rollup-darwin-arm64": "4.63.1", - "@rollup/rollup-darwin-x64": "4.63.1", - "@rollup/rollup-freebsd-arm64": "4.63.1", - "@rollup/rollup-freebsd-x64": "4.63.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.63.1", - "@rollup/rollup-linux-arm-musleabihf": "4.63.1", - "@rollup/rollup-linux-arm64-gnu": "4.63.1", - "@rollup/rollup-linux-arm64-musl": "4.63.1", - "@rollup/rollup-linux-loong64-gnu": "4.63.1", - "@rollup/rollup-linux-loong64-musl": "4.63.1", - "@rollup/rollup-linux-ppc64-gnu": "4.63.1", - "@rollup/rollup-linux-ppc64-musl": "4.63.1", - "@rollup/rollup-linux-riscv64-gnu": "4.63.1", - "@rollup/rollup-linux-riscv64-musl": "4.63.1", - "@rollup/rollup-linux-s390x-gnu": "4.63.1", - "@rollup/rollup-linux-x64-gnu": "4.63.1", - "@rollup/rollup-linux-x64-musl": "4.63.1", - "@rollup/rollup-openbsd-x64": "4.63.1", - "@rollup/rollup-openharmony-arm64": "4.63.1", - "@rollup/rollup-win32-arm64-msvc": "4.63.1", - "@rollup/rollup-win32-ia32-msvc": "4.63.1", - "@rollup/rollup-win32-x64-gnu": "4.63.1", - "@rollup/rollup-win32-x64-msvc": "4.63.1", - "fsevents": "~2.3.2" - } - }, - "node_modules/sade": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", - "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mri": "^1.1.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/set-cookie-parser": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.2.tgz", - "integrity": "sha512-5/r/lTwbJ3zQ+qwdUFZYeRNqda7P5HD8zQKqlSjdGt1/S0cjLAphHusj4Y58ahDtWn/g32xrIS58/ikOvwl0Lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true, - "license": "ISC" - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/sirv": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz", - "integrity": "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@polka/url": "^1.0.0-next.24", - "mrmime": "^2.0.0", - "totalist": "^3.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/source-map-js": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", - "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", - "dev": true, - "license": "MIT" - }, - "node_modules/std-env": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz", - "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==", - "dev": true, - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/svelte": { - "version": "5.57.0", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.0.tgz", - "integrity": "sha512-NdbDn7fl4be1ViUG0oq/lvG6OZy3oENolV2ONjiqqsfVoeAfzaQAKUcEX3MrQod/Bebv1PgwET9rfXhgn9s4Kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/remapping": "^2.3.4", - "@jridgewell/sourcemap-codec": "^1.5.0", - "@sveltejs/acorn-typescript": "^1.0.10", - "@types/estree": "^1.0.5", - "acorn": "^8.12.1", - "aria-query": "5.3.1", - "axobject-query": "^4.1.0", - "clsx": "^2.1.1", - "devalue": "^5.8.1", - "esm-env": "^1.2.1", - "esrap": "^2.2.12", - "is-reference": "^3.0.3", - "locate-character": "^3.0.0", - "magic-string": "^0.30.11", - "zimmerframe": "^1.1.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/svelte-check": { - "version": "4.7.6", - "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", - "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "@sveltejs/load-config": "^0.2.3", - "chokidar": "^4.0.1", - "fdir": "^6.2.0", - "picocolors": "^1.0.0", - "sade": "^1.7.4" - }, - "bin": { - "svelte-check": "bin/svelte-check" - }, - "engines": { - "node": ">= 18.0.0" - }, - "peerDependencies": { - "svelte": "^4.0.0 || ^5.0.0-next.0", - "typescript": "^5.0.0 || ^6.0.0" - } - }, - "node_modules/svelte/node_modules/is-reference": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", - "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.6" - } - }, - "node_modules/tar-fs": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", - "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", - "license": "MIT", - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/tinybench": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", - "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", - "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", - "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.4" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tinyrainbow": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.2.0.tgz", - "integrity": "sha512-LgO3D9yZJjApUiuUfl9iFAwrtaX4+lok3wJIqttGoKCHlWUqHqbQpnxCf82L8FjgKsh4iGo78hqJwgL8F6To2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/totalist": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", - "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/tsx": { - "version": "4.23.13", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", - "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.28.0" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/vite": { - "version": "7.3.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.7.tgz", - "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "^0.27.0 || ^0.28.0", - "fdir": "^6.5.0", - "picomatch": "^4.0.3", - "postcss": "^8.5.6", - "rollup": "^4.43.0", - "tinyglobby": "^0.2.15" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "lightningcss": "^1.21.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vitefu": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", - "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", - "dev": true, - "license": "MIT", - "workspaces": [ - "tests/deps/*", - "tests/projects/*", - "tests/projects/workspace/packages/*" - ], - "peerDependencies": { - "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "vite": { - "optional": true - } - } - }, - "node_modules/vitest": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", - "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.11", - "@vitest/mocker": "4.1.11", - "@vitest/pretty-format": "4.1.11", - "@vitest/runner": "4.1.11", - "@vitest/snapshot": "4.1.11", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.11", - "@vitest/browser-preview": "4.1.11", - "@vitest/browser-webdriverio": "4.1.11", - "@vitest/coverage-istanbul": "4.1.11", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "node_modules/vitest/node_modules/es-module-lexer": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", - "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", - "dev": true, - "license": "MIT" - }, - "node_modules/vitest/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/why-is-node-running": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", - "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, - "node_modules/zimmerframe": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", - "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", - "dev": true, - "license": "MIT" } - } } diff --git a/package.json b/package.json index b5e6887..3728d8d 100644 --- a/package.json +++ b/package.json @@ -1,55 +1,65 @@ { - "name": "electric-cherry", - "version": "0.1.0", - "description": "T5/T6 Plutonium Zombies fleet manager \u2014 replaces push_scripts.sh with a live web UI", - "private": true, - "type": "module", - "engines": { - "node": "^22.12.0 || >=24.0.0" - }, - "scripts": { - "dev": "vite dev", - "build": "vite build && npm run build:reports -- --outDir build/reports && npm run build:launch -- --outDir build/launch", - "preview": "vite preview", - "check": "npm run check:styles && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "test": "vitest run", - "test:watch": "vitest", - "benchmark:leaderboard": "node scripts/benchmark-leaderboard.mjs", - "diagnostics:database": "node scripts/database-diagnostics.mjs", - "seed:fixture": "node --import tsx scripts/seed-fixture.ts", - "agent:package": "python3 agent/package_release.py", - "prebuild": "npm run agent:package", - "test:agent": "python3 -m unittest discover -s agent/tests -v", - "build:reports": "vite build --config vite.reports.config.ts", - "build:launch": "vite build --config vite.launch.config.ts", - "predev": "npm run agent:package && npm run build:reports && npm run build:launch", - "pretest": "npm run build:reports && npm run build:launch", - "pretest:watch": "npm run build:reports && npm run build:launch", - "check:styles": "node scripts/check-ui-styles.mjs" - }, - "dependencies": { - "@fontsource/alfa-slab-one": "^5.3.0", - "@fontsource/inter": "^5.3.0", - "@fontsource/jetbrains-mono": "^5.3.0", - "better-sqlite3": "^11.3.0" - }, - "devDependencies": { - "@sveltejs/adapter-node": "^5.2.9", - "@sveltejs/kit": "^2.70.3", - "@sveltejs/vite-plugin-svelte": "^6.2.4", - "@types/better-sqlite3": "^7.6.11", - "@types/node": "^22.7.0", - "svelte": "^5.1.0", - "svelte-check": "^4.0.0", - "tsx": "^4.19.0", - "typescript": "^5.6.0", - "vite": "^7.3.7", - "vitest": "^4.1.11" - }, - "overrides": { - "@sveltejs/kit": { - "cookie": "^0.7.2" + "name": "electric-cherry", + "version": "0.1.0", + "description": "T5/T6 Plutonium Zombies fleet manager — replaces push_scripts.sh with a live web UI", + "private": true, + "type": "module", + "engines": { + "node": "^22.13.0 || >=24.0.0" }, - "devalue": "^5.9.4" - } + "scripts": { + "dev": "vite dev", + "build": "vite build && npm run build:reports -- --outDir build/reports && npm run build:launch -- --outDir build/launch", + "preview": "vite preview", + "check": "npm run check:styles && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", + "test": "vitest run", + "test:watch": "vitest", + "benchmark:leaderboard": "node scripts/benchmark-leaderboard.mjs", + "diagnostics:database": "node scripts/database-diagnostics.mjs", + "seed:fixture": "node --import tsx scripts/seed-fixture.ts", + "agent:package": "python3 agent/package_release.py", + "prebuild": "npm run agent:package", + "test:agent": "python3 -m unittest discover -s agent/tests -v", + "build:reports": "vite build --config vite.reports.config.ts", + "build:launch": "vite build --config vite.launch.config.ts", + "predev": "npm run agent:package && npm run build:reports && npm run build:launch", + "pretest": "npm run build:reports && npm run build:launch", + "pretest:watch": "npm run build:reports && npm run build:launch", + "check:styles": "node scripts/check-ui-styles.mjs", + "lint": "prettier --check . && eslint .", + "format": "prettier --write ." + }, + "dependencies": { + "@fontsource/alfa-slab-one": "^5.3.0", + "@fontsource/inter": "^5.3.0", + "@fontsource/jetbrains-mono": "^5.3.0", + "better-sqlite3": "^11.3.0" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@sveltejs/adapter-node": "^5.2.9", + "@sveltejs/kit": "^2.70.3", + "@sveltejs/vite-plugin-svelte": "^6.2.4", + "@types/better-sqlite3": "^7.6.11", + "@types/node": "^22.7.0", + "eslint": "^10.12.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-svelte": "^3.23.1", + "globals": "^17.13.0", + "prettier": "^3.9.9", + "prettier-plugin-svelte": "^3.5.2", + "svelte": "^5.1.0", + "svelte-check": "^4.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0", + "typescript-eslint": "^8.71.1", + "vite": "^7.3.7", + "vitest": "^4.1.11" + }, + "overrides": { + "@sveltejs/kit": { + "cookie": "^0.7.2" + }, + "devalue": "^5.9.4" + } } diff --git a/scripts/benchmark-leaderboard.mjs b/scripts/benchmark-leaderboard.mjs index becdc1c..dd7dc23 100644 --- a/scripts/benchmark-leaderboard.mjs +++ b/scripts/benchmark-leaderboard.mjs @@ -24,33 +24,51 @@ const allowed = new Set(['--scenario', '--runs', '--baseline-ref', '--json']); for (let index = 0; index < arguments_.length; index += 2) { const key = arguments_[index]; const value = arguments_[index + 1]; - if (!allowed.has(key) || !value) throw new Error('Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH'); + if (!allowed.has(key) || !value) + throw new Error( + 'Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH' + ); options[key.slice(2)] = value; } options.runs = Number(options.runs); -if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20) throw new Error('--runs must be an integer from 1 to 20'); -if (!['small', 'large', 'all'].includes(options.scenario)) throw new Error('--scenario must be small, large, or all'); +if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20) + throw new Error('--runs must be an integer from 1 to 20'); +if (!['small', 'large', 'all'].includes(options.scenario)) + throw new Error('--scenario must be small, large, or all'); const baseline = options['baseline-ref'] - ? execFileSync('git', ['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`], { cwd: repo, encoding: 'utf8' }).trim() + ? execFileSync( + 'git', + ['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`], + { cwd: repo, encoding: 'utf8' } + ).trim() : null; const definitions = [ { name: 'small', matches: 2000, players: 500 }, { name: 'large', matches: 50000, players: 10000 } -].filter(item => options.scenario === 'all' || item.name === options.scenario); +].filter((item) => options.scenario === 'all' || item.name === options.scenario); const temporary = mkdtempSync(join(tmpdir(), 'cherry-leaderboard-benchmark-')); -const source = (file, ref) => ref - ? execFileSync('git', ['show', `${ref}:${file}`], { cwd: repo, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }) - : readFileSync(join(repo, file), 'utf8'); -const percentile = (values, fraction) => [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1]; +const source = (file, ref) => + ref + ? execFileSync('git', ['show', `${ref}:${file}`], { + cwd: repo, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024 + }) + : readFileSync(join(repo, file), 'utf8'); +const percentile = (values, fraction) => + [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1]; function seed(handle, definition) { handle.exec(source('src/lib/server/db/schema.sql')); - handle.exec("CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)"); + handle.exec( + "CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)" + ); const match = handle.prepare(`INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, start_observed, start_players, highest_round, last_event, last_fingerprint) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 4, ?, ?, ?)`); - const identity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + const identity = + handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END, last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`); @@ -60,19 +78,38 @@ function seed(handle, definition) { first_observed_round, present_at_start) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`); let state = 20260918; - const random = () => { state = (Math.imul(state, 1664525) + 1013904223) >>> 0; return state / 4294967296; }; + const random = () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; handle.transaction(() => { for (let id = 1; id <= definition.matches; id++) { const mapIndex = Math.floor(random() * 12); const game = mapIndex < 6 ? 't5' : 't6'; const map = `map_${mapIndex}`; const status = id % 20 === 0 ? 'active' : id % 20 === 1 ? 'interrupted' : 'completed'; - const started = new Date(Date.UTC(2023, 0, 1) + Math.floor(id / definition.matches * 3 * 365 * 86400000)).toISOString(); - const ended = new Date(Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000).toISOString(); + const started = new Date( + Date.UTC(2023, 0, 1) + Math.floor((id / definition.matches) * 3 * 365 * 86400000) + ).toISOString(); + const ended = new Date( + Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000 + ).toISOString(); const round = id % 101 === 0 ? 1 : 2 + Math.floor(random() * 60); - match.run(id, `server_${id % 24}`, game, map, `${game}:${map}`, `Map ${mapIndex}`, - started, status === 'completed' ? ended : null, ended, status, round, - status === 'completed' ? 'game_end' : 'heartbeat', `event_${id}`); + match.run( + id, + `server_${id % 24}`, + game, + map, + `${game}:${map}`, + `Map ${mapIndex}`, + started, + status === 'completed' ? ended : null, + ended, + status, + round, + status === 'completed' ? 'game_end' : 'heartbeat', + `event_${id}` + ); const roster = new Set(); while (roster.size < 4) roster.add(Math.floor(random() ** 2 * definition.players)); // Reserve one completed participation for an explicitly ineligible player. @@ -84,16 +121,47 @@ function seed(handle, definition) { const kills = Math.floor(round * 18 + random() * 800); const presentAtStart = random() > 0.1; const firstRound = presentAtStart ? 0 : Math.max(1, Math.floor(random() * round)); - participation.run(id, guid, round, Math.max(0, round - Math.max(1, firstRound)), kills, - Math.floor(kills * random()), Math.floor(random() * 8), Math.floor(random() * 12), - Math.floor(random() * 100000), Math.max(0, round - 1), Number(status === 'active'), - Number(random() > 0.01), Number(status === 'completed' && random() > 0.15), - started, ended, firstRound, Number(presentAtStart)); + participation.run( + id, + guid, + round, + Math.max(0, round - Math.max(1, firstRound)), + kills, + Math.floor(kills * random()), + Math.floor(random() * 8), + Math.floor(random() * 12), + Math.floor(random() * 100000), + Math.max(0, round - 1), + Number(status === 'active'), + Number(random() > 0.01), + Number(status === 'completed' && random() > 0.15), + started, + ended, + firstRound, + Number(presentAtStart) + ); } if (id === definition.matches - 2) { identity.run(game, 'benchmark-low-sample', 'Low sample', started, ended); - participation.run(id, 'benchmark-low-sample', 2, 1, 20, 5, 0, 0, 1000, - 1, 0, 1, 1, started, ended, 0, 1); + participation.run( + id, + 'benchmark-low-sample', + 2, + 1, + 20, + 5, + 0, + 0, + 1000, + 1, + 0, + 1, + 1, + started, + ended, + 0, + 1 + ); } } })(); @@ -104,15 +172,27 @@ function loadImplementation(handle, ref) { const modules = new Map(); const hashes = {}; let recorded = null; - const database = { prepare(sql) { - const statement = handle.prepare(sql); - return Object.fromEntries(['all', 'get'].map(method => [method, (...parameters) => { - const start = performance.now(); - const result = statement[method](...parameters); - if (recorded) recorded.push({ sql, parameters, milliseconds: performance.now() - start }); - return result; - }])); - } }; + const database = { + prepare(sql) { + const statement = handle.prepare(sql); + return Object.fromEntries( + ['all', 'get'].map((method) => [ + method, + (...parameters) => { + const start = performance.now(); + const result = statement[method](...parameters); + if (recorded) + recorded.push({ + sql, + parameters, + milliseconds: performance.now() - start + }); + return result; + } + ]) + ); + } + }; function load(file) { if (file === 'src/lib/server/db/index.ts') return { db: () => database }; if (modules.has(file)) return modules.get(file); @@ -123,10 +203,14 @@ function loadImplementation(handle, ref) { }).outputText; const exports = {}; modules.set(file, exports); - const localRequire = specifier => { + const localRequire = (specifier) => { if (!specifier.startsWith('.')) return require(specifier); - const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(/\.js$/, '.ts'); - if (!target.startsWith('src/lib/')) throw new Error(`Unexpected benchmark dependency: ${target}`); + const target = relative(repo, resolve(repo, dirname(file), specifier)).replace( + /\.js$/, + '.ts' + ); + if (!target.startsWith('src/lib/')) + throw new Error(`Unexpected benchmark dependency: ${target}`); return load(target); }; new Function('require', 'exports', compiled)(localRequire, exports); @@ -134,10 +218,14 @@ function loadImplementation(handle, ref) { } const api = load('src/lib/server/fleet/leaderboard.ts'); return { - name: ref ? 'baseline' : 'current', ref, hashes, - metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map(metric => metric.key), + name: ref ? 'baseline' : 'current', + ref, + hashes, + metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map((metric) => metric.key), run(test) { - return test.guid ? api.getPlayerLeaderboardEntry(test.guid, test.options) : api.getLeaderboard(test.options); + return test.guid + ? api.getPlayerLeaderboardEntry(test.guid, test.options) + : api.getLeaderboard(test.options); }, measure(test) { recorded = []; @@ -152,8 +240,16 @@ function loadImplementation(handle, ref) { } const report = { - environment: { node: process.version, sqlite: null, platform: platform(), arch: arch(), cpu: cpus()[0]?.model }, - baselineRef: baseline, runs: options.runs, seed: 20260918, + environment: { + node: process.version, + sqlite: null, + platform: platform(), + arch: arch(), + cpu: cpus()[0]?.model + }, + baselineRef: baseline, + runs: options.runs, + seed: 20260918, notes: 'Synthetic file-backed SQLite, one warmup per case, rotating execution order, warm OS cache. Includes SQL preparation and source-record lookup; excludes HTTP and browser rendering. Shared local CPU; timings are not production latency. Schema and indexes are current for both implementations.', scenarios: [] }; @@ -161,77 +257,148 @@ try { for (const definition of definitions) { const handle = new Database(join(temporary, `${definition.name}.sqlite`)); try { - report.environment.sqlite = handle.prepare('SELECT sqlite_version() AS version').get().version; + report.environment.sqlite = handle + .prepare('SELECT sqlite_version() AS version') + .get().version; seed(handle, definition); - const implementations = [...(baseline ? [loadImplementation(handle, baseline)] : []), loadImplementation(handle)]; + const implementations = [ + ...(baseline ? [loadImplementation(handle, baseline)] : []), + loadImplementation(handle) + ]; const cases = [ { name: 'unfiltered', options: {} }, { name: 'kills', options: { metric: 'kills' } }, { name: 'game', options: { game: 't5', metric: 'kills' } }, { name: 'map', options: { game: 't6', map: 't6:map_8', metric: 'kills' } }, { name: 'search', options: { q: '00042', metric: 'kills' } }, - { name: 'deep-page', options: { metric: 'kills', offset: Math.floor(definition.players / 2) } }, + { + name: 'deep-page', + options: { metric: 'kills', offset: Math.floor(definition.players / 2) } + }, { name: 'efficiency', options: { metric: 'headshotRate' } }, { name: 'exact-player', guid: '10000000000000042', options: { metric: 'kills' } }, { name: 'empty-search', options: { q: 'No such player exists' } }, { name: 'past-end', options: { offset: 999999 } }, { name: 'empty-map', options: { map: 't5:missing' } } ]; - const counts = handle.prepare(`SELECT COUNT(*) AS participationRows, - COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`).get(); - const scenario = { ...definition, ...counts, maps: 12, servers: 24, completedShare: 0.9, results: [] }; + const counts = handle + .prepare( + `SELECT COUNT(*) AS participationRows, + COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players` + ) + .get(); + const scenario = { + ...definition, + ...counts, + maps: 12, + servers: 24, + completedShare: 0.9, + results: [] + }; report.scenarios.push(scenario); // Broad parity is separate from timings so every ratio and edge case is // checked without multiplying the larger fixture's timed request matrix. if (baseline && definition.name === 'small') { - const scopes = [{}, { game: 't5' }, { game: 't6', map: 't6:map_8' }, { q: '00042' }, - { q: 'No such player exists' }, { offset: 999999 }, { map: 't5:missing' }]; + const scopes = [ + {}, + { game: 't5' }, + { game: 't6', map: 't6:map_8' }, + { q: '00042' }, + { q: 'No such player exists' }, + { offset: 999999 }, + { map: 't5:missing' } + ]; let checked = 0; for (const metric of implementations[1].metrics) { for (const scope of scopes) { const test = { options: { ...scope, metric } }; - assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test), - `${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`); + assert.deepStrictEqual( + implementations[1].run(test), + implementations[0].run(test), + `${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity` + ); checked++; } - for (const guid of ['10000000000000042', 'benchmark-low-sample', 'benchmark-missing']) { + for (const guid of [ + '10000000000000042', + 'benchmark-low-sample', + 'benchmark-missing' + ]) { const test = { guid, options: { metric } }; - assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test), - `${definition.name}/${metric}/${guid}: exact-player parity`); + assert.deepStrictEqual( + implementations[1].run(test), + implementations[0].run(test), + `${definition.name}/${metric}/${guid}: exact-player parity` + ); checked++; } } scenario.parityCases = checked; - console.log(JSON.stringify({ scenario: definition.name, parityCases: checked, parity: true })); + console.log( + JSON.stringify({ + scenario: definition.name, + parityCases: checked, + parity: true + }) + ); } for (const test of cases) { const expected = implementations[0].run(test); for (const implementation of implementations.slice(1)) { - assert.deepStrictEqual(implementation.run(test), expected, `${definition.name}/${test.name}: baseline parity`); + assert.deepStrictEqual( + implementation.run(test), + expected, + `${definition.name}/${test.name}: baseline parity` + ); } - const samples = Object.fromEntries(implementations.map(item => [item.name, []])); + const samples = Object.fromEntries(implementations.map((item) => [item.name, []])); const latestStatements = {}; for (let index = 0; index < options.runs; index++) { const rotation = index % implementations.length; - for (const implementation of [...implementations.slice(rotation), ...implementations.slice(0, rotation)]) { + for (const implementation of [ + ...implementations.slice(rotation), + ...implementations.slice(0, rotation) + ]) { const measured = implementation.measure(test); samples[implementation.name].push(measured.milliseconds); latestStatements[implementation.name] = measured.statements; } } - const result = { ...test, parity: baseline ? true : null, implementations: implementations.map(implementation => ({ - name: implementation.name, sourceHashes: implementation.hashes, - medianMs: percentile(samples[implementation.name], 0.5), - p95Ms: percentile(samples[implementation.name], 0.95), samplesMs: samples[implementation.name], - statements: latestStatements[implementation.name].map(statement => ({ ...statement, - plan: handle.prepare(`EXPLAIN QUERY PLAN ${statement.sql}`).all(...statement.parameters) + const result = { + ...test, + parity: baseline ? true : null, + implementations: implementations.map((implementation) => ({ + name: implementation.name, + sourceHashes: implementation.hashes, + medianMs: percentile(samples[implementation.name], 0.5), + p95Ms: percentile(samples[implementation.name], 0.95), + samplesMs: samples[implementation.name], + statements: latestStatements[implementation.name].map((statement) => ({ + ...statement, + plan: handle + .prepare(`EXPLAIN QUERY PLAN ${statement.sql}`) + .all(...statement.parameters) + })) })) - })) }; + }; scenario.results.push(result); - console.log(JSON.stringify({ scenario: definition.name, case: test.name, parity: result.parity, - medianMs: Object.fromEntries(result.implementations.map(item => [item.name, Math.round(item.medianMs)])) })); + console.log( + JSON.stringify({ + scenario: definition.name, + case: test.name, + parity: result.parity, + medianMs: Object.fromEntries( + result.implementations.map((item) => [ + item.name, + Math.round(item.medianMs) + ]) + ) + }) + ); } - } finally { handle.close(); } + } finally { + handle.close(); + } } if (options.json) { const target = resolve(options.json); @@ -239,4 +406,6 @@ try { writeFileSync(target, JSON.stringify(report, null, 2)); console.log(`Detailed results: ${target}`); } -} finally { rmSync(temporary, { recursive: true, force: true }); } +} finally { + rmSync(temporary, { recursive: true, force: true }); +} diff --git a/scripts/check-ui-styles.mjs b/scripts/check-ui-styles.mjs index afafae0..b197887 100644 --- a/scripts/check-ui-styles.mjs +++ b/scripts/check-ui-styles.mjs @@ -12,15 +12,21 @@ async function visit(directory) { const source = await readFile(file, 'utf8'); for (const match of source.matchAll(/font-size\s*:\s*[\d.]+px\b/g)) { const line = source.slice(0, match.index).split('\n').length; - failures.push(`${path.relative(process.cwd(), file)}:${line}: use a shared type token (12 px minimum): ${match[0]}`); + failures.push( + `${path.relative(process.cwd(), file)}:${line}: use a shared type token (12 px minimum): ${match[0]}` + ); } } } } await visit(root); const tokens = await readFile(path.join(root, 'lib/ui/styles/tokens.css'), 'utf8'); -for (const match of tokens.matchAll(/--text-(?:xs|sm|base|md|lg|xl|2xl|3xl|input):\s*([\d.]+)px/g)) { +for (const match of tokens.matchAll( + /--text-(?:xs|sm|base|md|lg|xl|2xl|3xl|input):\s*([\d.]+)px/g +)) { if (Number(match[1]) < 12) failures.push(`Type tokens must be at least 12 px: ${match[0]}`); } -if (failures.length) { console.error(failures.join('\n')); process.exitCode = 1; } -else console.log('UI styles: shared font sizes and 12 px minimum verified.'); +if (failures.length) { + console.error(failures.join('\n')); + process.exitCode = 1; +} else console.log('UI styles: shared font sizes and 12 px minimum verified.'); diff --git a/scripts/database-diagnostics.mjs b/scripts/database-diagnostics.mjs index 7b4559a..545a4ac 100644 --- a/scripts/database-diagnostics.mjs +++ b/scripts/database-diagnostics.mjs @@ -4,13 +4,30 @@ import { statSync } from 'node:fs'; const path = process.argv[2] ?? process.env.DB_PATH ?? './data/cherry.db'; const db = new Database(path, { readonly: true, fileMustExist: true }); try { - console.table(['', '-wal', '-shm'].map(suffix => { - try { return { file: path + suffix, bytes: statSync(path + suffix).size }; } - catch (error) { if (error.code !== 'ENOENT') throw error; return { file: path + suffix, bytes: 0 }; } - })); - console.table(db.prepare(`SELECT name, ROUND(SUM(pgsize)/1048576.0, 2) AS mib - FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC LIMIT 20`).all()); - console.table(['page_size', 'page_count', 'freelist_count', 'auto_vacuum'].map(pragma => ({ - pragma, value: db.pragma(pragma, { simple: true }) - }))); -} finally { db.close(); } + console.table( + ['', '-wal', '-shm'].map((suffix) => { + try { + return { file: path + suffix, bytes: statSync(path + suffix).size }; + } catch (error) { + if (error.code !== 'ENOENT') throw error; + return { file: path + suffix, bytes: 0 }; + } + }) + ); + console.table( + db + .prepare( + `SELECT name, ROUND(SUM(pgsize)/1048576.0, 2) AS mib + FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC LIMIT 20` + ) + .all() + ); + console.table( + ['page_size', 'page_count', 'freelist_count', 'auto_vacuum'].map((pragma) => ({ + pragma, + value: db.pragma(pragma, { simple: true }) + })) + ); +} finally { + db.close(); +} diff --git a/scripts/fixture-data.ts b/scripts/fixture-data.ts index 21d9f84..e54d559 100644 --- a/scripts/fixture-data.ts +++ b/scripts/fixture-data.ts @@ -9,7 +9,14 @@ const DAY = 24 * 60 * MINUTE; const GIB = 1024 ** 3; const image = 'fixture.invalid/electric-cherry/preview:local'; -type Scenario = { game: 't5' | 't6'; slug: string; players: number; round: number; minutes: number; state: 'active' | 'idle' | 'stopped' | 'stale' | 'starting' | 'host-offline' }; +type Scenario = { + game: 't5' | 't6'; + slug: string; + players: number; + round: number; + minutes: number; + state: 'active' | 'idle' | 'stopped' | 'stale' | 'starting' | 'host-offline'; +}; const scenarios: Scenario[] = [ { game: 't5', slug: 'theater', players: 4, round: 27, minutes: 76, state: 'active' }, { game: 't5', slug: 'cosmodrome', players: 2, round: 16, minutes: 39, state: 'active' }, @@ -25,211 +32,594 @@ const scenarios: Scenario[] = [ { game: 't6', slug: 'highrise', players: 0, round: 0, minutes: 0, state: 'host-offline' } ]; -export const fixtureServers = scenarios.map((scenario, index) => ({ ...scenario, - name: `${scenario.game}_zombie_${scenario.slug}_1`, host: `fixture-host-${index === 11 ? 3 : index % 2 + 1}`, - instance: `fixture-instance-${index + 1}`, port: 28960 + index, +export const fixtureServers = scenarios.map((scenario, index) => ({ + ...scenario, + name: `${scenario.game}_zombie_${scenario.slug}_1`, + host: `fixture-host-${index === 11 ? 3 : (index % 2) + 1}`, + instance: `fixture-instance-${index + 1}`, + port: 28960 + index, map: zombieMapBySlug(scenario.game, scenario.slug)! })); -const names = ['Jamie Cross', 'Alex', 'Sam', 'Morgan', 'Riley', 'Taylor', 'Jordan', 'Casey', 'Quinn', 'Avery', 'Drew', 'Kai']; +const names = [ + 'Jamie Cross', + 'Alex', + 'Sam', + 'Morgan', + 'Riley', + 'Taylor', + 'Jordan', + 'Casey', + 'Quinn', + 'Avery', + 'Drew', + 'Kai' +]; const iso = (at: number) => new Date(at).toISOString(); -const player = (index: number) => ({ guid: index === 0 ? 'fixture-jamie' : `fixture-player-${index}`, - name: index === 0 ? names[0] : `${names[index % names.length]} ${Math.floor(index / names.length) + 1}` }); -const rosterFor = (server: typeof fixtureServers[number], index: number) => Array.from({ length: server.players }, (_, slot) => ({ ...player(slot === 0 ? 0 : index * 4 + slot), - ping: 29 + slot * 7, kills: (slot + 1) * 137, headshots: (slot + 1) * 41, downs: slot % 2, revives: slot, score: (slot + 1) * 4100 })); -const seenAt = (server: Scenario, now: number) => iso(now - (['stopped', 'host-offline'].includes(server.state) ? 15 * MINUTE : server.state === 'stale' ? 105_000 : 0)); -const containerState = (server: Scenario) => server.state === 'stopped' ? 'exited' : server.state === 'starting' ? 'starting' : 'running'; -const instanceStates = (host: string, now: number) => fixtureServers.filter(server => server.host === host).map(server => ({ - instanceId: server.instance, generation: 1, state: containerState(server), healthy: !['stopped', 'starting'].includes(server.state), - observedAt: iso(now - (host === 'fixture-host-3' ? 10 * MINUTE : 0)) -})); +const player = (index: number) => ({ + guid: index === 0 ? 'fixture-jamie' : `fixture-player-${index}`, + name: + index === 0 + ? names[0] + : `${names[index % names.length]} ${Math.floor(index / names.length) + 1}` +}); +const rosterFor = (server: (typeof fixtureServers)[number], index: number) => + Array.from({ length: server.players }, (_, slot) => ({ + ...player(slot === 0 ? 0 : index * 4 + slot), + ping: 29 + slot * 7, + kills: (slot + 1) * 137, + headshots: (slot + 1) * 41, + downs: slot % 2, + revives: slot, + score: (slot + 1) * 4100 + })); +const seenAt = (server: Scenario, now: number) => + iso( + now - + (['stopped', 'host-offline'].includes(server.state) + ? 15 * MINUTE + : server.state === 'stale' + ? 105_000 + : 0) + ); +const containerState = (server: Scenario) => + server.state === 'stopped' ? 'exited' : server.state === 'starting' ? 'starting' : 'running'; +const instanceStates = (host: string, now: number) => + fixtureServers + .filter((server) => server.host === host) + .map((server) => ({ + instanceId: server.instance, + generation: 1, + state: containerState(server), + healthy: !['stopped', 'starting'].includes(server.state), + observedAt: iso(now - (host === 'fixture-host-3' ? 10 * MINUTE : 0)) + })); /** Uses only a caller-owned, fully migrated database. No host or worker code is imported. */ export function seedFixtureData(handle: Database.Database, now: number): void { if (!Number.isFinite(now)) throw new Error('Invalid fixture time.'); - const occupied = ['servers', 'leaderboard_matches', 'ec_hosts', 'auth_users', 'scripts', 'server_deployments', 'deployments', 'betas', 'game_file_libraries'] - .some(table => handle.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get()); - if (occupied || handle.prepare('SELECT 1 FROM settings WHERE key=?').get(FIXTURE_KEY)) throw new Error('Fixture seeding requires a new database.'); + const occupied = [ + 'servers', + 'leaderboard_matches', + 'ec_hosts', + 'auth_users', + 'scripts', + 'server_deployments', + 'deployments', + 'betas', + 'game_file_libraries' + ].some((table) => handle.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get()); + if (occupied || handle.prepare('SELECT 1 FROM settings WHERE key=?').get(FIXTURE_KEY)) + throw new Error('Fixture seeding requires a new database.'); const insert = (table: string, values: Record) => { const keys = Object.keys(values); - handle.prepare(`INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})`).run(...Object.values(values)); + handle + .prepare( + `INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})` + ) + .run(...Object.values(values)); }; const started = iso(now - 30 * DAY); - const rememberPlayer = handle.prepare(`INSERT INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?) + const rememberPlayer = + handle.prepare(`INSERT INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?) ON CONFLICT(game,guid) DO UPDATE SET first_seen_at=MIN(first_seen_at,excluded.first_seen_at),last_seen_at=MAX(last_seen_at,excluded.last_seen_at)`); handle.transaction(() => { insert('settings', { key: FIXTURE_KEY, value: FIXTURE_VERSION }); // Preview observations must never opt into automatic lifecycle changes. - for (const [key, value] of Object.entries({ maintenance_paused: '1', maintenance_health: '0', maintenance_drift: '0', maintenance_updates: '0', watchdog_enabled: '0', autoscaling_paused: '1' })) { - handle.prepare('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value); + for (const [key, value] of Object.entries({ + maintenance_paused: '1', + maintenance_health: '0', + maintenance_drift: '0', + maintenance_updates: '0', + watchdog_enabled: '0', + autoscaling_paused: '1' + })) { + handle + .prepare( + 'INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value' + ) + .run(key, value); } - for (const [index, name] of ['Chicago · Primary', 'Dallas · Overflow', 'Frankfurt · Europe'].entries()) { + for (const [index, name] of [ + 'Chicago · Primary', + 'Dallas · Overflow', + 'Frankfurt · Europe' + ].entries()) { const id = `fixture-host-${index + 1}`; - insert('ec_hosts', { id, name, hostname: `host-${index + 1}.fixture.invalid`, created_at: started, enrolled_at: started, - last_heartbeat: iso(now - (index === 2 ? 10 * MINUTE : 0)), inventory: JSON.stringify({ protocolVersion: 1, agentVersion: '1.7.0', cpuCount: 16, - cpuUsagePercent: 24 + index * 9, memoryTotal: 32 * GIB, memoryAvailable: (20 - index * 3) * GIB, - storageTotal: 500 * GIB, storageAvailable: 240 * GIB, dockerRunning: true, directoriesWritable: true, - runtimeImages: [image], libraries: [{ id: 'fixture-t5', state: 'ready' }, { id: 'fixture-t6', state: 'ready' }], - capabilities: ['instance.heartbeat-state'], instanceStates: instanceStates(id, now) }) }); + insert('ec_hosts', { + id, + name, + hostname: `host-${index + 1}.fixture.invalid`, + created_at: started, + enrolled_at: started, + last_heartbeat: iso(now - (index === 2 ? 10 * MINUTE : 0)), + inventory: JSON.stringify({ + protocolVersion: 1, + agentVersion: '1.7.0', + cpuCount: 16, + cpuUsagePercent: 24 + index * 9, + memoryTotal: 32 * GIB, + memoryAvailable: (20 - index * 3) * GIB, + storageTotal: 500 * GIB, + storageAvailable: 240 * GIB, + dockerRunning: true, + directoriesWritable: true, + runtimeImages: [image], + libraries: [ + { id: 'fixture-t5', state: 'ready' }, + { id: 'fixture-t6', state: 'ready' } + ], + capabilities: ['instance.heartbeat-state'], + instanceStates: instanceStates(id, now) + }) + }); // Deliberately not a token hash; no bearer token can authenticate as these agents. - insert('ec_host_credentials', { id: `fixture-credential-${index + 1}`, host_id: id, token_hash: `invalid-fixture-${index + 1}`, created_at: started }); + insert('ec_host_credentials', { + id: `fixture-credential-${index + 1}`, + host_id: id, + token_hash: `invalid-fixture-${index + 1}`, + created_at: started + }); for (let minute = 0; minute < 7 * 1440; minute += 15) { - insert('ec_host_resource_samples', { host_id: id, sampled_at: iso(now - minute * MINUTE), cpu_count: 16, - cpu_usage_percent: 28 + index * 6 + 10 * Math.sin(minute / 95), memory_total: 32 * GIB, - memory_available: Math.round((18 - index * 2 + Math.sin(minute / 110)) * GIB) }); + insert('ec_host_resource_samples', { + host_id: id, + sampled_at: iso(now - minute * MINUTE), + cpu_count: 16, + cpu_usage_percent: 28 + index * 6 + 10 * Math.sin(minute / 95), + memory_total: 32 * GIB, + memory_available: Math.round((18 - index * 2 + Math.sin(minute / 110)) * GIB) + }); } } let liveId = 495; for (const [index, server] of fixtureServers.entries()) { const beginning = iso(now - server.minutes * MINUTE - 12_000); const roster = rosterFor(server, index); - insert('servers', { name: server.name, game: server.game, map: server.map.canonical, map_slug: server.slug, game_mode: 'zombie', - game_port: server.port, game_id: server.game === 't5' ? 't5sp' : 't6zm', image, - created_by: 'development fixture', created_at: started, last_players: server.players, last_map: server.map.canonical, last_gametype: 'zclassic', - last_roster: JSON.stringify(roster), last_seen_at: seenAt(server, now), - last_checked_at: iso(now), last_error: server.state === 'stale' ? 'timeout: no reply from game server' : null, + insert('servers', { + name: server.name, + game: server.game, + map: server.map.canonical, + map_slug: server.slug, + game_mode: 'zombie', + game_port: server.port, + game_id: server.game === 't5' ? 't5sp' : 't6zm', + image, + created_by: 'development fixture', + created_at: started, + last_players: server.players, + last_map: server.map.canonical, + last_gametype: 'zclassic', + last_roster: JSON.stringify(roster), + last_seen_at: seenAt(server, now), + last_checked_at: iso(now), + last_error: server.state === 'stale' ? 'timeout: no reply from game server' : null, rcon_failures: server.state === 'stale' ? 2 : 0, container_state: containerState(server), - container_checked_at: iso(now), desired_state: server.state === 'stopped' ? 'stopped' : 'running', - active_since: server.players ? beginning : null, inactive_since: server.players ? null : iso(now - 5 * MINUTE), - game_started_at: server.players ? beginning : null }); - insert('native_instances', { id: server.instance, server: server.name, host_id: server.host, - library_id: `fixture-${server.game}`, image, port: server.port, created_at: started }); - insert('plutonium_versions', { server: server.name, revision: 4878, checked_at: iso(now), error: null }); + container_checked_at: iso(now), + desired_state: server.state === 'stopped' ? 'stopped' : 'running', + active_since: server.players ? beginning : null, + inactive_since: server.players ? null : iso(now - 5 * MINUTE), + game_started_at: server.players ? beginning : null + }); + insert('native_instances', { + id: server.instance, + server: server.name, + host_id: server.host, + library_id: `fixture-${server.game}`, + image, + port: server.port, + created_at: started + }); + insert('plutonium_versions', { + server: server.name, + revision: 4878, + checked_at: iso(now), + error: null + }); if (server.state === 'active' || server.state === 'idle') { const matchId = server.players ? liveId++ : null; - const payload = { server: server.name, map: server.map.canonical, event: 'heartbeat', round: server.round, players: roster, receivedAt: iso(now), - timing: { matchId, players: Object.fromEntries(roster.map(p => [p.guid, { serverJoinedAt: beginning, matchJoinedAt: beginning, roundStarted: 1 }])) } }; - insert('server_telemetry', { server: server.name, round: server.round, players: server.players, received_at: iso(now), payload: JSON.stringify(payload) }); + const payload = { + server: server.name, + map: server.map.canonical, + event: 'heartbeat', + round: server.round, + players: roster, + receivedAt: iso(now), + timing: { + matchId, + players: Object.fromEntries( + roster.map((p) => [ + p.guid, + { + serverJoinedAt: beginning, + matchJoinedAt: beginning, + roundStarted: 1 + } + ]) + ) + } + }; + insert('server_telemetry', { + server: server.name, + round: server.round, + players: server.players, + received_at: iso(now), + payload: JSON.stringify(payload) + }); if (matchId !== null) { - insert('leaderboard_matches', { id: matchId, server: server.name, game: server.game, map: server.map.canonical, - map_key: server.slug, map_name: server.map.friendly, started_at: beginning, last_received_at: iso(now), status: 'active', - start_observed: 1, start_players: server.players, highest_round: server.round, last_event: 'heartbeat', last_fingerprint: `fixture-live-${index}` }); + insert('leaderboard_matches', { + id: matchId, + server: server.name, + game: server.game, + map: server.map.canonical, + map_key: server.slug, + map_name: server.map.friendly, + started_at: beginning, + last_received_at: iso(now), + status: 'active', + start_observed: 1, + start_players: server.players, + highest_round: server.round, + last_event: 'heartbeat', + last_fingerprint: `fixture-live-${index}` + }); for (const p of roster) { rememberPlayer.run(server.game, p.guid, p.name, beginning, iso(now)); - insert('leaderboard_match_players', { match_id: matchId, guid: p.guid, highest_round: server.round, rounds_survived: server.round, - kills: p.kills, headshots: p.headshots, downs: p.downs, revives: p.revives, best_score: p.score, - last_round: server.round, present: 1, has_stats: 1, completed: 0, first_seen_at: beginning, last_seen_at: iso(now), - first_observed_round: 1, present_at_start: 1 }); + insert('leaderboard_match_players', { + match_id: matchId, + guid: p.guid, + highest_round: server.round, + rounds_survived: server.round, + kills: p.kills, + headshots: p.headshots, + downs: p.downs, + revives: p.revives, + best_score: p.score, + last_round: server.round, + present: 1, + has_stats: 1, + completed: 0, + first_seen_at: beginning, + last_seen_at: iso(now), + first_observed_round: 1, + present_at_start: 1 + }); } } } for (const phase of ['startup', 'running']) { - for (let minute = 0; minute < 60; minute++) insert('ec_instance_resource_samples', { - host_id: server.host, instance_id: server.instance, generation: 1, phase, bucket: iso(now - minute * MINUTE), - cpu_cores: phase === 'startup' ? 0.7 + index / 40 : 0.25 + index / 50, - memory_bytes: Math.round((phase === 'startup' ? 1.1 : 0.7) * GIB) }); + for (let minute = 0; minute < 60; minute++) + insert('ec_instance_resource_samples', { + host_id: server.host, + instance_id: server.instance, + generation: 1, + phase, + bucket: iso(now - minute * MINUTE), + cpu_cores: phase === 'startup' ? 0.7 + index / 40 : 0.25 + index / 50, + memory_bytes: Math.round((phase === 'startup' ? 1.1 : 0.7) * GIB) + }); } } for (let index = 0; index < 494; index++) { // A large Kino duo cohort exercises ranking and profile pagination. const server = fixtureServers[index < 64 ? 0 : index % fixtureServers.length]; - const squad = index < 64 ? 2 : Math.floor(index / 12) % 4 + 1; + const squad = index < 64 ? 2 : (Math.floor(index / 12) % 4) + 1; const unknownStart = index >= 64 && index % 37 === 0; const interrupted = index >= 64 && index % 31 === 0; const end = now - (180 + index * 37) * MINUTE; - const beginning = end - (30 + index % 100) * MINUTE; - const round = 10 + index * 7 % 56; - insert('leaderboard_matches', { id: index + 1, server: server.name, game: server.game, map: server.map.canonical, - map_key: server.slug, map_name: server.map.friendly, started_at: iso(beginning), ended_at: iso(end), last_received_at: iso(end), - status: interrupted ? 'interrupted' : 'completed', start_observed: unknownStart ? 0 : 1, start_players: unknownStart ? null : squad, - highest_round: round, last_event: interrupted ? 'interrupted' : 'game_end', last_fingerprint: `fixture-match-${index + 1}` }); + const beginning = end - (30 + (index % 100)) * MINUTE; + const round = 10 + ((index * 7) % 56); + insert('leaderboard_matches', { + id: index + 1, + server: server.name, + game: server.game, + map: server.map.canonical, + map_key: server.slug, + map_name: server.map.friendly, + started_at: iso(beginning), + ended_at: iso(end), + last_received_at: iso(end), + status: interrupted ? 'interrupted' : 'completed', + start_observed: unknownStart ? 0 : 1, + start_players: unknownStart ? null : squad, + highest_round: round, + last_event: interrupted ? 'interrupted' : 'game_end', + last_fingerprint: `fixture-match-${index + 1}` + }); for (let slot = 0; slot < squad; slot++) { - const p = player(slot === 0 ? 0 : (index + slot * 19) % 80 + 1); + const p = player(slot === 0 ? 0 : ((index + slot * 19) % 80) + 1); rememberPlayer.run(server.game, p.guid, p.name, iso(beginning), iso(end)); - insert('leaderboard_match_players', { match_id: index + 1, guid: p.guid, highest_round: round, rounds_survived: round - (unknownStart ? 4 : 0), - kills: round * 35 + slot * 71, headshots: round * 9, downs: index % 4, revives: slot + index % 3, - best_score: round * 2100 + slot * 1000, last_round: round, present: slot > 0 && index % 9 === 0 ? 0 : 1, has_stats: 1, - completed: !interrupted && !(slot > 0 && index % 9 === 0) ? 1 : 0, first_seen_at: iso(beginning), last_seen_at: iso(end), - first_observed_round: unknownStart ? 5 : 1, present_at_start: unknownStart ? null : 1 }); + insert('leaderboard_match_players', { + match_id: index + 1, + guid: p.guid, + highest_round: round, + rounds_survived: round - (unknownStart ? 4 : 0), + kills: round * 35 + slot * 71, + headshots: round * 9, + downs: index % 4, + revives: slot + (index % 3), + best_score: round * 2100 + slot * 1000, + last_round: round, + present: slot > 0 && index % 9 === 0 ? 0 : 1, + has_stats: 1, + completed: !interrupted && !(slot > 0 && index % 9 === 0) ? 1 : 0, + first_seen_at: iso(beginning), + last_seen_at: iso(end), + first_observed_round: unknownStart ? 5 : 1, + present_at_start: unknownStart ? null : 1 + }); } } - const metric = handle.prepare('INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,?,?,?,?,4)'); + const metric = handle.prepare( + 'INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,?,?,?,?,4)' + ); for (let minute = 0; minute < 7 * 1440; minute++) { for (const [index, server] of fixtureServers.entries()) { const count = Math.floor((Math.sin(minute / 90 + index) + 1) * 2.49); - metric.run(now - minute * MINUTE, server.name, server.game, server.map.friendly, - minute % 720 < 8 && index === 5 ? null : count); + metric.run( + now - minute * MINUTE, + server.name, + server.game, + server.map.friendly, + minute % 720 < 8 && index === 5 ? null : count + ); } } - for (const scope of ['*', ...[1, 2, 3].map(index => `fixture-host-${index}`)]) for (const phase of ['startup', 'running']) { - insert('ec_resource_allowances', { scope, phase, cpu_cores: phase === 'startup' ? 1 : 0.5, - memory_bytes: Math.round((phase === 'startup' ? 1.2 : 0.8) * GIB), samples: 240, - window_start: iso(now - DAY), observed_at: iso(now), computed_at: iso(now), usage_margin: 1 }); - } + for (const scope of ['*', ...[1, 2, 3].map((index) => `fixture-host-${index}`)]) + for (const phase of ['startup', 'running']) { + insert('ec_resource_allowances', { + scope, + phase, + cpu_cores: phase === 'startup' ? 1 : 0.5, + memory_bytes: Math.round((phase === 'startup' ? 1.2 : 0.8) * GIB), + samples: 240, + window_start: iso(now - DAY), + observed_at: iso(now), + computed_at: iso(now), + usage_margin: 1 + }); + } // Real artifact blobs, so details and downloads work without game files. for (const [index, kind] of ['script', 'baseline', 'config'].entries()) { - const content = Buffer.from(kind === 'script' ? '// Synthetic UI fixture; no gameplay behavior.\n' : '// Synthetic UI fixture configuration.\nset sv_maxclients 4\n'); + const content = Buffer.from( + kind === 'script' + ? '// Synthetic UI fixture; no gameplay behavior.\n' + : '// Synthetic UI fixture configuration.\nset sv_maxclients 4\n' + ); const hash = createHash('sha256').update(content).digest('hex'); - handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(hash, content.length, content); - insert('scripts', { id: index + 1, name: ['Fixture telemetry', 'Fixture Black Ops I baseline', 'Fixture lobby settings'][index], - target_rel_path: index === 0 ? 'fixture.gsc' : index === 1 ? 'dedicated_sp.cfg' : 'fixture.cfg', - scope_type: 'game', scope_value: 't5', current_version_id: index + 1, kind, created_at: started }); - insert('script_versions', { id: index + 1, script_id: index + 1, version: 1, sha256: hash, content_hash: hash, - size: content.length, notes: 'Synthetic preview content.', state: 'tested', kind, uploaded_at: started }); + handle + .prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)') + .run(hash, content.length, content); + insert('scripts', { + id: index + 1, + name: [ + 'Fixture telemetry', + 'Fixture Black Ops I baseline', + 'Fixture lobby settings' + ][index], + target_rel_path: + index === 0 ? 'fixture.gsc' : index === 1 ? 'dedicated_sp.cfg' : 'fixture.cfg', + scope_type: 'game', + scope_value: 't5', + current_version_id: index + 1, + kind, + created_at: started + }); + insert('script_versions', { + id: index + 1, + script_id: index + 1, + version: 1, + sha256: hash, + content_hash: hash, + size: content.length, + notes: 'Synthetic preview content.', + state: 'tested', + kind, + uploaded_at: started + }); } for (let index = 0; index < 36; index++) { const failed = index % 7 === 0; const server = fixtureServers[failed ? 5 : index % fixtureServers.length]; const id = `fixture-operation-${index + 1}`; const state = failed ? 'failed' : index % 11 === 0 ? 'cancelled' : 'done'; - insert('server_deployments', { id, server: server.name, action: ['restart', 'start', 'rebuild', 'stop'][index % 4], state, - stage: failed ? 'Verification failed' : state === 'done' ? 'Completed' : 'Cancelled before dispatch', - started_at: iso(now - (60 + index * 2) * MINUTE), finished_at: iso(now - (59 + index * 2) * MINUTE), triggered_by: 'Fixture operator', - error: failed ? 'Synthetic verification timeout. No host command was sent.' : null, warnings: '[]' }); - insert('native_provisioning', { deployment_id: id, instance_id: server.instance, generation: 1 }); - insert('server_deployment_events', { deployment_id: id, created_at: iso(now - (59 + index * 2) * MINUTE), message: 'Synthetic retained history; no command was sent.' }); + insert('server_deployments', { + id, + server: server.name, + action: ['restart', 'start', 'rebuild', 'stop'][index % 4], + state, + stage: failed + ? 'Verification failed' + : state === 'done' + ? 'Completed' + : 'Cancelled before dispatch', + started_at: iso(now - (60 + index * 2) * MINUTE), + finished_at: iso(now - (59 + index * 2) * MINUTE), + triggered_by: 'Fixture operator', + error: failed ? 'Synthetic verification timeout. No host command was sent.' : null, + warnings: '[]' + }); + insert('native_provisioning', { + deployment_id: id, + instance_id: server.instance, + generation: 1 + }); + insert('server_deployment_events', { + deployment_id: id, + created_at: iso(now - (59 + index * 2) * MINUTE), + message: 'Synthetic retained history; no command was sent.' + }); } for (let index = 0; index < 8; index++) { const server = fixtureServers[index % fixtureServers.length]; - insert('ec_host_operations', { id: `fixture-host-command-${index + 1}`, host_id: server.host, instance_id: server.instance, - server_name: server.name, kind: index % 2 ? 'instance.inspect' : 'instance.restart', payload: '{}', - state: index % 3 ? 'completed' : 'failed', result: '{}', error: index % 3 ? null : 'Synthetic command failure.', - created_at: iso(now - (90 + index * 3) * MINUTE), updated_at: iso(now - (89 + index * 3) * MINUTE), source: 'Fixture operator' }); + insert('ec_host_operations', { + id: `fixture-host-command-${index + 1}`, + host_id: server.host, + instance_id: server.instance, + server_name: server.name, + kind: index % 2 ? 'instance.inspect' : 'instance.restart', + payload: '{}', + state: index % 3 ? 'completed' : 'failed', + result: '{}', + error: index % 3 ? null : 'Synthetic command failure.', + created_at: iso(now - (90 + index * 3) * MINUTE), + updated_at: iso(now - (89 + index * 3) * MINUTE), + source: 'Fixture operator' + }); } for (let index = 0; index < 3; index++) { const server = fixtureServers[index === 0 ? 1 : index === 1 ? 5 : index]; - insert('maintenance_jobs', { id: index + 1, server: server.name, state: index === 1 ? 'failed' : 'done', target_revision: 4878, - requested_at: iso(now - (120 + index * 4) * MINUTE), finished_at: iso(now - (119 + index * 4) * MINUTE), - detail: index === 1 ? 'Synthetic update could not be verified.' : 'Synthetic scheduled restart completed.', kind: 'schedule', origin: 'manual', - provisioning_id: index === 0 ? 'fixture-operation-2' : null }); - insert('watchdog_incidents', { id: `fixture-recovery-${index + 1}`, server: server.name, kind: 'health', state: 'resolved', - summary: 'Synthetic recovery finding resolved.', details: '{}', first_seen_at: iso(now - (130 + index * 4) * MINUTE), - last_seen_at: iso(now - (129 + index * 4) * MINUTE), finished_at: iso(now - (129 + index * 4) * MINUTE) }); - insert('deployments', { id: `fixture-campaign-${index + 1}`, action: 'drift', started_at: iso(now - (140 + index * 4) * MINUTE), - finished_at: iso(now - (139 + index * 4) * MINUTE), triggered_by: 'Fixture operator' }); - insert('deployment_servers', { deployment_id: `fixture-campaign-${index + 1}`, server: server.name, files_changed: 1, restart_state: 'done', players: 0 }); + insert('maintenance_jobs', { + id: index + 1, + server: server.name, + state: index === 1 ? 'failed' : 'done', + target_revision: 4878, + requested_at: iso(now - (120 + index * 4) * MINUTE), + finished_at: iso(now - (119 + index * 4) * MINUTE), + detail: + index === 1 + ? 'Synthetic update could not be verified.' + : 'Synthetic scheduled restart completed.', + kind: 'schedule', + origin: 'manual', + provisioning_id: index === 0 ? 'fixture-operation-2' : null + }); + insert('watchdog_incidents', { + id: `fixture-recovery-${index + 1}`, + server: server.name, + kind: 'health', + state: 'resolved', + summary: 'Synthetic recovery finding resolved.', + details: '{}', + first_seen_at: iso(now - (130 + index * 4) * MINUTE), + last_seen_at: iso(now - (129 + index * 4) * MINUTE), + finished_at: iso(now - (129 + index * 4) * MINUTE) + }); + insert('deployments', { + id: `fixture-campaign-${index + 1}`, + action: 'drift', + started_at: iso(now - (140 + index * 4) * MINUTE), + finished_at: iso(now - (139 + index * 4) * MINUTE), + triggered_by: 'Fixture operator' + }); + insert('deployment_servers', { + deployment_id: `fixture-campaign-${index + 1}`, + server: server.name, + files_changed: 1, + restart_state: 'done', + players: 0 + }); } for (let index = 0; index < 2; index++) { - insert('betas', { id: index + 1, script_id: 1, script_version_id: 1, trigger_kind: 'ui', state: index ? 'rolled-back' : 'promoted', - opened_at: iso(now - (150 + index * 3) * MINUTE), closed_at: iso(now - (149 + index * 3) * MINUTE), notes: 'Synthetic completed beta.' }); - insert('beta_scripts', { beta_id: index + 1, script_id: 1, script_version_id: 1, kind: 'script' }); - insert('beta_servers', { beta_id: index + 1, server: fixtureServers[index].name, had_original: 0, stage: 'ready', updated_at: iso(now - (149 + index * 3) * MINUTE) }); + insert('betas', { + id: index + 1, + script_id: 1, + script_version_id: 1, + trigger_kind: 'ui', + state: index ? 'rolled-back' : 'promoted', + opened_at: iso(now - (150 + index * 3) * MINUTE), + closed_at: iso(now - (149 + index * 3) * MINUTE), + notes: 'Synthetic completed beta.' + }); + insert('beta_scripts', { + beta_id: index + 1, + script_id: 1, + script_version_id: 1, + kind: 'script' + }); + insert('beta_servers', { + beta_id: index + 1, + server: fixtureServers[index].name, + had_original: 0, + stage: 'ready', + updated_at: iso(now - (149 + index * 3) * MINUTE) + }); } - insert('provisioning_sources', { operation_id: 'fixture-operation-2', kind: 'maintenance', source_id: '1', server: fixtureServers[1].name }); - insert('provisioning_sources', { operation_id: 'fixture-operation-3', kind: 'autoscaling', source_id: 'fixture-paused-plan', server: fixtureServers[2].name }); + insert('provisioning_sources', { + operation_id: 'fixture-operation-2', + kind: 'maintenance', + source_id: '1', + server: fixtureServers[1].name + }); + insert('provisioning_sources', { + operation_id: 'fixture-operation-3', + kind: 'autoscaling', + source_id: 'fixture-paused-plan', + server: fixtureServers[2].name + }); })(); } /** Refreshes existing synthetic observations only; never queues work or writes real files. */ export function refreshFixtureData(handle: Database.Database, now: number): void { - const marker = handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY) as { value: string } | undefined; - if (marker?.value !== FIXTURE_VERSION) throw new Error('Refusing to refresh an unmarked database.'); + const marker = handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY) as + { value: string } | undefined; + if (marker?.value !== FIXTURE_VERSION) + throw new Error('Refusing to refresh an unmarked database.'); if (!Number.isFinite(now)) throw new Error('Invalid fixture time.'); handle.transaction(() => { for (const [index, server] of fixtureServers.entries()) { - handle.prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,container_checked_at=?,container_state=?, - last_players=?,last_map=?,last_roster=?,last_error=?,rcon_failures=?,last_reply_snippet=NULL WHERE name=?`) - .run(seenAt(server, now), iso(now), iso(now), containerState(server), server.players, server.map.canonical, - JSON.stringify(rosterFor(server, index)), server.state === 'stale' ? 'timeout: no reply from game server' : null, - server.state === 'stale' ? 2 : 0, server.name); - handle.prepare(`UPDATE server_telemetry SET received_at=?,payload=json_set(payload,'$.receivedAt',?) WHERE server=?`).run(iso(now), iso(now), server.name); - handle.prepare("UPDATE leaderboard_matches SET last_received_at=? WHERE server=? AND status='active' AND id BETWEEN 495 AND 500").run(iso(now), server.name); - handle.prepare('UPDATE plutonium_versions SET checked_at=? WHERE server=?').run(iso(now), server.name); + handle + .prepare( + `UPDATE servers SET last_seen_at=?,last_checked_at=?,container_checked_at=?,container_state=?, + last_players=?,last_map=?,last_roster=?,last_error=?,rcon_failures=?,last_reply_snippet=NULL WHERE name=?` + ) + .run( + seenAt(server, now), + iso(now), + iso(now), + containerState(server), + server.players, + server.map.canonical, + JSON.stringify(rosterFor(server, index)), + server.state === 'stale' ? 'timeout: no reply from game server' : null, + server.state === 'stale' ? 2 : 0, + server.name + ); + handle + .prepare( + `UPDATE server_telemetry SET received_at=?,payload=json_set(payload,'$.receivedAt',?) WHERE server=?` + ) + .run(iso(now), iso(now), server.name); + handle + .prepare( + "UPDATE leaderboard_matches SET last_received_at=? WHERE server=? AND status='active' AND id BETWEEN 495 AND 500" + ) + .run(iso(now), server.name); + handle + .prepare('UPDATE plutonium_versions SET checked_at=? WHERE server=?') + .run(iso(now), server.name); } for (let index = 1; index <= 3; index++) { const id = `fixture-host-${index}`; - handle.prepare("UPDATE ec_hosts SET last_heartbeat=?,inventory=json_set(inventory,'$.instanceStates',json(?)) WHERE id=?") - .run(iso(now - (index === 3 ? 10 * MINUTE : 0)), JSON.stringify(instanceStates(id, now)), id); + handle + .prepare( + "UPDATE ec_hosts SET last_heartbeat=?,inventory=json_set(inventory,'$.instanceStates',json(?)) WHERE id=?" + ) + .run( + iso(now - (index === 3 ? 10 * MINUTE : 0)), + JSON.stringify(instanceStates(id, now)), + id + ); } })(); } diff --git a/scripts/fixture-options.ts b/scripts/fixture-options.ts index 3805a85..38304d9 100644 --- a/scripts/fixture-options.ts +++ b/scripts/fixture-options.ts @@ -1,29 +1,49 @@ import { mkdir, open, realpath, rm } from 'node:fs/promises'; import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; -export interface FixtureOptions { dbPath: string; now: number; watch: boolean } +export interface FixtureOptions { + dbPath: string; + now: number; + watch: boolean; +} export function parseFixtureOptions(args: string[], clock = Date.now()): FixtureOptions { - let dbPath = '', now = clock, watch = false, explicitTime = false; + let dbPath = '', + now = clock, + watch = false, + explicitTime = false; const seen = new Set(); for (let index = 0; index < args.length; index++) { const option = args[index]; if (seen.has(option)) throw new Error(`Duplicate option: ${option}`); seen.add(option); - if (option === '--watch') { watch = true; continue; } + if (option === '--watch') { + watch = true; + continue; + } if (option !== '--db' && option !== '--now') throw new Error(`Unknown option: ${option}`); const value = args[++index]; if (!value || value.startsWith('--')) throw new Error(`Missing value for ${option}.`); if (option === '--db') dbPath = value; else { - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(value) - || !Number.isFinite(Date.parse(value))) throw new Error('--now must be an ISO timestamp with a timezone.'); - now = Date.parse(value); explicitTime = true; + if ( + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test( + value + ) || + !Number.isFinite(Date.parse(value)) + ) + throw new Error('--now must be an ISO timestamp with a timezone.'); + now = Date.parse(value); + explicitTime = true; } } - if (!dbPath || !isAbsolute(dbPath)) throw new Error('Provide --db with an absolute path inside a new fixture directory. DB_PATH is never used.'); + if (!dbPath || !isAbsolute(dbPath)) + throw new Error( + 'Provide --db with an absolute path inside a new fixture directory. DB_PATH is never used.' + ); if (!Number.isFinite(now)) throw new Error('Invalid fixture clock.'); - if (watch && explicitTime) throw new Error('Use --watch for live time or --now for a fixed snapshot, separately.'); + if (watch && explicitTime) + throw new Error('Use --watch for live time or --now for a fixed snapshot, separately.'); return { dbPath: resolve(dbPath), now, watch }; } @@ -36,9 +56,14 @@ export async function reserveFixture(dbPath: string): Promise<{ root: string; db // but never follow a pre-existing fixture directory or database symlink. const parent = await realpath(dirname(requestedRoot)); const root = join(parent, basename(requestedRoot)); - try { await mkdir(root, { mode: 0o700 }); } - catch (cause) { - if ((cause as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('Fixture directory already exists. Choose a new directory; existing data is never overwritten.'); + try { + await mkdir(root, { mode: 0o700 }); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === 'EEXIST') + throw new Error( + 'Fixture directory already exists. Choose a new directory; existing data is never overwritten.', + { cause } + ); throw cause; } const target = join(root, basename(requested)); diff --git a/scripts/reset-admin.mjs b/scripts/reset-admin.mjs index e346656..c511745 100644 --- a/scripts/reset-admin.mjs +++ b/scripts/reset-admin.mjs @@ -18,38 +18,67 @@ function secret(prompt) { process.stdout.write('\n'); }; const listener = (text, key) => { - if (key?.ctrl && key.name === 'c') { cleanup(); reject(new Error('Cancelled.')); } - else if (key?.name === 'return' || key?.name === 'enter') { cleanup(); resolve(value); } - else if (key?.name === 'backspace') value = [...value].slice(0,-1).join(''); - else if (text && !key?.ctrl && !key?.meta && !/[\x00-\x1f\x7f]/.test(text) && [...value].length < 128) value += text; + if (key?.ctrl && key.name === 'c') { + cleanup(); + reject(new Error('Cancelled.')); + } else if (key?.name === 'return' || key?.name === 'enter') { + cleanup(); + resolve(value); + } else if (key?.name === 'backspace') value = [...value].slice(0, -1).join(''); + else if ( + text && + !key?.ctrl && + !key?.meta && + // eslint-disable-next-line no-control-regex -- Reject terminal control characters while collecting password input. + !/[\x00-\x1f\x7f]/.test(text) && + [...value].length < 128 + ) + value += text; }; - process.stdin.on('keypress',listener); + process.stdin.on('keypress', listener); }); } try { const username = process.argv[2]?.toLowerCase(); - if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username)) throw new Error('Usage: node scripts/reset-admin.mjs '); - if (!process.stdin.isTTY) throw new Error('Run this command in an interactive terminal. Passwords are never accepted as command arguments.'); - const database = new Database(process.env.DB_PATH || './data/cherry.db',{fileMustExist:true}); + if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username)) + throw new Error('Usage: node scripts/reset-admin.mjs '); + if (!process.stdin.isTTY) + throw new Error( + 'Run this command in an interactive terminal. Passwords are never accepted as command arguments.' + ); + const database = new Database(process.env.DB_PATH || './data/cherry.db', { + fileMustExist: true + }); database.pragma('busy_timeout=5000'); - const user = database.prepare("SELECT id FROM auth_users WHERE username=? AND role='admin' AND disabled=0").get(username); + const user = database + .prepare("SELECT id FROM auth_users WHERE username=? AND role='admin' AND disabled=0") + .get(username); if (!user) throw new Error('An enabled administrator with that username was not found.'); const password = await secret('New administrator password (8–128 characters): '); const confirmation = await secret('Repeat password: '); if (password !== confirmation) throw new Error('Passwords did not match.'); const passwordHash = await hashPassword(password); - database.transaction(() => { - const changed = database.prepare("UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0") - .run(passwordHash,Date.now(),user.id); - if (changed.changes !== 1) throw new Error('Administrator changed during recovery. Try again.'); - database.prepare('DELETE FROM auth_sessions WHERE user_id=?').run(user.id); - database.prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)') - .run('password.recovered',user.id,'{"source":"local-cli"}',Date.now()); - }).immediate(); + database + .transaction(() => { + const changed = database + .prepare( + "UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0" + ) + .run(passwordHash, Date.now(), user.id); + if (changed.changes !== 1) + throw new Error('Administrator changed during recovery. Try again.'); + database.prepare('DELETE FROM auth_sessions WHERE user_id=?').run(user.id); + database + .prepare( + 'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)' + ) + .run('password.recovered', user.id, '{"source":"local-cli"}', Date.now()); + }) + .immediate(); database.close(); console.log('Administrator password reset. Existing sessions have been revoked.'); } catch (cause) { console.error(cause instanceof Error ? cause.message : 'Administrator recovery failed.'); - process.exitCode=1; + process.exitCode = 1; } diff --git a/scripts/seed-fixture.ts b/scripts/seed-fixture.ts index e40b4b3..ebdf1c2 100644 --- a/scripts/seed-fixture.ts +++ b/scripts/seed-fixture.ts @@ -5,7 +5,13 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createServer, type ViteDevServer } from 'vite'; import type Database from 'better-sqlite3'; -import { FIXTURE_KEY, FIXTURE_VERSION, fixtureServers, refreshFixtureData, seedFixtureData } from './fixture-data.js'; +import { + FIXTURE_KEY, + FIXTURE_VERSION, + fixtureServers, + refreshFixtureData, + seedFixtureData +} from './fixture-data.js'; import { parseFixtureOptions, reserveFixture } from './fixture-options.js'; const help = `Create an isolated synthetic UI preview database. @@ -22,7 +28,10 @@ No web server, worker, game agent or network request is started by this command. async function main() { const args = process.argv.slice(2); - if (args.length === 1 && ['--help', '-h'].includes(args[0])) { console.log(help); return; } + if (args.length === 1 && ['--help', '-h'].includes(args[0])) { + console.log(help); + return; + } const options = parseFixtureOptions(args); process.umask(0o077); const destination = await reserveFixture(options.dbPath); @@ -30,70 +39,154 @@ async function main() { let closeDb: (() => void) | undefined; let complete = false; try { - const paths = { DB_PATH: destination.dbPath, SERVERS_ROOT: join(destination.root, 't5'), T6_SERVERS_ROOT: join(destination.root, 't6'), - BETA_ARCHIVE_DIR: join(destination.root, 'beta-archive'), GAME_FILES_ROOT: join(destination.root, 'game-files'), SQLITE_TMPDIR: join(destination.root, 'sqlite-tmp') }; + const paths = { + DB_PATH: destination.dbPath, + SERVERS_ROOT: join(destination.root, 't5'), + T6_SERVERS_ROOT: join(destination.root, 't6'), + BETA_ARCHIVE_DIR: join(destination.root, 'beta-archive'), + GAME_FILES_ROOT: join(destination.root, 'game-files'), + SQLITE_TMPDIR: join(destination.root, 'sqlite-tmp') + }; for (const [key, value] of Object.entries(paths)) { process.env[key] = value; if (key !== 'DB_PATH') await mkdir(value, { mode: 0o700 }); } - Object.assign(process.env, { RCON_PASSWORD: '', RCON_HOST: '127.0.0.1', EC_AUTH_ALLOW_HTTP: 'true', - DEPLOY_KEEP_DAYS: '7', DEPLOYMENT_RESTART_CONCURRENCY: '1', DEPLOYMENT_CREATE_CONCURRENCY: '1', - EC_SQLITE_MMAP_MB: '0', PLUTAINER_IMAGE: 'fixture.invalid/electric-cherry/preview:local' }); + Object.assign(process.env, { + RCON_PASSWORD: '', + RCON_HOST: '127.0.0.1', + EC_AUTH_ALLOW_HTTP: 'true', + DEPLOY_KEEP_DAYS: '7', + DEPLOYMENT_RESTART_CONCURRENCY: '1', + DEPLOYMENT_CREATE_CONCURRENCY: '1', + EC_SQLITE_MMAP_MB: '0', + PLUTAINER_IMAGE: 'fixture.invalid/electric-cherry/preview:local' + }); const repository = resolve(dirname(fileURLToPath(import.meta.url)), '..'); - vite = await createServer({ root: repository, cacheDir: join(destination.root, 'vite-cache'), configFile: false, envFile: false, appType: 'custom', + vite = await createServer({ + root: repository, + cacheDir: join(destination.root, 'vite-cache'), + configFile: false, + envFile: false, + appType: 'custom', // Vite 5 otherwise opens its HMR WebSocket port even with hmr:false. // Attach it to an HTTP server that is deliberately never listened on. - server: { middlewareMode: true, hmr: { server: createHttpServer() }, watch: null }, optimizeDeps: { noDiscovery: true }, - resolve: { alias: { $lib: join(repository, 'src/lib'), $ui: join(repository, 'src/lib/ui') } }, - logLevel: 'error' }); + server: { middlewareMode: true, hmr: { server: createHttpServer() }, watch: null }, + optimizeDeps: { noDiscovery: true }, + resolve: { + alias: { $lib: join(repository, 'src/lib'), $ui: join(repository, 'src/lib/ui') } + }, + logLevel: 'error' + }); // Load the real migrations through Vite's ?raw support. Importing hooks // would start queues and agents; this script never loads the app router. - const database = await vite.ssrLoadModule('/src/lib/server/db/index.ts') as { db: () => Database.Database; closeDb: () => void }; + const database = (await vite.ssrLoadModule('/src/lib/server/db/index.ts')) as { + db: () => Database.Database; + closeDb: () => void; + }; closeDb = database.closeDb; const handle = database.db(); const hosts = await vite.ssrLoadModule('/src/lib/server/hosts/schema.ts'); const auth = await vite.ssrLoadModule('/src/lib/server/auth/schema.ts'); const files = await vite.ssrLoadModule('/src/lib/server/game-files/schema.ts'); - hosts.ensureHostSchema(handle); auth.ensureAuthSchema(handle); files.ensureGameFileSchema(handle); + hosts.ensureHostSchema(handle); + auth.ensureAuthSchema(handle); + files.ensureGameFileSchema(handle); seedFixtureData(handle, options.now); const password = randomBytes(24).toString('base64url'); const { hashPassword } = await vite.ssrLoadModule('/src/lib/server/auth/passwords.js'); - handle.prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,created_at,updated_at) - VALUES('fixture-admin','demo','Demo operator',?,'admin',?,?)`).run(await hashPassword(password), options.now, options.now); - if ((handle.pragma('foreign_key_check') as unknown[]).length || handle.pragma('integrity_check', { simple: true }) !== 'ok') { + handle + .prepare( + `INSERT INTO auth_users(id,username,display_name,password_hash,role,created_at,updated_at) + VALUES('fixture-admin','demo','Demo operator',?,'admin',?,?)` + ) + .run(await hashPassword(password), options.now, options.now); + if ( + (handle.pragma('foreign_key_check') as unknown[]).length || + handle.pragma('integrity_check', { simple: true }) !== 'ok' + ) { throw new Error('Fixture integrity validation failed.'); } - const environment = { ...paths, RCON_PASSWORD: '', RCON_HOST: '127.0.0.1', EC_AUTH_ALLOW_HTTP: 'true', - PLUTAINER_IMAGE: process.env.PLUTAINER_IMAGE! }; + const environment = { + ...paths, + RCON_PASSWORD: '', + RCON_HOST: '127.0.0.1', + EC_AUTH_ALLOW_HTTP: 'true', + PLUTAINER_IMAGE: process.env.PLUTAINER_IMAGE! + }; const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; - await writeFile(join(destination.root, 'preview.env'), Object.entries(environment).map(([key, value]) => `export ${key}=${shellQuote(value)}`).join('\n') + '\n', { flag: 'wx', mode: 0o600 }); - await writeFile(join(destination.root, 'fixture.json'), JSON.stringify({ format: FIXTURE_VERSION, marker: FIXTURE_KEY, - dbPath: destination.dbPath, createdAt: new Date(options.now).toISOString(), username: 'demo', password, - hosts: 3, servers: fixtureServers.map(server => server.name), matches: 500, - note: 'Synthetic local data only. Host credentials cannot authenticate; game images and library IDs are fictitious.' }, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + await writeFile( + join(destination.root, 'preview.env'), + Object.entries(environment) + .map(([key, value]) => `export ${key}=${shellQuote(value)}`) + .join('\n') + '\n', + { flag: 'wx', mode: 0o600 } + ); + await writeFile( + join(destination.root, 'fixture.json'), + JSON.stringify( + { + format: FIXTURE_VERSION, + marker: FIXTURE_KEY, + dbPath: destination.dbPath, + createdAt: new Date(options.now).toISOString(), + username: 'demo', + password, + hosts: 3, + servers: fixtureServers.map((server) => server.name), + matches: 500, + note: 'Synthetic local data only. Host credentials cannot authenticate; game images and library IDs are fictitious.' + }, + null, + 2 + ) + '\n', + { flag: 'wx', mode: 0o600 } + ); handle.pragma('wal_checkpoint(TRUNCATE)'); complete = true; - console.log(`Created ${destination.dbPath}\n3 hosts · 12 servers · 500 matches · 7 days of metrics\nLocal sign-in details: ${join(destination.root, 'fixture.json')}\nPreview environment: ${join(destination.root, 'preview.env')}`); + console.log( + `Created ${destination.dbPath}\n3 hosts · 12 servers · 500 matches · 7 days of metrics\nLocal sign-in details: ${join(destination.root, 'fixture.json')}\nPreview environment: ${join(destination.root, 'preview.env')}` + ); if (options.watch) { refreshFixtureData(handle, Date.now()); - console.log('Refreshing synthetic observations every 15 seconds. Leave this terminal open; Ctrl+C stops the feeder.'); + console.log( + 'Refreshing synthetic observations every 15 seconds. Leave this terminal open; Ctrl+C stops the feeder.' + ); await new Promise((resolvePromise, reject) => { - const finish = (error?: unknown) => { clearInterval(timer); process.off('SIGINT', stop); process.off('SIGTERM', stop); if (error) reject(error); else resolvePromise(); }; + const finish = (error?: unknown) => { + clearInterval(timer); + process.off('SIGINT', stop); + process.off('SIGTERM', stop); + if (error) reject(error); + else resolvePromise(); + }; const stop = () => finish(); - const timer = setInterval(() => { try { refreshFixtureData(handle, Date.now()); } catch (error) { finish(error); } }, 15_000); - process.on('SIGINT', stop); process.on('SIGTERM', stop); + const timer = setInterval(() => { + try { + refreshFixtureData(handle, Date.now()); + } catch (error) { + finish(error); + } + }, 15_000); + process.on('SIGINT', stop); + process.on('SIGTERM', stop); }); } } finally { - try { closeDb?.(); } - finally { - try { await vite?.close(); } - finally { if (!complete) await rm(destination.root, { recursive: true, force: true }); } + try { + closeDb?.(); + } finally { + try { + await vite?.close(); + } finally { + if (!complete) await rm(destination.root, { recursive: true, force: true }); + } } } } -main().catch(error => { - console.error(`Fixture creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`); +main().catch((error) => { + console.error( + `Fixture creation failed: ${error instanceof Error ? error.message : 'Unknown error'}` + ); process.exitCode = 1; }); diff --git a/src/hooks.server.ts b/src/hooks.server.ts index a6f5d5e..6fa6eb7 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -13,7 +13,7 @@ let rejectionLoggerRegistered = false; function boot(): void { if (booted) return; if (!rejectionLoggerRegistered) { - process.on('unhandledRejection', reason => { + process.on('unhandledRejection', (reason) => { console.error('[process] unhandled rejection:', reason); }); rejectionLoggerRegistered = true; @@ -30,13 +30,31 @@ function boot(): void { export const handle: Handle = async (input) => { boot(); const started = performance.now(); - const response = await withHostRequestContext(() => withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`, - () => authenticationHandle(input))); + const response = await withHostRequestContext(() => + withOperationSource( + `${input.event.request.method} ${input.event.route.id ?? 'Web request'}`, + () => authenticationHandle(input) + ) + ); const elapsed = performance.now() - started; response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`); - if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) { - console.info('[request]', { route: input.event.route.id, method: input.event.request.method, status: response.status, elapsedMs: Math.round(elapsed) }); + if ( + process.env.EC_PERF_LOG === 'true' && + elapsed > 250 && + !input.event.route.id?.startsWith('/api/agent/') + ) { + console.info('[request]', { + route: input.event.route.id, + method: input.event.request.method, + status: response.status, + elapsedMs: Math.round(elapsed) + }); } - if (input.event.locals.user && !['GET','HEAD','OPTIONS'].includes(input.event.request.method) && response.status < 400) invalidateFleet(); + if ( + input.event.locals.user && + !['GET', 'HEAD', 'OPTIONS'].includes(input.event.request.method) && + response.status < 400 + ) + invalidateFleet(); return response; }; diff --git a/src/lib/activity.ts b/src/lib/activity.ts index 7d64287..e95331f 100644 --- a/src/lib/activity.ts +++ b/src/lib/activity.ts @@ -1,17 +1,37 @@ export const activityTypes = { - create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server', - stop: 'Stop server', remove: 'Destroy server', update: 'Game update', files: 'File changes', - health: 'Health finding', drift: 'File drift', host: 'Host operation', beta: 'Beta test' + create: 'Create server', + start: 'Start server', + restart: 'Restart server', + rebuild: 'Rebuild server', + stop: 'Stop server', + remove: 'Destroy server', + update: 'Game update', + files: 'File changes', + health: 'Health finding', + drift: 'File drift', + host: 'Host operation', + beta: 'Beta test' } as const; export type ActivityType = keyof typeof activityTypes; export const activitySources = { - provisioning: 'Server operations', maintenance: 'Maintenance & recovery', recovery: 'Recovery', - campaign: 'Campaigns', beta: 'Betas', host: 'Hosts', autoscaling: 'Autoscaling' + provisioning: 'Server operations', + maintenance: 'Maintenance & recovery', + recovery: 'Recovery', + campaign: 'Campaigns', + beta: 'Betas', + host: 'Hosts', + autoscaling: 'Autoscaling' } as const; export type ActivitySource = keyof typeof activitySources; export const activityStates = { - queued: 'Queued', running: 'In progress', succeeded: 'Completed', warning: 'Completed with warnings', - failed: 'Failed', cancelled: 'Cancelled', attention: 'Needs attention', unknown: 'Unknown' + queued: 'Queued', + running: 'In progress', + succeeded: 'Completed', + warning: 'Completed with warnings', + failed: 'Failed', + cancelled: 'Cancelled', + attention: 'Needs attention', + unknown: 'Unknown' } as const; export type ActivityState = keyof typeof activityStates; @@ -54,8 +74,11 @@ export interface ActivityPage { query: ActivityQuery; } -function selection(value: string | null, labels: Record): T | undefined { - return value !== null && Object.hasOwn(labels, value) ? value as T : undefined; +function selection( + value: string | null, + labels: Record +): T | undefined { + return value !== null && Object.hasOwn(labels, value) ? (value as T) : undefined; } function integer(value: string | null, fallback: number, min: number, max: number): number { if (value === null || !/^\d+$/.test(value)) return fallback; @@ -72,7 +95,10 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery { if (params.get('excludeProvisioning') === 'true') query.excludeProvisioning = true; if (params.get('excludeMaintenance') === 'true') query.excludeMaintenance = true; for (const key of ['server', 'host', 'q'] as const) { - const value = params.get(key)?.trim().slice(0, key === 'q' ? 200 : 255); + const value = params + .get(key) + ?.trim() + .slice(0, key === 'q' ? 200 : 255); if (value) query[key] = value; } query.type = selection(params.get('type'), activityTypes); @@ -86,6 +112,7 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery { export function normalizeActivityQuery(query: ActivityQuery): ActivityQuery { const params = new URLSearchParams(); - for (const [key, value] of Object.entries(query)) if (value !== undefined) params.set(key, String(value)); + for (const [key, value] of Object.entries(query)) + if (value !== undefined) params.set(key, String(value)); return parseActivityQuery(params); } diff --git a/src/lib/artifact-version.ts b/src/lib/artifact-version.ts index a5d03ff..88164b3 100644 --- a/src/lib/artifact-version.ts +++ b/src/lib/artifact-version.ts @@ -1,5 +1,7 @@ /** Keep the upload number visible: two builds may share a publisher's release tag. */ -export function artifactVersionLabel(version: { version: number; release_version?: string | null } | null | undefined): string { +export function artifactVersionLabel( + version: { version: number; release_version?: string | null } | null | undefined +): string { if (!version) return '—'; return version.release_version ? `${version.release_version} (upload ${version.version})` diff --git a/src/lib/auth.ts b/src/lib/auth.ts index dd297f2..44c5d3d 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -16,5 +16,7 @@ export interface AuthUser { } export const roleLabels: Record = { - admin: 'Administrator', operator: 'Operator', viewer: 'Viewer' + admin: 'Administrator', + operator: 'Operator', + viewer: 'Viewer' }; diff --git a/src/lib/autoscaling.ts b/src/lib/autoscaling.ts index ae5fdb9..ef83a29 100644 --- a/src/lib/autoscaling.ts +++ b/src/lib/autoscaling.ts @@ -1,56 +1,128 @@ import { zombieMapBySlug, type ZombieGame } from './map-aliases.js'; export const autoscalingModes = ['off', 'create-destroy'] as const; -export type AutoscalingMode = typeof autoscalingModes[number]; +export type AutoscalingMode = (typeof autoscalingModes)[number]; export interface ScalingSettings { game: ZombieGame; /** '*' applies the targets independently to every map. */ mapSlug: string; mode: AutoscalingMode; - minimum: number; maximum: number; spares: number; emptyMinutes: number; + minimum: number; + maximum: number; + spares: number; + emptyMinutes: number; } export interface ScalingPolicy extends ScalingSettings { - id: string; revision: number; pausedReason: string | null; updatedAt: string; + id: string; + revision: number; + pausedReason: string | null; + updatedAt: string; } export interface ScalingMember { - name: string; mapSlug?: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected'; - players: number | null; observedAt: number | null; source: string | null; - emptySince: number | null; blocker: string | null; operationId: string | null; + name: string; + mapSlug?: string; + identity: string; + state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected'; + players: number | null; + observedAt: number | null; + source: string | null; + emptySince: number | null; + blocker: string | null; + operationId: string | null; emptyStartedAt?: number | null; } export interface ScalingSnapshot { - members: ScalingMember[]; reservedNames: string[]; keyPool: { total: number; available: number }; blockers: string[]; + members: ScalingMember[]; + reservedNames: string[]; + keyPool: { total: number; available: number }; + blockers: string[]; } export interface ScalingDecision { - action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null; - desired: number; deficit: number; occupied: number; empty: number; pending: number; unknown: number; + action: 'create' | 'destroy' | 'wait'; + reason: string; + server: string | null; + desired: number; + deficit: number; + occupied: number; + empty: number; + pending: number; + unknown: number; +} +export interface ScalingEvaluation { + policy: ScalingPolicy; + snapshot: ScalingSnapshot; + decision: ScalingDecision; + blockers: string[]; +} +export interface ScalingHistory { + id: string; + at: number; + lastSeenAt: number; + decision: ScalingDecision; + operationId: string | null; + operationState: string | null; } -export interface ScalingEvaluation { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; blockers: string[] } -export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null } export interface ScalingView extends ScalingEvaluation { - history: ScalingHistory[]; lastEvaluatedAt: number | null; - totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number }; - operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null }; + history: ScalingHistory[]; + lastEvaluatedAt: number | null; + totals: { + observedMinutes: number; + emptyLobbyHours: number; + deficitMinutes: number; + unknownMinutes: number; + }; + operations: { + creates: number; + removals: number; + cancelled: number; + failed: number; + readinessSamples: number; + readinessMeanSeconds: number | null; + readinessMaxSeconds: number | null; + }; ownsMap: boolean; unsettled: { id: string; action: string; state: string; stage: string }[]; } -export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean } +export interface ScalingReport { + policies: ScalingView[]; + globalPaused: boolean; + liveAvailable: boolean; + removalAvailable: boolean; +} export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-destroy'; export function scalingSettings(input: unknown): ScalingSettings { - if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.'); + if (!input || typeof input !== 'object' || Array.isArray(input)) + throw new Error('Provide an autoscaling policy.'); const value = input as Record; const { game, mapSlug } = value; - if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)) throw new Error('Choose a supported game and map variant, or all maps.'); + if ( + (game !== 't5' && game !== 't6') || + typeof mapSlug !== 'string' || + (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug) + ) + throw new Error('Choose a supported game and map variant, or all maps.'); const mode = value.mode ?? 'off'; - if (!autoscalingModes.includes(mode as AutoscalingMode)) throw new Error('Choose a valid autoscaling mode.'); + if (!autoscalingModes.includes(mode as AutoscalingMode)) + throw new Error('Choose a valid autoscaling mode.'); const integer = (key: string, fallback: number | undefined, max: number) => { const n = value[key] ?? fallback; - if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max) throw new Error(`${key} must be an integer from 1 to ${max}.`); + if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max) + throw new Error(`${key} must be an integer from 1 to ${max}.`); return n; }; - const minimum = integer('minimum', 1, 20), maximum = integer('maximum', undefined, 20), spares = integer('spares', 1, 20); - if (minimum > maximum || spares > maximum) throw new Error('Minimum and spare target cannot exceed the maximum.'); - return { game, mapSlug, mode: mode as AutoscalingMode, minimum, maximum, spares, - emptyMinutes: integer('emptyMinutes', 15, 1440) }; + const minimum = integer('minimum', 1, 20), + maximum = integer('maximum', undefined, 20), + spares = integer('spares', 1, 20); + if (minimum > maximum || spares > maximum) + throw new Error('Minimum and spare target cannot exceed the maximum.'); + return { + game, + mapSlug, + mode: mode as AutoscalingMode, + minimum, + maximum, + spares, + emptyMinutes: integer('emptyMinutes', 15, 1440) + }; } diff --git a/src/lib/beta-draft.ts b/src/lib/beta-draft.ts index 6877d9f..b6b3f86 100644 --- a/src/lib/beta-draft.ts +++ b/src/lib/beta-draft.ts @@ -2,9 +2,12 @@ import { requirementKinds, type BetaPlan } from './beta-plan.js'; export const betaDraftKey = 'electric-cherry:beta-draft:v1'; export interface BetaDraft { - version: 1; savedAt: string; step: number; + version: 1; + savedAt: string; + step: number; picks: Array<{ scriptId: number | null; versionId: number | null }>; - servers: string[]; testPlan: BetaPlan; + servers: string[]; + testPlan: BetaPlan; selectionMode: 'automatic' | 'manual'; instanceCount: number | null; label?: string; @@ -14,55 +17,138 @@ export function readBetaDraft(raw: string | null): BetaDraft | null { if (!raw) return null; try { const d = JSON.parse(raw); - if (d?.version !== 1 || !Array.isArray(d.picks) || !d.picks.length || d.picks.length > 100 - || !d.picks.every((p: BetaDraft['picks'][number]) => p && [p.scriptId, p.versionId].every(v => v === null || Number.isSafeInteger(v) && v! > 0)) - || !Array.isArray(d.servers) || !d.servers.every((s: unknown) => typeof s === 'string') - || d.testPlan?.version !== 1 || !Array.isArray(d.testPlan.requirements) || d.testPlan.requirements.length > 30 - || !d.testPlan.requirements.every((r: BetaPlan['requirements'][number]) => r && requirementKinds.includes(r.kind) && typeof r.id === 'string' && /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) && typeof r.label === 'string')) return null; - if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode)) return null; - if (d.instancesPerGame !== undefined && (!d.instancesPerGame || typeof d.instancesPerGame !== 'object' - || Array.isArray(d.instancesPerGame) || !Object.entries(d.instancesPerGame).every(([game, count]) => - ['t5', 't6'].includes(game) && (count === null || typeof count === 'number' && Number.isFinite(count))))) return null; - if (d.instanceCount !== undefined && d.instanceCount !== null - && (typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))) return null; + if ( + d?.version !== 1 || + !Array.isArray(d.picks) || + !d.picks.length || + d.picks.length > 100 || + !d.picks.every( + (p: BetaDraft['picks'][number]) => + p && + [p.scriptId, p.versionId].every( + (v) => v === null || (Number.isSafeInteger(v) && v! > 0) + ) + ) || + !Array.isArray(d.servers) || + !d.servers.every((s: unknown) => typeof s === 'string') || + d.testPlan?.version !== 1 || + !Array.isArray(d.testPlan.requirements) || + d.testPlan.requirements.length > 30 || + !d.testPlan.requirements.every( + (r: BetaPlan['requirements'][number]) => + r && + requirementKinds.includes(r.kind) && + typeof r.id === 'string' && + /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) && + typeof r.label === 'string' + ) + ) + return null; + if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode)) + return null; + if ( + d.instancesPerGame !== undefined && + (!d.instancesPerGame || + typeof d.instancesPerGame !== 'object' || + Array.isArray(d.instancesPerGame) || + !Object.entries(d.instancesPerGame).every( + ([game, count]) => + ['t5', 't6'].includes(game) && + (count === null || (typeof count === 'number' && Number.isFinite(count))) + )) + ) + return null; + if ( + d.instanceCount !== undefined && + d.instanceCount !== null && + (typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount)) + ) + return null; const previousCounts = Object.values(d.instancesPerGame ?? {}) as Array; - const instanceCount = d.instanceCount !== undefined ? d.instanceCount - : previousCounts.includes(null) ? null : previousCounts.length ? Math.max(...previousCounts as number[]) : 1; + const instanceCount = + d.instanceCount !== undefined + ? d.instanceCount + : previousCounts.includes(null) + ? null + : previousCounts.length + ? Math.max(...(previousCounts as number[])) + : 1; // Existing drafts represent explicit selections and must not silently switch modes. const selectionMode = d.selectionMode ?? (d.servers.length ? 'manual' : 'automatic'); // Revisit server setup when old per-game counts cannot be retained as one value. - const changedCounts = selectionMode === 'automatic' && d.instanceCount === undefined && new Set(previousCounts).size > 1; - return { ...d, selectionMode, - instanceCount, step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0)) }; - } catch { return null; } + const changedCounts = + selectionMode === 'automatic' && + d.instanceCount === undefined && + new Set(previousCounts).size > 1; + return { + ...d, + selectionMode, + instanceCount, + step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0)) + }; + } catch { + return null; + } } -interface BetaFileScope { scope_type: 'game' | 'map' | null; scope_value: string | null } -interface BetaServerScope { name: string; game: string | null; map: string | null } +interface BetaFileScope { + scope_type: 'game' | 'map' | null; + scope_value: string | null; +} +interface BetaServerScope { + name: string; + game: string | null; + map: string | null; +} export function betaScopeMatches(file: BetaFileScope, server: BetaServerScope): boolean { - return !file.scope_type || (file.scope_type === 'game' ? server.game : server.map) === file.scope_value; + return ( + !file.scope_type || + (file.scope_type === 'game' ? server.game : server.map) === file.scope_value + ); } /** A bundle installs each file only on its own matching participants. */ -export function betaCandidateServers(files: readonly BetaFileScope[], servers: readonly T[]): T[] { - return servers.filter(server => files.some(file => betaScopeMatches(file, server))); +export function betaCandidateServers( + files: readonly BetaFileScope[], + servers: readonly T[] +): T[] { + return servers.filter((server) => files.some((file) => betaScopeMatches(file, server))); } -export function betaApplicableGames(files: readonly BetaFileScope[], servers: readonly BetaServerScope[]): string[] { - return [...new Set([ - ...files.filter(file => file.scope_type === 'game').map(file => file.scope_value), - ...betaCandidateServers(files, servers).map(server => server.game) - ].filter((game): game is string => game === 't5' || game === 't6'))].sort(); +export function betaApplicableGames( + files: readonly BetaFileScope[], + servers: readonly BetaServerScope[] +): string[] { + return [ + ...new Set( + [ + ...files + .filter((file) => file.scope_type === 'game') + .map((file) => file.scope_value), + ...betaCandidateServers(files, servers).map((server) => server.game) + ].filter((game): game is string => game === 't5' || game === 't6') + ) + ].sort(); } /** Expand the shared count only to games represented by the selected files. */ -export function betaInstanceCounts(games: readonly string[], count: number | null): Record { +export function betaInstanceCounts( + games: readonly string[], + count: number | null +): Record { if (count === null || !Number.isSafeInteger(count) || count < 1 || count > 1000) return {}; - return Object.fromEntries(games.map(game => [game, count])); + return Object.fromEntries(games.map((game) => [game, count])); } -export function compatibleBetaServers(selected: Iterable, candidates: readonly string[]): { kept: Set; removed: string[] } { - const allowed = new Set(candidates), names = [...selected]; - return { kept: new Set(names.filter(name => allowed.has(name))), removed: names.filter(name => !allowed.has(name)) }; +export function compatibleBetaServers( + selected: Iterable, + candidates: readonly string[] +): { kept: Set; removed: string[] } { + const allowed = new Set(candidates), + names = [...selected]; + return { + kept: new Set(names.filter((name) => allowed.has(name))), + removed: names.filter((name) => !allowed.has(name)) + }; } diff --git a/src/lib/beta-evidence.ts b/src/lib/beta-evidence.ts index a107eff..3b0d5ef 100644 --- a/src/lib/beta-evidence.ts +++ b/src/lib/beta-evidence.ts @@ -1,23 +1,35 @@ import type { BetaRequirement, BetaTestView } from './beta-plan.js'; import { needsGameTelemetry } from './beta-plan-summary.js'; -export function betaWaitingCondition(requirement: BetaRequirement, observation?: BetaTestView['servers'][number]['observation']): string { - if (requirement.kind === 'manual') return 'Waiting for a tester to record their name and observation.'; - if (requirement.kind === 'recovery') return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.'; - if (requirement.kind === 'healthy-minutes') return observation?.healthy === true - ? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.' - : 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.'; - if (requirement.kind === 'failure-budget') return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.'; +export function betaWaitingCondition( + requirement: BetaRequirement, + observation?: BetaTestView['servers'][number]['observation'] +): string { + if (requirement.kind === 'manual') + return 'Waiting for a tester to record their name and observation.'; + if (requirement.kind === 'recovery') + return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.'; + if (requirement.kind === 'healthy-minutes') + return observation?.healthy === true + ? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.' + : 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.'; + if (requirement.kind === 'failure-budget') + return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.'; if (needsGameTelemetry(requirement)) { - if (!observation?.lastGameplayAt) return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.'; + if (!observation?.lastGameplayAt) + return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.'; if (observation.gameEnded) return 'Waiting for the next game and its qualifying reports.'; - if (requirement.kind === 'population') return `Waiting for a game report containing at least ${requirement.target} players.`; + if (requirement.kind === 'population') + return `Waiting for a game report containing at least ${requirement.target} players.`; const players = `at least ${requirement.minPlayers} player${requirement.minPlayers === 1 ? '' : 's'}`; - if ((observation.players ?? 0) < requirement.minPlayers) return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`; - if (requirement.kind === 'matches') return requirement.minMinutes && !observation.matchStarted - ? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.' - : `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`; - if (requirement.kind === 'round') return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`; + if ((observation.players ?? 0) < requirement.minPlayers) + return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`; + if (requirement.kind === 'matches') + return requirement.minMinutes && !observation.matchStarted + ? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.' + : `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`; + if (requirement.kind === 'round') + return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`; return `Time is credited when the next report arrives, with ${players} at both ends of the interval. It does not advance continuously between reports.`; } return 'Waiting for qualifying evidence.'; @@ -25,13 +37,26 @@ export function betaWaitingCondition(requirement: BetaRequirement, observation?: export function betaEvidenceSummary(event: BetaTestView['evidence'][number]): string { let payload: Record; - try { payload = JSON.parse(event.payload); } catch { return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); } + try { + payload = JSON.parse(event.payload); + } catch { + return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); + } if (!payload || typeof payload !== 'object') return event.kind; if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) { - const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : event.kind === 'heartbeat' ? 'Heartbeat' : 'Round changed'; + const label = + event.kind === 'game_start' + ? 'Game started' + : event.kind === 'game_end' + ? 'Game ended' + : event.kind === 'heartbeat' + ? 'Heartbeat' + : 'Round changed'; return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`; } - if (event.kind === 'manual-sign-off') return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`; - if (event.kind === 'health' || event.kind === 'status-sample') return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`; + if (event.kind === 'manual-sign-off') + return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`; + if (event.kind === 'health' || event.kind === 'status-sample') + return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`; return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); } diff --git a/src/lib/beta-plan-summary.ts b/src/lib/beta-plan-summary.ts index 4545932..f097700 100644 --- a/src/lib/beta-plan-summary.ts +++ b/src/lib/beta-plan-summary.ts @@ -1,34 +1,66 @@ import type { BetaPlan, BetaRequirement } from './beta-plan.js'; const count = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`; -const duration = (minutes: number) => minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute'); +const duration = (minutes: number) => + minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute'); export const presetDescriptions: Record = { 'Quick smoke test': 'A quick check that the server starts and stays healthy.', 'Gameplay validation': 'Prove the build through real matches and populated playtime.', 'Long soak': 'Exercise stability and participation over a longer test.' }; export function requirementSummary(r: BetaRequirement): string { - const players = r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present'; + const players = + r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present'; switch (r.kind) { - case 'round': return `Reach round ${r.target}${players}`; - case 'matches': return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`; - case 'populated-minutes': return `Accumulate ${duration(r.target)}${players}`; - case 'player-hours': return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`; - case 'population': return `Reach ${count(r.target, 'player')} online`; - case 'sustained-population': return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`; - case 'healthy-minutes': return `Stay continuously healthy for ${duration(r.target)}`; - case 'failure-budget': return r.target === 0 ? 'Allow no health outages' : `Allow at most ${count(r.target, 'health outage')}`; - case 'recovery': return `Repeat the checks after ${count(r.target, 'recovery restart')}`; - case 'manual': return r.label; + case 'round': + return `Reach round ${r.target}${players}`; + case 'matches': + return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`; + case 'populated-minutes': + return `Accumulate ${duration(r.target)}${players}`; + case 'player-hours': + return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`; + case 'population': + return `Reach ${count(r.target, 'player')} online`; + case 'sustained-population': + return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`; + case 'healthy-minutes': + return `Stay continuously healthy for ${duration(r.target)}`; + case 'failure-budget': + return r.target === 0 + ? 'Allow no health outages' + : `Allow at most ${count(r.target, 'health outage')}`; + case 'recovery': + return `Repeat the checks after ${count(r.target, 'recovery restart')}`; + case 'manual': + return r.label; } } export function coverageSummary(plan: BetaPlan): string { - return plan.scope === 'all' ? 'Every selected server' : plan.scope === 'quorum' ? `At least ${count(plan.quorum, 'selected server')}` - : plan.scope === 'map' ? 'One server per selected map' : plan.scope === 'game' ? 'One server per selected game' : 'One server per game / map'; + return plan.scope === 'all' + ? 'Every selected server' + : plan.scope === 'quorum' + ? `At least ${count(plan.quorum, 'selected server')}` + : plan.scope === 'map' + ? 'One server per selected map' + : plan.scope === 'game' + ? 'One server per selected game' + : 'One server per game / map'; } export function needsGameTelemetry(r: BetaRequirement): boolean { - return ['round', 'matches', 'population', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind); + return [ + 'round', + 'matches', + 'population', + 'populated-minutes', + 'player-hours', + 'sustained-population' + ].includes(r.kind); } export function requirementCategory(r: BetaRequirement): string { - return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget'].includes(r.kind) ? 'Continuous check' : 'Milestone'; + return r.kind === 'manual' + ? 'Tester sign-off' + : ['healthy-minutes', 'failure-budget'].includes(r.kind) + ? 'Continuous check' + : 'Milestone'; } diff --git a/src/lib/beta-plan.ts b/src/lib/beta-plan.ts index 3c94436..caccf56 100644 --- a/src/lib/beta-plan.ts +++ b/src/lib/beta-plan.ts @@ -1,6 +1,16 @@ -export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population', - 'sustained-population', 'healthy-minutes', 'failure-budget', 'recovery', 'manual'] as const; -export type RequirementKind = typeof requirementKinds[number]; +export const requirementKinds = [ + 'round', + 'matches', + 'populated-minutes', + 'player-hours', + 'population', + 'sustained-population', + 'healthy-minutes', + 'failure-budget', + 'recovery', + 'manual' +] as const; +export type RequirementKind = (typeof requirementKinds)[number]; export interface BetaRequirement { id: string; kind: RequirementKind; @@ -21,90 +31,306 @@ export interface BetaPlan { waveSize: number; // 0 deploys all selected servers; otherwise pass each wave before expanding. } export const requirementLabels: Record = { - round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes', - 'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)', - 'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages', - recovery: 'Successful recovery cycles', manual: 'Tester checklist' + round: 'Reach round', + matches: 'Complete qualifying matches', + 'populated-minutes': 'Populated minutes', + 'player-hours': 'Player-hours', + population: 'Reach player count', + 'sustained-population': 'Sustained population (minutes)', + 'healthy-minutes': 'Continuous healthy minutes', + 'failure-budget': 'Maximum health outages', + recovery: 'Successful recovery cycles', + manual: 'Tester checklist' }; export function requirement(kind: RequirementKind, id: string): BetaRequirement { - return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1, - minRound: 0, minPlayers: 1, minMinutes: 0 }; + return { + id, + kind, + label: requirementLabels[kind], + target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1, + minRound: 0, + minPlayers: 1, + minMinutes: 0 + }; } export function defaultPlan(): BetaPlan { - return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1, - timeoutHours: 24, failureAction: 'fail', rollbackRestart: 'when-empty', waveSize: 0 }; + return { + version: 1, + requirements: [requirement('healthy-minutes', 'health')], + scope: 'all', + quorum: 1, + timeoutHours: 24, + failureAction: 'fail', + rollbackRestart: 'when-empty', + waveSize: 0 + }; } export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [ { name: 'Quick smoke test', plan: defaultPlan() }, - { name: 'Gameplay validation', plan: { ...defaultPlan(), requirements: [ - { ...requirement('matches', 'matches'), target: 2, minRound: 25 }, - { ...requirement('populated-minutes', 'playtime'), target: 90 }, requirement('failure-budget', 'outages') - ] } }, - { name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [ - { ...requirement('healthy-minutes', 'health'), target: 360 }, - { ...requirement('player-hours', 'participation'), target: 10 }, requirement('failure-budget', 'outages') - ] } } + { + name: 'Gameplay validation', + plan: { + ...defaultPlan(), + requirements: [ + { ...requirement('matches', 'matches'), target: 2, minRound: 25 }, + { ...requirement('populated-minutes', 'playtime'), target: 90 }, + requirement('failure-budget', 'outages') + ] + } + }, + { + name: 'Long soak', + plan: { + ...defaultPlan(), + timeoutHours: 72, + requirements: [ + { ...requirement('healthy-minutes', 'health'), target: 360 }, + { ...requirement('player-hours', 'participation'), target: 10 }, + requirement('failure-budget', 'outages') + ] + } + } ]; /** Strict allow-list validation at every boundary (including stored presets). */ export function parseBetaPlan(value: unknown): BetaPlan { - if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Test plan must be an object'); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Test plan must be an object'); const p = value as BetaPlan; - if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) throw new Error('Invalid test plan version or scope'); - if (!['fail', 'pause', 'rollback'].includes(p.failureAction) || !['manual', 'when-empty'].includes(p.rollbackRestart)) throw new Error('Invalid failure policy'); - for (const [key, min, max] of [['quorum', 1, 1000], ['timeoutHours', 0.01, 8760], ['waveSize', 0, 1000]] as const) { - if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) throw new Error(`Invalid ${key}`); + if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) + throw new Error('Invalid test plan version or scope'); + if ( + !['fail', 'pause', 'rollback'].includes(p.failureAction) || + !['manual', 'when-empty'].includes(p.rollbackRestart) + ) + throw new Error('Invalid failure policy'); + for (const [key, min, max] of [ + ['quorum', 1, 1000], + ['timeoutHours', 0.01, 8760], + ['waveSize', 0, 1000] + ] as const) { + if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) + throw new Error(`Invalid ${key}`); } - if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) throw new Error('Quorum and wave size must be integers'); - if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) throw new Error('Choose 1–30 requirements'); + if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) + throw new Error('Quorum and wave size must be integers'); + if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) + throw new Error('Choose 1–30 requirements'); const ids = new Set(); - const requirements = p.requirements.map(r => { - if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) throw new Error('Unknown requirement'); - if (typeof r.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || r.id in Object.prototype || ids.has(r.id)) throw new Error('Requirement IDs must be unique, non-reserved identifiers'); + const requirements = p.requirements.map((r) => { + if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) + throw new Error('Unknown requirement'); + if ( + typeof r.id !== 'string' || + !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || + r.id in Object.prototype || + ids.has(r.id) + ) + throw new Error('Requirement IDs must be unique, non-reserved identifiers'); ids.add(r.id); - if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) throw new Error('Requirement needs a label (up to 200 characters)'); + if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) + throw new Error('Requirement needs a label (up to 200 characters)'); for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) { - if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`); + if ( + typeof r[key] !== 'number' || + !Number.isFinite(r[key]) || + r[key] < 0 || + r[key] > 1_000_000 + ) + throw new Error(`Invalid requirement ${key}`); } - if (r.kind !== 'failure-budget' && r.target <= 0) throw new Error('Requirement target must be positive'); - if (['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers'); - if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers'); - if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item'); - return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound, - minPlayers: r.minPlayers, minMinutes: r.minMinutes }; + if (r.kind !== 'failure-budget' && r.target <= 0) + throw new Error('Requirement target must be positive'); + if ( + ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes( + r.kind + ) && + !Number.isInteger(r.target) + ) + throw new Error('Count targets must be integers'); + if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) + throw new Error('Round and player qualifiers must be integers'); + if (r.kind === 'manual' && r.target !== 1) + throw new Error('Add a separate requirement for each checklist item'); + return { + id: r.id, + kind: r.kind, + label: r.label.trim(), + target: r.target, + minRound: r.minRound, + minPlayers: r.minPlayers, + minMinutes: r.minMinutes + }; }); - return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours, - failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize }; + return { + version: 1, + requirements, + scope: p.scope, + quorum: p.quorum, + timeoutHours: p.timeoutHours, + failureAction: p.failureAction, + rollbackRestart: p.rollbackRestart, + waveSize: p.waveSize + }; } -export interface RequirementProgress { id: string; label: string; current: number; target: number; passed: boolean; failed: boolean; detail: string } +export interface RequirementProgress { + id: string; + label: string; + current: number; + target: number; + passed: boolean; + failed: boolean; + detail: string; +} export interface BetaTestView { plan: BetaPlan; outcome: string; reason: string | null; - servers: Array<{ server: string; server_deleted_at: string | null; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null; - observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>; - evidence: Array<{ id: number; server: string; server_deleted_at: string | null; received_at: string; kind: string; payload: string }>; + servers: Array<{ + server: string; + server_deleted_at: string | null; + phase: number; + passed: boolean; + progress: RequirementProgress[]; + observingAt?: string | null; + observation?: { + lastGameplayAt: string | null; + lastEvent: string | null; + players: number | null; + round: number | null; + matchStarted: boolean; + gameEnded: boolean; + healthy: boolean | null; + healthCheckedAt: string | null; + }; + }>; + evidence: Array<{ + id: number; + server: string; + server_deleted_at: string | null; + received_at: string; + kind: string; + payload: string; + }>; } -export interface BetaPlanIssue { field: string; message: string; requirementId?: string } +export interface BetaPlanIssue { + field: string; + message: string; + requirementId?: string; +} export function betaPlanIssues(plan: BetaPlan, serverCount?: number): BetaPlanIssue[] { const issues: BetaPlanIssue[] = []; - const check = (field: string, value: number, min: number, max: number, integer: boolean, message: string, requirementId?: string) => { - if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) issues.push({ field, message, requirementId }); + const check = ( + field: string, + value: number, + min: number, + max: number, + integer: boolean, + message: string, + requirementId?: string + ) => { + if ( + typeof value !== 'number' || + !Number.isFinite(value) || + value < min || + value > max || + (integer && !Number.isInteger(value)) + ) + issues.push({ field, message, requirementId }); }; - check('timeoutHours', plan.timeoutHours, .01, 8760, false, 'Enter a testing window between 0.01 and 8,760 hours.'); - check('waveSize', plan.waveSize, 0, 1000, true, 'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.'); - if (plan.scope === 'quorum') check('quorum', plan.quorum, 1, serverCount ?? 1000, true, `Choose between 1 and ${serverCount ?? 1000} qualifying servers.`); - if (!plan.requirements.length) issues.push({ field: 'requirements', message: 'Add at least one requirement to continue.' }); + check( + 'timeoutHours', + plan.timeoutHours, + 0.01, + 8760, + false, + 'Enter a testing window between 0.01 and 8,760 hours.' + ); + check( + 'waveSize', + plan.waveSize, + 0, + 1000, + true, + 'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.' + ); + if (plan.scope === 'quorum') + check( + 'quorum', + plan.quorum, + 1, + serverCount ?? 1000, + true, + `Choose between 1 and ${serverCount ?? 1000} qualifying servers.` + ); + if (!plan.requirements.length) + issues.push({ + field: 'requirements', + message: 'Add at least one requirement to continue.' + }); for (const r of plan.requirements) { - if (!r.label?.trim()) issues.push({ field: `${r.id}-label`, message: 'Describe what this requirement checks.', requirementId: r.id }); - const integral = ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind); - check(`${r.id}-target`, r.target, r.kind === 'failure-budget' ? 0 : .000001, 1_000_000, integral, `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`, r.id); - if (['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)) check(`${r.id}-minPlayers`, r.minPlayers, 1, 1_000_000, true, 'Enter at least one player, using a whole number.', r.id); + if (!r.label?.trim()) + issues.push({ + field: `${r.id}-label`, + message: 'Describe what this requirement checks.', + requirementId: r.id + }); + const integral = [ + 'round', + 'matches', + 'population', + 'failure-budget', + 'recovery', + 'manual' + ].includes(r.kind); + check( + `${r.id}-target`, + r.target, + r.kind === 'failure-budget' ? 0 : 0.000001, + 1_000_000, + integral, + `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`, + r.id + ); + if ( + [ + 'round', + 'matches', + 'populated-minutes', + 'player-hours', + 'sustained-population' + ].includes(r.kind) + ) + check( + `${r.id}-minPlayers`, + r.minPlayers, + 1, + 1_000_000, + true, + 'Enter at least one player, using a whole number.', + r.id + ); if (r.kind === 'matches') { - check(`${r.id}-minRound`, r.minRound, 0, 1_000_000, true, 'Enter a whole round number; 0 allows any round.', r.id); - check(`${r.id}-minMinutes`, r.minMinutes, 0, 1_000_000, false, 'Enter a duration of zero or more minutes.', r.id); + check( + `${r.id}-minRound`, + r.minRound, + 0, + 1_000_000, + true, + 'Enter a whole round number; 0 allows any round.', + r.id + ); + check( + `${r.id}-minMinutes`, + r.minMinutes, + 0, + 1_000_000, + false, + 'Enter a duration of zero or more minutes.', + r.id + ); } } return issues; diff --git a/src/lib/beta-promotion.ts b/src/lib/beta-promotion.ts index b85df3a..e895b98 100644 --- a/src/lib/beta-promotion.ts +++ b/src/lib/beta-promotion.ts @@ -3,12 +3,24 @@ export interface BetaPromotionPreview { token: string; createdAt: string; files: Array<{ - id: number; name: string; kind: string; path: string; versionId: number; - version: number; release_version: string | null; sha256: string; - scope: string; servers: string[]; + id: number; + name: string; + kind: string; + path: string; + versionId: number; + version: number; + release_version: string | null; + sha256: string; + scope: string; + servers: string[]; }>; servers: Array<{ name: string; tested: boolean; game: string | null; map: string | null }>; - configWrites: Array<{ server: string; path: string; sha256: string | null; error: string | null }>; + configWrites: Array<{ + server: string; + path: string; + sha256: string | null; + error: string | null; + }>; blockers: string[]; restartPolicy: 'additional-when-empty'; } diff --git a/src/lib/beta-revision.ts b/src/lib/beta-revision.ts index ac96fa3..14800a0 100644 --- a/src/lib/beta-revision.ts +++ b/src/lib/beta-revision.ts @@ -1,11 +1,23 @@ import type { BetaTestView } from './beta-plan.js'; export interface BetaRevisionFile { - script_id: number; name: string; kind: string; path: string; - version_id: number; version: number; release_version: string | null; sha256: string; + script_id: number; + name: string; + kind: string; + path: string; + version_id: number; + version: number; + release_version: string | null; + sha256: string; } export interface BetaRevision { - revision: number; started_at: string; notes: string | null; + revision: number; + started_at: string; + notes: string | null; files: BetaRevisionFile[]; - results: { state: string; test: BetaTestView | null; servers: Array<{ server: string; stage: string; error: string | null }> } | null; + results: { + state: string; + test: BetaTestView | null; + servers: Array<{ server: string; stage: string; error: string | null }>; + } | null; } diff --git a/src/lib/capacity.ts b/src/lib/capacity.ts index e689e93..52d3a73 100644 --- a/src/lib/capacity.ts +++ b/src/lib/capacity.ts @@ -2,12 +2,19 @@ import type { CapacityCalibration, HostReadiness } from './hosts.js'; export type CapacityRange = '1h' | '24h' | '7d'; export type CapacityGame = 'all' | 't5' | 't6'; -export interface CapacityQuery { range: CapacityRange; hostId: string | null; game: CapacityGame } +export interface CapacityQuery { + range: CapacityRange; + hostId: string | null; + game: CapacityGame; +} export function parseCapacityQuery(params: URLSearchParams): CapacityQuery { const range = params.get('range'); const game = params.get('game'); - return { range: range === '1h' || range === '7d' ? range : '24h', - game: game === 't5' || game === 't6' ? game : 'all', hostId: params.get('host')?.trim().slice(0, 100) || null }; + return { + range: range === '1h' || range === '7d' ? range : '24h', + game: game === 't5' || game === 't6' ? game : 'all', + hostId: params.get('host')?.trim().slice(0, 100) || null + }; } export interface CapacityHistoryPoint { @@ -60,4 +67,8 @@ export interface CapacityCurrent { hostOptions: { id: string; name: string }[]; warnings: string[]; } -export interface CapacityData { query: CapacityQuery; current: CapacityCurrent; history: CapacityHistory } +export interface CapacityData { + query: CapacityQuery; + current: CapacityCurrent; + history: CapacityHistory; +} diff --git a/src/lib/config-workbench.ts b/src/lib/config-workbench.ts index 2e9919a..313a9f4 100644 --- a/src/lib/config-workbench.ts +++ b/src/lib/config-workbench.ts @@ -27,33 +27,45 @@ export interface ConfigInspection { reason?: string; } -export interface ConfigServer { name: string; game: string | null; map: string | null; } +export interface ConfigServer { + name: string; + game: string | null; + map: string | null; +} /** The same lowercase `set` grammar that the overlay composer accepts. */ export function configLines(text: string) { return text.split(/\r?\n/).flatMap((line, index) => { const match = /^(\s*set\s+)(\S+)(\s+)(.*)$/.exec(line); - return match ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }] : []; + return match + ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }] + : []; }); } export function changeConfigValue(text: string, index: number, value: string): string { // Keep every separator, comment and unrelated line byte-for-byte intact. const parts = text.split(/(\r?\n)/); - const entry = configLines(text).find(line => line.index === index); + const entry = configLines(text).find((line) => line.index === index); if (entry) parts[index * 2] = entry.prefix + value.replace(/[\r\n]/g, ''); return parts.join(''); } /** Add one assignment without changing existing commands or line endings. */ export function addConfigSetting(text: string, key: string, value: string): string { - key = key.trim(); value = value.trim(); - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) throw new Error('Use a setting name containing letters, numbers, and underscores.'); - if (!value || /[\r\n]/.test(value) || value.startsWith('//')) throw new Error('Enter a value on one line.'); - if (configLines(text).some(entry => entry.key.toLowerCase() === key.toLowerCase())) throw new Error('This setting already exists. Find it above to edit its value.'); + key = key.trim(); + value = value.trim(); + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) + throw new Error('Use a setting name containing letters, numbers, and underscores.'); + if (!value || /[\r\n]/.test(value) || value.startsWith('//')) + throw new Error('Enter a value on one line.'); + if (configLines(text).some((entry) => entry.key.toLowerCase() === key.toLowerCase())) + throw new Error('This setting already exists. Find it above to edit its value.'); const newline = text.includes('\r\n') ? '\r\n' : '\n'; const formatted = /\s/.test(value) && !value.startsWith('"') ? JSON.stringify(value) : value; - return text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline; + return ( + text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline + ); } export function configWarnings(text: string, overlay: boolean): string[] { @@ -61,15 +73,20 @@ export function configWarnings(text: string, overlay: boolean): string[] { const seen = new Map(); for (const entry of configLines(text)) { const key = entry.key.toLowerCase(); - if (seen.has(key)) warnings.push(`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`); + if (seen.has(key)) + warnings.push( + `Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.` + ); seen.set(key, entry.index + 1); - if (!entry.value.trim() || entry.value.trim().startsWith('//')) warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`); + if (!entry.value.trim() || entry.value.trim().startsWith('//')) + warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`); } - if (overlay) text.split(/\r?\n/).forEach((line, index) => { - if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) { - warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`); - } - }); + if (overlay) + text.split(/\r?\n/).forEach((line, index) => { + if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) { + warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`); + } + }); if (/^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/im.test(text)) { warnings.push('For recognized maps, the server’s selected map overrides sv_maprotation.'); } diff --git a/src/lib/deployment-plan.ts b/src/lib/deployment-plan.ts index ecb8acc..71d9618 100644 --- a/src/lib/deployment-plan.ts +++ b/src/lib/deployment-plan.ts @@ -55,9 +55,21 @@ export interface CollectionPreview { mode: CollectionMode; retainedServers: string[]; removals: { name: string; reason: string }[]; - entries: { count: number; existing: number; missing: number; existingServers: string[]; - coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[]; - servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[]; + entries: { + count: number; + existing: number; + missing: number; + existingServers: string[]; + coverage: { name: string; health: string; href: string }[]; + recipe: LaunchRecipeSummary; + }[]; + servers: { + native?: import('./hosts.js').NativePlacement; + entryIndex: number; + name: string | null; + port: number | null; + hasSavedKey: boolean; + }[]; blockers: string[]; warnings?: string[]; availability?: LaunchAvailability; @@ -87,10 +99,32 @@ export interface CollectionDetail { canCancel: boolean; retainedServers: string[]; retainedServerDeletedAt?: Record; - queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number }; - servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string; - started_at: string; finished_at: string | null; error: string | null; server_exists: number; server_deleted_at?: string | null; queuePosition: number | null; - cancellation: { cleanupId: string; state: string; stage: string; error: string | null } | null }[]; + queue: { + concurrency: number; + active: { id: string; server: string; state: string; stage: string }[]; + queued: number; + }; + servers: { + id: string; + server: string; + action: 'create' | 'destroy'; + port: number | null; + map_slug: string | null; + state: string; + stage: string; + started_at: string; + finished_at: string | null; + error: string | null; + server_exists: number; + server_deleted_at?: string | null; + queuePosition: number | null; + cancellation: { + cleanupId: string; + state: string; + stage: string; + error: string | null; + } | null; + }[]; } export interface LaunchPreview { diff --git a/src/lib/fleet-overview.ts b/src/lib/fleet-overview.ts index af02b98..51e2b87 100644 --- a/src/lib/fleet-overview.ts +++ b/src/lib/fleet-overview.ts @@ -9,7 +9,10 @@ import { mapTitle, serverTitle } from './ui/server-labels.js'; export interface FleetServer { plutonium?: import('./plutonium.js').PlutoniumVersion; host_id?: string | null; - host?: Pick | null; + host?: Pick< + import('./hosts.js').Host, + 'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat' + > | null; container_state?: string | null; container_checked_at?: string | null; last_checked_at?: string | null; @@ -56,49 +59,103 @@ export type FleetFilter = 'all' | 'occupied' | 'empty' | 'attention' | 'testing' export type FleetSort = 'name' | 'players' | 'round' | 'duration'; export function freshServer(server: FleetServer, now: number): boolean { - return server.health === 'online' && !server.last_error && freshStatusTimestamp(server.last_seen_at, now); + return ( + server.health === 'online' && + !server.last_error && + freshStatusTimestamp(server.last_seen_at, now) + ); } export function fleetIdentity(server: FleetServer) { const parsed = parseServerName(server.name); const game = server.game ?? parsed?.game ?? ''; const mode = server.game_mode ?? parsed?.gameMode; - const selection = !mode || mode === 'zombie' - ? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '') : undefined; + const selection = + !mode || mode === 'zombie' + ? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '') + : undefined; const map = server.last_map ?? server.map; // A live map change must not keep the registered map's launch shortcut. - const matchingSelection = selection && (!map || selection.canonical === map) ? selection : undefined; + const matchingSelection = + selection && (!map || selection.canonical === map) ? selection : undefined; return { game, title: serverTitle(server.name), map: matchingSelection?.friendly ?? mapTitle(map), group: `${game}:${matchingSelection?.slug ?? map ?? 'unknown'}`, - launch: matchingSelection && (game === 't5' || game === 't6') - ? { game: game as ZombieGame, slug: matchingSelection.slug } : null + launch: + matchingSelection && (game === 't5' || game === 't6') + ? { game: game as ZombieGame, slug: matchingSelection.slug } + : null }; } -export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null { - if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry; - if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at - || server.game_ended_at || (server.last_players ?? 0) === 0) return null; +export function currentRound( + server: FleetServer, + report: GameTelemetry | undefined, + now: number +): number | null { + if ( + server.telemetry && + (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt)) + ) + report = server.telemetry; + if ( + !report || + (report.event !== 'round_change' && report.event !== 'heartbeat') || + !freshServer(server, now) || + !server.active_since || + !server.game_started_at || + server.game_ended_at || + (server.last_players ?? 0) === 0 + ) + return null; const received = Date.parse(report.receivedAt); - return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at) - && received >= Date.parse(server.active_since) ? report.round : null; + return freshTelemetry(report, now, server.telemetry_after) && + received >= Date.parse(server.game_started_at) && + received >= Date.parse(server.active_since) + ? report.round + : null; } -export function matchesFleet(server: FleetServer, query: string, filter: FleetFilter, game: string, - pinned: readonly string[], operations: readonly FleetOperation[], now: number): boolean { +export function matchesFleet( + server: FleetServer, + query: string, + filter: FleetFilter, + game: string, + pinned: readonly string[], + operations: readonly FleetOperation[], + now: number +): boolean { const identity = fleetIdentity(server); if (game && identity.game !== game) return false; const fresh = freshServer(server, now); if (filter === 'occupied' && !(fresh && (server.last_players ?? 0) > 0)) return false; if (filter === 'empty' && !(fresh && server.last_players === 0)) return false; - if (filter === 'attention' && !(server.health === 'offline' || server.health === 'stale' - || (server.health === 'online' && !fresh) || operations.some(op => op.server === server.name && op.state === 'attention'))) return false; + if ( + filter === 'attention' && + !( + server.health === 'offline' || + server.health === 'stale' || + (server.health === 'online' && !fresh) || + operations.some((op) => op.server === server.name && op.state === 'attention') + ) + ) + return false; if (filter === 'testing' && !server.held.length) return false; if (filter === 'pinned' && !pinned.includes(server.name)) return false; - const text = [server.name, identity.title, identity.map, identity.game, - ...(server.roster ?? []).map(player => player.name)].join(' ').toLocaleLowerCase(); - return query.trim().toLocaleLowerCase().split(/\s+/).every(term => text.includes(term)); + const text = [ + server.name, + identity.title, + identity.map, + identity.game, + ...(server.roster ?? []).map((player) => player.name) + ] + .join(' ') + .toLocaleLowerCase(); + return query + .trim() + .toLocaleLowerCase() + .split(/\s+/) + .every((term) => text.includes(term)); } diff --git a/src/lib/fleet-settings.ts b/src/lib/fleet-settings.ts index a2d44a8..79e3c69 100644 --- a/src/lib/fleet-settings.ts +++ b/src/lib/fleet-settings.ts @@ -33,7 +33,6 @@ export interface FleetSettings { lifecycleConcurrency: number; rebuildConcurrency: number; preparationConcurrency: number; - } export const defaultFleetSettings: FleetSettings = { @@ -70,7 +69,7 @@ export const defaultFleetSettings: FleetSettings = { destroyConcurrencyPerHost: 0, lifecycleConcurrency: 0, rebuildConcurrency: 0, - preparationConcurrency: 0, + preparationConcurrency: 0 }; export const fleetSettingLabels: Record = { @@ -107,12 +106,12 @@ export const fleetSettingLabels: Record = { destroyConcurrencyPerHost: 'Concurrent server removals per host', lifecycleConcurrency: 'Combined lifecycle limit', rebuildConcurrency: 'Concurrent rebuilds', - preparationConcurrency: 'Hosts preparing game files simultaneously', - + preparationConcurrency: 'Hosts preparing game files simultaneously' }; export function validateFleetSettings(input: unknown): FleetSettings { - if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide fleet settings.'); + if (!input || typeof input !== 'object' || Array.isArray(input)) + throw new Error('Provide fleet settings.'); const values = input as Record; for (const key of Object.keys(values)) { if (!Object.hasOwn(defaultFleetSettings, key)) throw new Error(`Unknown setting: ${key}`); @@ -121,20 +120,68 @@ export function validateFleetSettings(input: unknown): FleetSettings { const value = values[key]; const minimum = settingMinimum(key); if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { - throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`); + throw new Error( + `${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.` + ); } if (key.endsWith('Days') && value > 36500) { throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`); } } - return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings; + return Object.fromEntries( + Object.keys(defaultFleetSettings).map((key) => [key, values[key]]) + ) as unknown as FleetSettings; } -export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"]; -export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys]; -export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount']; +export const additionalConcurrencyKeys: (keyof FleetSettings)[] = [ + 'restartConcurrencyPerHost', + 'createConcurrencyPerHost', + 'destroyConcurrencyPerHost', + 'lifecycleConcurrency', + 'rebuildConcurrency', + 'preparationConcurrency' +]; +export const concurrencyKeys: (keyof FleetSettings)[] = [ + 'restartConcurrency', + 'createConcurrency', + 'destroyConcurrency', + ...additionalConcurrencyKeys +]; +export const retentionKeys: (keyof FleetSettings)[] = [ + 'deployKeepDays', + 'observationKeepDays', + 'provisioningKeepDays', + 'actionKeepDays', + 'failureKeepDays', + 'maintenanceKeepDays', + 'recoveryKeepDays', + 'hostResourceKeepDays', + 'instanceResourceKeepDays', + 'metricsKeepDays', + 'telemetryKeepDays', + 'telemetryMaxPerServer', + 'telemetryMaxTotal', + 'matchKeepDays', + 'betaKeepDays', + 'fileContentKeepDays', + 'payloadKeepDays', + 'versionKeepDays', + 'versionKeepCount', + 'gameFileGraceDays', + 'gameFileKeepCount', + 'uploadKeepDays', + 'auditKeepDays', + 'historyKeepCount' +]; export function settingMinimum(key: keyof FleetSettings): number { - return ['restartConcurrency','createConcurrency','destroyConcurrency','gameFileKeepCount'].includes(key) ? 1 : 0; + return [ + 'restartConcurrency', + 'createConcurrency', + 'destroyConcurrency', + 'gameFileKeepCount' + ].includes(key) + ? 1 + : 0; } export interface SettingsGroup { @@ -146,34 +193,94 @@ export interface SettingsGroup { /** Presentation groups do not change the saved schema or the section PATCH API. */ export const settingsGroups: SettingsGroup[] = [ - { id: 'concurrency', title: 'Concurrency', description: 'Limit simultaneous server operations and game-file preparation.', keys: concurrencyKeys }, - { id: 'operation-history', title: 'Operation history', description: 'Campaigns, server operations, maintenance, and routine checks.', keys: [ - 'deployKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'observationKeepDays', 'failureKeepDays', - 'maintenanceKeepDays', 'recoveryKeepDays', 'betaKeepDays', 'historyKeepCount' - ] }, - { id: 'matches-players', title: 'Matches & players', description: 'Saved matches, player records, and leaderboard history.', keys: ['matchKeepDays'] }, - { id: 'telemetry-metrics', title: 'Telemetry & metrics', description: 'Game diagnostics, fleet activity, and host resource samples.', keys: [ - 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'metricsKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays' - ] }, - { id: 'files-versions', title: 'Files & versions', description: 'File snapshots, saved versions, game files, and abandoned uploads.', keys: [ - 'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays' - ] }, - { id: 'accounts-audit', title: 'Accounts & audit', description: 'Sign-in and account administration records.', keys: ['auditKeepDays'] } + { + id: 'concurrency', + title: 'Concurrency', + description: 'Limit simultaneous server operations and game-file preparation.', + keys: concurrencyKeys + }, + { + id: 'operation-history', + title: 'Operation history', + description: 'Campaigns, server operations, maintenance, and routine checks.', + keys: [ + 'deployKeepDays', + 'provisioningKeepDays', + 'actionKeepDays', + 'observationKeepDays', + 'failureKeepDays', + 'maintenanceKeepDays', + 'recoveryKeepDays', + 'betaKeepDays', + 'historyKeepCount' + ] + }, + { + id: 'matches-players', + title: 'Matches & players', + description: 'Saved matches, player records, and leaderboard history.', + keys: ['matchKeepDays'] + }, + { + id: 'telemetry-metrics', + title: 'Telemetry & metrics', + description: 'Game diagnostics, fleet activity, and host resource samples.', + keys: [ + 'telemetryKeepDays', + 'telemetryMaxPerServer', + 'telemetryMaxTotal', + 'metricsKeepDays', + 'hostResourceKeepDays', + 'instanceResourceKeepDays' + ] + }, + { + id: 'files-versions', + title: 'Files & versions', + description: 'File snapshots, saved versions, game files, and abandoned uploads.', + keys: [ + 'fileContentKeepDays', + 'payloadKeepDays', + 'versionKeepDays', + 'versionKeepCount', + 'gameFileGraceDays', + 'gameFileKeepCount', + 'uploadKeepDays' + ] + }, + { + id: 'accounts-audit', + title: 'Accounts & audit', + description: 'Sign-in and account administration records.', + keys: ['auditKeepDays'] + } ]; /** Zero has a different meaning for age limits, count caps, and minimums. */ export function settingZeroMode(key: keyof FleetSettings): string | null { if (settingMinimum(key) > 0) return null; - if (additionalConcurrencyKeys.includes(key) || key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal') return 'No limit'; + if ( + additionalConcurrencyKeys.includes(key) || + key === 'telemetryMaxPerServer' || + key === 'telemetryMaxTotal' + ) + return 'No limit'; if (key === 'versionKeepDays') return 'No age expiry'; if (key === 'gameFileGraceDays') return 'No delay'; if (key === 'versionKeepCount' || key === 'historyKeepCount') return 'No minimum'; return key.endsWith('Days') ? 'Forever' : null; } -export function settingUnit(key: keyof FleetSettings): 'days' | 'records' | 'versions' | 'operations' | 'hosts' { +export function settingUnit( + key: keyof FleetSettings +): 'days' | 'records' | 'versions' | 'operations' | 'hosts' { if (key.endsWith('Days')) return 'days'; - if (key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal' || key === 'historyKeepCount') return 'records'; + if ( + key === 'telemetryMaxPerServer' || + key === 'telemetryMaxTotal' || + key === 'historyKeepCount' + ) + return 'records'; if (key === 'versionKeepCount' || key === 'gameFileKeepCount') return 'versions'; return key === 'preparationConcurrency' ? 'hosts' : 'operations'; } @@ -188,38 +295,64 @@ export function settingDisplayValue(key: keyof FleetSettings, value: number): st } export const settingHelp: Partial> = { - provisioningKeepDays: 'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.', - actionKeepDays: 'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.', - failureKeepDays: 'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.', - maintenanceKeepDays: 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.', - recoveryKeepDays: 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.', + provisioningKeepDays: + 'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.', + actionKeepDays: + 'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.', + failureKeepDays: + 'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.', + maintenanceKeepDays: + 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.', + recoveryKeepDays: + 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.', hostResourceKeepDays: 'Host CPU and memory history.', - instanceResourceKeepDays: 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.', + instanceResourceKeepDays: + 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.', metricsKeepDays: 'Historical fleet occupancy and utilization samples.', - telemetryKeepDays: 'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.', + telemetryKeepDays: + 'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.', telemetryMaxPerServer: 'Count cap per server, in addition to the age limit.', - telemetryMaxTotal: 'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.', - matchKeepDays: 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.', - betaKeepDays: 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.', - fileContentKeepDays: 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.', - payloadKeepDays: 'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.', - versionKeepDays: 'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.', - versionKeepCount: 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.', - gameFileGraceDays: 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.', - gameFileKeepCount: 'Keep this many recent published versions per game, in addition to every referenced or protected version.', - uploadKeepDays: 'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.', + telemetryMaxTotal: + 'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.', + matchKeepDays: + 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.', + betaKeepDays: + 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.', + fileContentKeepDays: + 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.', + payloadKeepDays: + 'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.', + versionKeepDays: + 'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.', + versionKeepCount: + 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.', + gameFileGraceDays: + 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.', + gameFileKeepCount: + 'Keep this many recent published versions per game, in addition to every referenced or protected version.', + uploadKeepDays: + 'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.', auditKeepDays: 'Recorded authentication and account-administration events.', - historyKeepCount: 'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.', + historyKeepCount: + 'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.', - restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.', - createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.', + restartConcurrency: + 'Fleet recovery slots stay occupied until container and game health are verified.', + createConcurrency: + 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.', destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.', - restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.', + restartConcurrencyPerHost: + 'Additional recovery limit on each host, including starts and rebuilds.', createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.', - destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.', - lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.', + destroyConcurrencyPerHost: + 'Additional removal limit on each host, including migration source removal.', + lifecycleConcurrency: + 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.', rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.', - preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.', - deployKeepDays: 'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.', - observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained.' + preparationConcurrency: + 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.', + deployKeepDays: + 'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.', + observationKeepDays: + 'Successful monitoring and file-read records. Failures and actions are retained.' }; diff --git a/src/lib/game-chat.ts b/src/lib/game-chat.ts index 4f9e97e..77194db 100644 --- a/src/lib/game-chat.ts +++ b/src/lib/game-chat.ts @@ -45,12 +45,20 @@ function logSeconds(time: string): number { } function plainText(value: string): string { - return value.replace(/\^[0-9]/g, '').replace(/[\x00-\x1f\x7f]/g, '').trim(); + return ( + value + .replace(/\^[0-9]/g, '') + // eslint-disable-next-line no-control-regex -- Strip control characters from displayed chat text. + .replace(/[\x00-\x1f\x7f]/g, '') + .trim() + ); } /** Preserve log order: elapsed timestamps can reset between matches. */ export function parseGameChat(lines: readonly string[]): ChatMessage[] { - return parseGameChatTimeline(lines).flatMap(entry => entry.type === 'message' ? [entry.message] : []); + return parseGameChatTimeline(lines).flatMap((entry) => + entry.type === 'message' ? [entry.message] : [] + ); } /** Keep match boundaries alongside messages, including when the log clock resets. */ @@ -89,8 +97,16 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde if (!match) continue; const message = plainText(match[5]); if (!message) continue; - push({ type: 'message', message: { time: match[1], guid: match[2], slot: Number(match[3]), - player: plainText(match[4]) || 'Unknown player', message } }); + push({ + type: 'message', + message: { + time: match[1], + guid: match[2], + slot: Number(match[3]), + player: plainText(match[4]) || 'Unknown player', + message + } + }); } return entries; } @@ -100,11 +116,14 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde * segment to the file's last write, then join observed gameplay at that time. * Resets have unknown downtime, so earlier segments cannot share this anchor. */ -export function enrichGameChatTimeline(lines: readonly string[], options: { - mtime: string | null; - matches: readonly ChatMatch[]; - reports: readonly ChatRoundReport[]; -}): ContextualChatEntry[] { +export function enrichGameChatTimeline( + lines: readonly string[], + options: { + mtime: string | null; + matches: readonly ChatMatch[]; + reports: readonly ChatRoundReport[]; + } +): ContextualChatEntry[] { let segmentStart = 0; let previousSeconds: number | null = null; let lastLine = -1; @@ -115,30 +134,43 @@ export function enrichGameChatTimeline(lines: readonly string[], options: { const event = line.slice(timestamp?.[0].length ?? 0).trimStart(); const lifecycle = /^(?:InitGame|ExitLevel|ShutdownGame)(?:\s|:|$)/.test(event); if (!timestamp) { - if (lifecycle) { segmentStart = index + 1; previousSeconds = null; } + if (lifecycle) { + segmentStart = index + 1; + previousSeconds = null; + } continue; } const seconds = logSeconds(timestamp[1]); - if (previousSeconds !== null && (seconds < previousSeconds - || (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))) segmentStart = index; + if ( + previousSeconds !== null && + (seconds < previousSeconds || (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event))) + ) + segmentStart = index; previousSeconds = seconds; } // An untimed final write cannot locate the preceding timed event in real time. const finalTimestamp = lastLine < 0 ? null : logTimestamp(lines[lastLine]); const writtenAt = options.mtime ? Date.parse(options.mtime) : NaN; const offset = finalTimestamp ? writtenAt - logSeconds(finalTimestamp[1]) * 1000 : NaN; - const reports = options.reports.map(report => ({ ...report, at: Date.parse(report.receivedAt) })) + const reports = options.reports + .map((report) => ({ ...report, at: Date.parse(report.receivedAt) })) .sort((a, b) => b.at - a.at); - const matches = options.matches.map(match => ({ ...match, start: Date.parse(match.startedAt) })) + const matches = options.matches + .map((match) => ({ ...match, start: Date.parse(match.startedAt) })) .sort((a, b) => b.start - a.start) .map((match, index, sorted) => { const nextStart = sorted[index - 1]?.start ?? Infinity; // Identical heartbeats are deduplicated by the leaderboard, but still // establish that this match was running when the message was sent. - const latest = reports.find(report => report.at >= match.start && report.at < nextStart); + const latest = reports.find( + (report) => report.at >= match.start && report.at < nextStart + ); const lastSeen = Math.max(Date.parse(match.lastReceivedAt), latest?.at ?? -Infinity); - const end = match.endedAt ? Date.parse(match.endedAt) - : match.status === 'active' ? lastSeen + 120_000 : Date.parse(match.lastReceivedAt); + const end = match.endedAt + ? Date.parse(match.endedAt) + : match.status === 'active' + ? lastSeen + 120_000 + : Date.parse(match.lastReceivedAt); return { ...match, end: Math.min(end, nextStart) }; }); let logStartedAt: number | null = null; @@ -147,8 +179,10 @@ export function enrichGameChatTimeline(lines: readonly string[], options: { return indexedTimeline(lines).map(({ entry, lineIndex }) => { const context: ChatContext = { realTime: null, matchElapsedMs: null, round: null }; const time = entry.type === 'message' ? entry.message.time : entry.time; - const at = time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN; - if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime())) context.realTime = new Date(at).toISOString(); + const at = + time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN; + if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime())) + context.realTime = new Date(at).toISOString(); if (entry.type === 'boundary' && entry.event !== 'end') { // The first event after an implicit reset can be several seconds into // the new process. Its clock origin is the lower bound, not that event. @@ -156,20 +190,44 @@ export function enrichGameChatTimeline(lines: readonly string[], options: { logStartedAt = context.realTime ? (entry.event === 'reset' ? offset : at) : Infinity; logEnded = false; } - const match = context.realTime && !logEnded ? matches.find(match => at >= match.start && at <= match.end - // Allow the log's one-second precision at a start, but never cross a later InitGame. - && (logStartedAt === null || match.start + 1000 >= logStartedAt - || reports.some(report => report.event === 'game_start' && report.playerCount > 0 - && report.at >= match.start && report.at <= at && report.at + 1000 >= logStartedAt!))) : undefined; + const match = + context.realTime && !logEnded + ? matches.find( + (match) => + at >= match.start && + at <= match.end && + // Allow the log's one-second precision at a start, but never cross a later InitGame. + (logStartedAt === null || + match.start + 1000 >= logStartedAt || + reports.some( + (report) => + report.event === 'game_start' && + report.playerCount > 0 && + report.at >= match.start && + report.at <= at && + report.at + 1000 >= logStartedAt! + )) + ) + : undefined; if (match) { - const report = reports.find(report => report.at <= at && report.at >= match.start); + const report = reports.find((report) => report.at <= at && report.at >= match.start); if (!report || (report.event !== 'game_end' && report.playerCount > 0)) { // A repeated start updates the fleet timer even if the leaderboard // deduplicates it. Use that same observed start for later messages. - const start = reports.find(report => report.event === 'game_start' && report.playerCount > 0 - && report.at >= match.start && report.at <= at)?.at ?? (match.startObserved ? match.start : null); + const start = + reports.find( + (report) => + report.event === 'game_start' && + report.playerCount > 0 && + report.at >= match.start && + report.at <= at + )?.at ?? (match.startObserved ? match.start : null); if (start !== null) context.matchElapsedMs = at - start; - if (report && at - report.at <= 120_000 && (report.event === 'round_change' || report.event === 'heartbeat')) { + if ( + report && + at - report.at <= 120_000 && + (report.event === 'round_change' || report.event === 'heartbeat') + ) { context.round = report.round; } } diff --git a/src/lib/game-files.ts b/src/lib/game-files.ts index c23d3a1..9241101 100644 --- a/src/lib/game-files.ts +++ b/src/lib/game-files.ts @@ -4,33 +4,65 @@ export const GAME_FILE_GAMES = [ { id: 't6', name: 'Black Ops II' } ] as const; export type GameLibraryState = 'importing' | 'ready' | 'failed'; -export type LibraryCacheState = 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; +export type LibraryCacheState = + 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; -export interface GameFileEntry { path: string; size: number; sha256: string } +export interface GameFileEntry { + path: string; + size: number; + sha256: string; +} export interface LibraryTransferProgress { - phase: 'downloading' | 'verifying'; startedAt: string; - elapsedSeconds: number; phaseElapsedSeconds: number; - bytesPerSecond: number | null; etaSeconds: number | null; + phase: 'downloading' | 'verifying'; + startedAt: string; + elapsedSeconds: number; + phaseElapsedSeconds: number; + bytesPerSecond: number | null; + etaSeconds: number | null; } export interface HostLibraryCache { - hostId: string; libraryVersionId: string; status: LibraryCacheState; - bytesDownloaded: number; bytesTotal: number; manifestSha256: string | null; - error: string | null; updatedAt: string; operationId: string | null; - bytesVerified?: number; transfer?: LibraryTransferProgress; + hostId: string; + libraryVersionId: string; + status: LibraryCacheState; + bytesDownloaded: number; + bytesTotal: number; + manifestSha256: string | null; + error: string | null; + updatedAt: string; + operationId: string | null; + bytesVerified?: number; + transfer?: LibraryTransferProgress; } export interface GameLibraryVersion { - id: string; libraryId: string; game: GameLibraryGame; version: string; - description: string; state: GameLibraryState; sizeBytes: number; - fileCount: number; manifestSha256: string | null; files: GameFileEntry[]; - createdAt: string; publishedAt: string | null; storageLocation: string; - sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null; + id: string; + libraryId: string; + game: GameLibraryGame; + version: string; + description: string; + state: GameLibraryState; + sizeBytes: number; + fileCount: number; + manifestSha256: string | null; + files: GameFileEntry[]; + createdAt: string; + publishedAt: string | null; + storageLocation: string; + sourceHostId: string | null; + sourceVolume: string | null; + error: string | null; + operationId: string | null; caches: HostLibraryCache[]; filesRemoved?: boolean; filesRemoving?: boolean; } export interface GameLibrary { - id: string; name: string; game: GameLibraryGame; description: string; - createdAt: string; latestVersionId: string | null; versions: GameLibraryVersion[]; + id: string; + name: string; + game: GameLibraryGame; + description: string; + createdAt: string; + latestVersionId: string | null; + versions: GameLibraryVersion[]; } export function formatGameBytes(bytes: number): string { diff --git a/src/lib/host-operation-labels.ts b/src/lib/host-operation-labels.ts index d5a471f..52ef0e6 100644 --- a/src/lib/host-operation-labels.ts +++ b/src/lib/host-operation-labels.ts @@ -1,14 +1,28 @@ export const hostOperationTitles: Record = { - 'instance.create': 'Create server container', 'instance.rebuild': 'Rebuild server container', - 'instance.start': 'Start server container', 'instance.restart': 'Restart server container', - 'instance.stop': 'Stop server container', 'instance.remove': 'Destroy server container', - 'instance.cancel-create': 'Cancel server creation', 'instance.inspect': 'Check server container', - 'host.inspect': 'Check host', 'host.decommission': 'Remove host', 'runtime.pull': 'Prepare container runtime', - 'library.download': 'Download game files', 'library.import': 'Import game files', - 'plutonium.prepare': 'Prepare game update', 'files.stage': 'Stage server files', - 'files.apply': 'Apply server files', 'files.read': 'Read server files', 'files.list': 'List server files', - 'files.stat': 'Check server files', 'files.manifest': 'Check file versions', 'files.remove': 'Remove server files', - 'rcon.status-batch': 'Check game status', 'rcon.execute': 'Game command', 'logs.read': 'Read server logs' + 'instance.create': 'Create server container', + 'instance.rebuild': 'Rebuild server container', + 'instance.start': 'Start server container', + 'instance.restart': 'Restart server container', + 'instance.stop': 'Stop server container', + 'instance.remove': 'Destroy server container', + 'instance.cancel-create': 'Cancel server creation', + 'instance.inspect': 'Check server container', + 'host.inspect': 'Check host', + 'host.decommission': 'Remove host', + 'runtime.pull': 'Prepare container runtime', + 'library.download': 'Download game files', + 'library.import': 'Import game files', + 'plutonium.prepare': 'Prepare game update', + 'files.stage': 'Stage server files', + 'files.apply': 'Apply server files', + 'files.read': 'Read server files', + 'files.list': 'List server files', + 'files.stat': 'Check server files', + 'files.manifest': 'Check file versions', + 'files.remove': 'Remove server files', + 'rcon.status-batch': 'Check game status', + 'rcon.execute': 'Game command', + 'logs.read': 'Read server logs' }; export function hostOperationTitle(kind: string): string { diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index 633826e..b47e158 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -1,31 +1,67 @@ import type { HostLibraryCache } from './game-files.js'; export type HostMode = 'active' | 'draining' | 'disabled'; -export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting'; -export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown'; +export type HostReadiness = + | 'preparing' + | 'needs-runtime' + | 'needs-game-files' + | 'ready' + | 'degraded' + | 'draining' + | 'disabled' + | 'deleting'; +export type HostOperationState = + 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown'; export function hostReadinessLabel(readiness: HostReadiness): string { - return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files', - ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness]; + return { + preparing: 'Preparing', + 'needs-runtime': 'Needs runtime', + 'needs-game-files': 'Needs game files', + ready: 'Ready', + degraded: 'Degraded', + draining: 'Draining', + disabled: 'Disabled', + deleting: 'Deleting' + }[readiness]; } -export interface HostDeletion { allowed: boolean; blockers: string[] } +export interface HostDeletion { + allowed: boolean; + blockers: string[]; +} export interface HostDecommission { - operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed'; - requestedAt: string; updatedAt: string; error: string | null; retryable: boolean; + operationId: string; + stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed'; + requestedAt: string; + updatedAt: string; + error: string | null; + retryable: boolean; } export function hostDeletionMessage(host: Pick): string { if (!host.decommission) return ''; switch (host.decommission.stage) { - case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.'; - case 'running': return 'Removing host files, cached game libraries, and the agent installation.'; - case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.'; - case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.'; - case 'completed': return 'Host files and agent removal confirmed.'; + case 'queued': + return host.connected + ? 'Host cleanup is queued.' + : 'Waiting for the host to reconnect before removing its files and agent.'; + case 'running': + return 'Removing host files, cached game libraries, and the agent installation.'; + case 'cleaned': + return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.'; + case 'failed': + return 'Cleanup needs attention. Complete removal has not been confirmed.'; + case 'completed': + return 'Host files and agent removal confirmed.'; } } -export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string } +export interface HostLibrary { + id: string; + state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; + progress?: number; + error?: string; +} export interface HostAgentUpdate { enabled: boolean; state: 'current' | 'checking' | 'deferred' | 'updating' | 'updated' | 'failed'; @@ -37,50 +73,93 @@ export interface HostAgentUpdate { export interface HostAgentLane { name: 'health-1' | 'health-2' | 'read' | 'write'; - alive: boolean; busy: boolean; lastPollAt: string | null; pollAgeSeconds: number; + alive: boolean; + busy: boolean; + lastPollAt: string | null; + pollAgeSeconds: number; } export function hostAgentLaneLabel(name: HostAgentLane['name']): string { - return { 'health-1': 'Health check 1', 'health-2': 'Health check 2', read: 'File reads', write: 'Actions' }[name]; + return { + 'health-1': 'Health check 1', + 'health-2': 'Health check 2', + read: 'File reads', + write: 'Actions' + }[name]; } -export function hostAgentLaneState(lane: HostAgentLane): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' { +export function hostAgentLaneState( + lane: HostAgentLane +): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' { if (!lane.alive) return 'stopped'; // Reads and mutations can legitimately take minutes. Health work has a // bounded budget; an old poll there indicates a blocked worker even if busy. - if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-'))) return 'stalled'; + if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-'))) + return 'stalled'; return lane.busy ? 'busy' : lane.lastPollAt === null ? 'starting' : 'polling'; } export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean { - return inventory.agentUpdate?.enabled === true && inventory.capabilities?.includes('agent.update') === true; + return ( + inventory.agentUpdate?.enabled === true && + inventory.capabilities?.includes('agent.update') === true + ); } export function hostAgentUpdateLabel(inventory: HostInventory): string { if (!hostAgentUpdatesEnabled(inventory)) return 'Setup required'; const update = inventory.agentUpdate!; if (update.state === 'current' && !update.checkedAt) return 'Awaiting first check'; - return { current: 'Current', checking: 'Checking', deferred: 'Deferred', - updating: 'Updating', updated: 'Updated', failed: 'Needs attention' }[update.state]; + return { + current: 'Current', + checking: 'Checking', + deferred: 'Deferred', + updating: 'Updating', + updated: 'Updated', + failed: 'Needs attention' + }[update.state]; } export function hostAgentUpdateMessage(inventory: HostInventory): string { - if (!hostAgentUpdatesEnabled(inventory)) return 'Run the current installer once on this host to enable automatic agent updates.'; + if (!hostAgentUpdatesEnabled(inventory)) + return 'Run the current installer once on this host to enable automatic agent updates.'; const update = inventory.agentUpdate!; switch (update.state) { - case 'current': return update.checkedAt ? 'The last check found no newer compatible agent release.' : 'The host timer will check for a compatible agent release automatically.'; - case 'checking': return 'The host is checking the controller release and verifying its download.'; - case 'deferred': return 'The updater is waiting for the reported condition to clear. The host checks again automatically.'; - case 'updating': return 'The host is replacing agent code and checking that the new process connects successfully.'; - case 'updated': return 'The new agent connected successfully. Automatic checks continue.'; - case 'failed': return 'The update did not complete successfully. Review the reported error before retrying setup.'; + case 'current': + return update.checkedAt + ? 'The last check found no newer compatible agent release.' + : 'The host timer will check for a compatible agent release automatically.'; + case 'checking': + return 'The host is checking the controller release and verifying its download.'; + case 'deferred': + return 'The updater is waiting for the reported condition to clear. The host checks again automatically.'; + case 'updating': + return 'The host is replacing agent code and checking that the new process connects successfully.'; + case 'updated': + return 'The new agent connected successfully. Automatic checks continue.'; + case 'failed': + return 'The update did not complete successfully. Review the reported error before retrying setup.'; } } export interface HostInventory { - hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string; - cpu?: string; cpuCount?: number; cpuUsagePercent?: number; memoryTotal?: number; memoryAvailable?: number; - storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean; - directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[]; - currentOperation?: string | null; [key: string]: unknown; + hostname?: string; + agentVersion?: string; + protocolVersion?: number; + os?: string; + architecture?: string; + cpu?: string; + cpuCount?: number; + cpuUsagePercent?: number; + memoryTotal?: number; + memoryAvailable?: number; + storageTotal?: number; + storageAvailable?: number; + dockerVersion?: string; + dockerRunning?: boolean; + directoriesWritable?: boolean; + runtimeImages?: string[]; + libraries?: HostLibrary[]; + currentOperation?: string | null; + [key: string]: unknown; capabilities?: string[]; plutoniumCache?: { revision: number; bytes: number }; agentUpdate?: HostAgentUpdate; @@ -90,37 +169,82 @@ export interface HostInventory { instanceStates?: InstanceState[]; } export interface InstanceState { - instanceId: string; generation: number; state: string; healthy: boolean; observedAt: string; + instanceId: string; + generation: number; + state: string; + healthy: boolean; + observedAt: string; } export interface InstanceResources { - instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number; + instanceId: string; + generation: number; + uptimeSeconds: number; + cpuCores: number; + memoryBytes: number; } export interface ResourceAllowance { - cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap'; - samples: number; observedAt: string | null; + cpuCores: number; + memoryBytes: number; + source: 'host' | 'fleet' | 'bootstrap'; + samples: number; + observedAt: string | null; +} +export interface CapacityCalibration { + running: ResourceAllowance; + startup: ResourceAllowance; } -export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance } export interface CapacitySample { - hostId: string; hostName: string | null; instanceId: string; server: string | null; - generation: number; phase: 'startup' | 'running'; timestamp: string; cpuCores: number; memoryBytes: number; + hostId: string; + hostName: string | null; + instanceId: string; + server: string | null; + generation: number; + phase: 'startup' | 'running'; + timestamp: string; + cpuCores: number; + memoryBytes: number; } export interface CapacitySamplePage { - samples: CapacitySample[]; total: number; page: number; pageSize: number; + samples: CapacitySample[]; + total: number; + page: number; + pageSize: number; } export interface Host { - id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; - bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string; - lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline'; - readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory; - enrolled: boolean; credentialActive: boolean; + id: string; + name: string; + hostname: string; + mode: HostMode; + portMin: number; + portMax: number; + bindAddress: string; + publicAddress: string; + minimumFreeBytes: number; + createdAt: string; + lastHeartbeat: string | null; + connected: boolean; + connectionState: 'online' | 'offline'; + readiness: HostReadiness; + readinessReasons: string[]; + inventory: HostInventory; + enrolled: boolean; + credentialActive: boolean; decommission: HostDecommission | null; preparation?: HostPreparation; enablingSince?: string | null; } export interface HostOperation { - id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null; - payload: Record; state: HostOperationState; result: unknown; error: string | null; - createdAt: string; updatedAt: string; + id: string; + hostId: string; + kind: string; + instanceId: string | null; + generation: number | null; + payload: Record; + state: HostOperationState; + result: unknown; + error: string | null; + createdAt: string; + updatedAt: string; } export interface HostOperationSummary extends Omit { hostName: string | null; @@ -130,11 +254,16 @@ export interface HostOperationSummary extends Omit>; image: string; - bindAddress?: string; publicAddress?: string; publicPort?: number; + libraryIds: Partial>; + image: string; + bindAddress?: string; + publicAddress?: string; + publicPort?: number; } /** Queued automatic launches have no assigned host or port until provisioning. */ export interface NativePlacement { - hostId?: string; libraryId: string; image?: string; - bindAddress?: string; publicAddress?: string; publicPort?: number; + hostId?: string; + libraryId: string; + image?: string; + bindAddress?: string; + publicAddress?: string; + publicPort?: number; automatic?: boolean; network?: { bindAddress?: string; publicAddress?: string; publicPort?: number }; } diff --git a/src/lib/leaderboard-efficiency.ts b/src/lib/leaderboard-efficiency.ts index fe68e3d..cee00d8 100644 --- a/src/lib/leaderboard-efficiency.ts +++ b/src/lib/leaderboard-efficiency.ts @@ -5,20 +5,40 @@ export const EFFICIENCY_MIN_KILLS = 100; export const EFFICIENCY_MIN_DOWNS = 5; export const efficiencyMetrics = [ - { key: 'headshotRate', label: 'Headshots / kills', format: 'percent', - description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.` }, - { key: 'killsPerDown', label: 'Kills per down', format: 'number', - description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.` }, - { key: 'averageKills', label: 'Average kills', format: 'number', - description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` }, - { key: 'averageRounds', label: 'Average rounds', format: 'number', - description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` }, - { key: 'completionRate', label: 'Completion rate', format: 'percent', - description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.` } + { + key: 'headshotRate', + label: 'Headshots / kills', + format: 'percent', + description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.` + }, + { + key: 'killsPerDown', + label: 'Kills per down', + format: 'number', + description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.` + }, + { + key: 'averageKills', + label: 'Average kills', + format: 'number', + description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` + }, + { + key: 'averageRounds', + label: 'Average rounds', + format: 'number', + description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` + }, + { + key: 'completionRate', + label: 'Completion rate', + format: 'percent', + description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.` + } ] as const; -export type EfficiencyMetric = typeof efficiencyMetrics[number]['key']; -const metrics = new Set(efficiencyMetrics.map(metric => metric.key)); +export type EfficiencyMetric = (typeof efficiencyMetrics)[number]['key']; +const metrics = new Set(efficiencyMetrics.map((metric) => metric.key)); export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric { return metrics.has(metric); @@ -26,11 +46,16 @@ export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric { function sample(stats: PlayerStats, metric: EfficiencyMetric): [number, number] { switch (metric) { - case 'headshotRate': return [stats.headshots, stats.kills]; - case 'killsPerDown': return [stats.kills, stats.downs]; - case 'averageKills': return [stats.kills, stats.gamesPlayed]; - case 'averageRounds': return [stats.roundsSurvived, stats.gamesPlayed]; - case 'completionRate': return [stats.gamesCompleted, stats.gamesPlayed]; + case 'headshotRate': + return [stats.headshots, stats.kills]; + case 'killsPerDown': + return [stats.kills, stats.downs]; + case 'averageKills': + return [stats.kills, stats.gamesPlayed]; + case 'averageRounds': + return [stats.roundsSurvived, stats.gamesPlayed]; + case 'completionRate': + return [stats.gamesCompleted, stats.gamesPlayed]; } } @@ -60,11 +85,16 @@ const count = (value: number, singular: string, plural = `${singular}s`) => export function efficiencySample(stats: PlayerStats, metric: EfficiencyMetric): string { switch (metric) { - case 'headshotRate': return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`; - case 'killsPerDown': return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`; - case 'averageKills': return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`; - case 'averageRounds': return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`; - case 'completionRate': return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`; + case 'headshotRate': + return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`; + case 'killsPerDown': + return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`; + case 'averageKills': + return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`; + case 'averageRounds': + return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`; + case 'completionRate': + return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`; } } diff --git a/src/lib/leaderboard-navigation.ts b/src/lib/leaderboard-navigation.ts index ff23cb6..aad2b66 100644 --- a/src/lib/leaderboard-navigation.ts +++ b/src/lib/leaderboard-navigation.ts @@ -12,10 +12,12 @@ export function leaderboardReturnHref(value: string | null | undefined): string const params = new URLSearchParams(); for (const key of contextKeys) { const entry = url.searchParams.get(key); - if (entry || key === 'map' && entry !== null) params.set(key, entry!); + if (entry || (key === 'map' && entry !== null)) params.set(key, entry!); } return `/leaderboard${params.size ? `?${params}` : ''}`; - } catch { return '/leaderboard'; } + } catch { + return '/leaderboard'; + } } export function playerContextHref(guid: string, source: string): string { @@ -31,7 +33,8 @@ export function playerContextHref(guid: string, source: string): string { if (game === 't5' || game === 't6') params.set('game', game); if (map !== null && map.length <= 1024) params.set('map', map); if (players && /^[1-4]$/.test(players)) params.set('players', players); - if (metric && leaderboardMetrics.some(item => item.key === metric)) params.set('metric', metric); + if (metric && leaderboardMetrics.some((item) => item.key === metric)) + params.set('metric', metric); return `/leaderboard/players/${encodeURIComponent(guid)}?${params}`; } diff --git a/src/lib/leaderboard.ts b/src/lib/leaderboard.ts index b0145b6..4458a0a 100644 --- a/src/lib/leaderboard.ts +++ b/src/lib/leaderboard.ts @@ -14,8 +14,8 @@ export const playerStatMetrics = [ ] as const; export const leaderboardMetrics = [...playerStatMetrics, ...efficiencyMetrics] as const; -export type LeaderboardMetric = typeof leaderboardMetrics[number]['key']; -export type PlayerStatMetric = typeof playerStatMetrics[number]['key']; +export type LeaderboardMetric = (typeof leaderboardMetrics)[number]['key']; +export type PlayerStatMetric = (typeof playerStatMetrics)[number]['key']; export type PlayerStats = Record; export type PersonalBestMetric = 'highestRound' | 'bestKills' | 'bestScore'; diff --git a/src/lib/maintenance.ts b/src/lib/maintenance.ts index 7f963e6..a0284db 100644 --- a/src/lib/maintenance.ts +++ b/src/lib/maintenance.ts @@ -2,7 +2,12 @@ import type { WatchdogOverview } from './watchdog.js'; import type { PlutoniumStatus } from './plutonium.js'; export type MaintenancePolicy = 'health' | 'drift' | 'updates'; -export interface MaintenancePolicies { paused: boolean; health: boolean; drift: boolean; updates: boolean } +export interface MaintenancePolicies { + paused: boolean; + health: boolean; + drift: boolean; + updates: boolean; +} export interface MaintenanceOverview { policies: MaintenancePolicies; recovery: WatchdogOverview; diff --git a/src/lib/map-aliases.ts b/src/lib/map-aliases.ts index 22fed8a..f391292 100644 --- a/src/lib/map-aliases.ts +++ b/src/lib/map-aliases.ts @@ -36,14 +36,54 @@ export const T5_ZOMBIE_MAPS: readonly T5MapEntry[] = Object.freeze([ // Rulesets from xerxes-at/T6ServerConfigs, also seeded by Plutainer. // Several selections share an engine map; slug preserves the selected variant. export const T6_ZOMBIE_MAPS: readonly ZombieMapEntry[] = Object.freeze([ - { friendly: 'TranZit', slug: 'transit', canonical: 'zm_transit', gameSettings: 'zm_classic_transit.cfg' }, - { friendly: 'Town Survival', slug: 'town', canonical: 'zm_transit', gameSettings: 'zm_standard_town.cfg' }, - { friendly: 'Farm Survival', slug: 'farm', canonical: 'zm_transit', gameSettings: 'zm_standard_farm.cfg' }, - { friendly: 'Bus Depot Survival', slug: 'busdepot', canonical: 'zm_transit', gameSettings: 'zm_standard_transit.cfg' }, - { friendly: 'Nuketown', slug: 'nuked', canonical: 'zm_nuked', gameSettings: 'zm_standard_nuked.cfg' }, - { friendly: 'Die Rise', slug: 'highrise', canonical: 'zm_highrise', gameSettings: 'zm_classic_rooftop.cfg' }, - { friendly: 'Mob of the Dead', slug: 'prison', canonical: 'zm_prison', gameSettings: 'zm_classic_prison.cfg' }, - { friendly: 'Buried', slug: 'buried', canonical: 'zm_buried', gameSettings: 'zm_classic_processing.cfg' }, + { + friendly: 'TranZit', + slug: 'transit', + canonical: 'zm_transit', + gameSettings: 'zm_classic_transit.cfg' + }, + { + friendly: 'Town Survival', + slug: 'town', + canonical: 'zm_transit', + gameSettings: 'zm_standard_town.cfg' + }, + { + friendly: 'Farm Survival', + slug: 'farm', + canonical: 'zm_transit', + gameSettings: 'zm_standard_farm.cfg' + }, + { + friendly: 'Bus Depot Survival', + slug: 'busdepot', + canonical: 'zm_transit', + gameSettings: 'zm_standard_transit.cfg' + }, + { + friendly: 'Nuketown', + slug: 'nuked', + canonical: 'zm_nuked', + gameSettings: 'zm_standard_nuked.cfg' + }, + { + friendly: 'Die Rise', + slug: 'highrise', + canonical: 'zm_highrise', + gameSettings: 'zm_classic_rooftop.cfg' + }, + { + friendly: 'Mob of the Dead', + slug: 'prison', + canonical: 'zm_prison', + gameSettings: 'zm_classic_prison.cfg' + }, + { + friendly: 'Buried', + slug: 'buried', + canonical: 'zm_buried', + gameSettings: 'zm_classic_processing.cfg' + }, { friendly: 'Origins', slug: 'tomb', canonical: 'zm_tomb', gameSettings: 'zm_classic_tomb.cfg' } ]); @@ -55,23 +95,29 @@ export function zombieMapBySlug(game: string, slug: string): ZombieMapEntry | un return zombieMaps(game).find((entry) => entry.slug === slug.toLowerCase()); } -export function selectedZombieMap(game: string | null, map: string | null, slug?: string | null): ZombieMapEntry | undefined { +export function selectedZombieMap( + game: string | null, + map: string | null, + slug?: string | null +): ZombieMapEntry | undefined { const maps = zombieMaps(game); // T5 maps are unambiguous, including legacy friendly slugs. T6 variants // require the stored selection; never fall back from an unknown variant. - return maps.find((entry) => entry.canonical === map && ( - game !== 't6' || !slug || entry.slug === slug - || entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug - )); + return maps.find( + (entry) => + entry.canonical === map && + (game !== 't6' || + !slug || + entry.slug === slug || + entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug) + ); } export function mapRotation(entry: ZombieMapEntry): string { return `${entry.gameSettings ? `execgts ${entry.gameSettings} ` : ''}map ${entry.canonical}`; } -const BY_SLUG: ReadonlyMap = new Map( - T5_ZOMBIE_MAPS.map((m) => [m.slug, m]) -); +const BY_SLUG: ReadonlyMap = new Map(T5_ZOMBIE_MAPS.map((m) => [m.slug, m])); export function t5ZombieMapBySlug(slug: string): T5MapEntry | undefined { return BY_SLUG.get(slug.toLowerCase()); @@ -80,10 +126,10 @@ export function t5ZombieMapBySlug(slug: string): T5MapEntry | undefined { export type GameMode = 'zombie' | 'mp' | 'sp'; export interface ParsedServerName { - game: string; // 't5' - gameMode: GameMode | null; // 'zombie' — null when the second segment isn't a known mode - mapSlug: string; // 'cosmodrome' — joined without separators for lookup - mapCanonical: string | null;// 'zombie_cosmodrome' — null when slug not in catalog + game: string; // 't5' + gameMode: GameMode | null; // 'zombie' — null when the second segment isn't a known mode + mapSlug: string; // 'cosmodrome' — joined without separators for lookup + mapCanonical: string | null; // 'zombie_cosmodrome' — null when slug not in catalog lobby: number | null; } @@ -122,10 +168,11 @@ export function parseServerName(name: string): ParsedServerName | null { const mapSlug = slugParts.join('').toLowerCase(); const catalog = zombieMaps(game.toLowerCase()); - const entry = gameMode === null || gameMode === 'zombie' - ? catalog.find((m) => m.slug === mapSlug) - ?? catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug) - : undefined; + const entry = + gameMode === null || gameMode === 'zombie' + ? (catalog.find((m) => m.slug === mapSlug) ?? + catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug)) + : undefined; const mapCanonical = entry?.canonical ?? null; return { game, gameMode, mapSlug, mapCanonical, lobby }; } diff --git a/src/lib/match-history-query.ts b/src/lib/match-history-query.ts index 722e0dc..858e45d 100644 --- a/src/lib/match-history-query.ts +++ b/src/lib/match-history-query.ts @@ -1,12 +1,26 @@ export const matchPageSize = 25; export const matchSorts = ['started', 'round', 'duration', 'players'] as const; -export type MatchSort = typeof matchSorts[number]; +export type MatchSort = (typeof matchSorts)[number]; export interface MatchHistoryQuery { - game: string; map: string; q: string; days: number; + game: string; + map: string; + q: string; + days: number; status: '' | 'active' | 'completed' | 'interrupted'; - sort: MatchSort; direction: 'asc' | 'desc'; page: number; + sort: MatchSort; + direction: 'asc' | 'desc'; + page: number; } -export const matchQueryKeys = ['game', 'map', 'q', 'days', 'status', 'sort', 'direction', 'page'] as const; +export const matchQueryKeys = [ + 'game', + 'map', + 'q', + 'days', + 'status', + 'sort', + 'direction', + 'page' +] as const; export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery { const game = params.get('game') ?? ''; const map = params.get('map') ?? ''; @@ -16,11 +30,29 @@ export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery { const sort = params.get('sort') || 'started'; const direction = params.get('direction') || 'desc'; const page = Number(params.get('page') || 1); - if (!['', 't5', 't6'].includes(game) || map.length > 1024 || q.length > 100 - || ![0, 7, 30, 90].includes(days) || !['', 'active', 'completed', 'interrupted'].includes(status) - || !matchSorts.includes(sort as MatchSort) || !['asc', 'desc'].includes(direction) - || !Number.isSafeInteger(page) || page < 1 || page > 1000000) throw new Error('Invalid match history filters.'); - return { game, map, q, days, status: status as MatchHistoryQuery['status'], sort: sort as MatchSort, direction: direction as 'asc' | 'desc', page }; + if ( + !['', 't5', 't6'].includes(game) || + map.length > 1024 || + q.length > 100 || + ![0, 7, 30, 90].includes(days) || + !['', 'active', 'completed', 'interrupted'].includes(status) || + !matchSorts.includes(sort as MatchSort) || + !['asc', 'desc'].includes(direction) || + !Number.isSafeInteger(page) || + page < 1 || + page > 1000000 + ) + throw new Error('Invalid match history filters.'); + return { + game, + map, + q, + days, + status: status as MatchHistoryQuery['status'], + sort: sort as MatchSort, + direction: direction as 'asc' | 'desc', + page + }; } /** Namespaced filters also work inside an instance's Matches tab. */ export function matchApiParams(url: URL): URLSearchParams { @@ -31,13 +63,17 @@ export function matchApiParams(url: URL): URLSearchParams { } return params; } -export function matchHistoryHref(url: URL, changes: Partial>): string { +export function matchHistoryHref( + url: URL, + changes: Partial> +): string { const next = new URL(url); next.searchParams.delete('matches'); // Retire the former growing-list limit. next.searchParams.delete('matchPage'); for (const [key, value] of Object.entries(changes)) { const name = `match${key[0].toUpperCase()}${key.slice(1)}`; - if (value) next.searchParams.set(name, value); else next.searchParams.delete(name); + if (value) next.searchParams.set(name, value); + else next.searchParams.delete(name); } return next.pathname + next.search; } diff --git a/src/lib/metrics-insights.ts b/src/lib/metrics-insights.ts index 7d7adac..a9be523 100644 --- a/src/lib/metrics-insights.ts +++ b/src/lib/metrics-insights.ts @@ -23,51 +23,95 @@ export interface MetricRecommendation { /** Apply the same visible rules to all selected maps, including sparse observations. */ export function metricRecommendations(maps: MapMetrics[]): MetricRecommendation[] { - return maps.map(map => { - const utilization = map.observed > 0 ? map.active / map.observed : null; - const pressure = map.capacityObserved > 0 ? map.full / map.capacityObserved : null; - const knownShare = map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0; - const base = { - game: map.game, - map: map.map, - evidence: `${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` + - (pressure === null ? 'Capacity unavailable.' : `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`), - tentative: map.observed < HOSTING_RULES.minimumEvidenceHours || pressure === null || map.capacityObserved < HOSTING_RULES.minimumEvidenceHours || knownShare < 0.8 - }; - if (utilization === null) return { - ...base, kind: 'capacity' as const, title: map.unknown > 0 ? 'Check occupancy reporting' : 'Let observations accumulate', - reason: map.unknown > 0 - ? 'Captured status is unknown. Check a server’s telemetry or RCON diagnostics before choosing a hosting change.' - : 'This map has no elapsed observations in the selected window. Wait for another status check or try a wider range.', - outcome: 'Watch for known server time in the next report.', tentative: true - }; - if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion) return { - ...base, kind: 'expand' as const, title: 'Trial one additional lobby', - reason: 'Recorded lobbies were all full for at least 20% of known capacity time. Try one extra lobby during busy hours.', - outcome: 'Compare all-full time and use of the additional lobby over the next comparable period.' - }; - if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion) return { - ...base, kind: 'expand' as const, title: 'Trial an extra lobby during busy hours', - reason: 'Players were present for at least 80% of known server time. Capacity is unknown, so this experiment uses occupancy as a proxy, not evidence of saturation.', - outcome: 'Check whether players use the extra lobby, and measure capacity if the server reports it.', tentative: true - }; - if (utilization <= HOSTING_RULES.idleUtilization && (pressure === null || pressure < HOSTING_RULES.quietPressure)) return { - ...base, kind: 'consolidate' as const, title: 'Trial less idle hosting', - reason: 'Players were present for at most 10% of known time, with less than 1% measured pressure or unavailable capacity. Review one idle lobby or a shorter hosting window; preserve access to this map.', - outcome: 'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.' - }; - if (pressure === null) return { - ...base, kind: 'capacity' as const, title: 'Check capacity during busy hours', - reason: 'Occupancy is recorded, but player limits are unavailable. Check capacity reporting while keeping the current lineup.', - outcome: 'Look for known-capacity time and all-full periods before the next hosting trial.' - }; - return { - ...base, kind: 'watch' as const, title: 'Keep the lineup and watch peaks', - reason: 'These observations do not trigger the expansion or idle-hosting rules. Keep the current lineup for another comparison period.', - outcome: 'Watch whether occupied time and measured capacity pressure change.' - }; - }).sort((a, b) => { - const priority = { expand: 0, consolidate: 1, capacity: 2, watch: 3 }; - return priority[a.kind] - priority[b.kind] || a.map.localeCompare(b.map) || a.game.localeCompare(b.game); - }); + return maps + .map((map) => { + const utilization = map.observed > 0 ? map.active / map.observed : null; + const pressure = map.capacityObserved > 0 ? map.full / map.capacityObserved : null; + const knownShare = + map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0; + const base = { + game: map.game, + map: map.map, + evidence: + `${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` + + (pressure === null + ? 'Capacity unavailable.' + : `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`), + tentative: + map.observed < HOSTING_RULES.minimumEvidenceHours || + pressure === null || + map.capacityObserved < HOSTING_RULES.minimumEvidenceHours || + knownShare < 0.8 + }; + if (utilization === null) + return { + ...base, + kind: 'capacity' as const, + title: + map.unknown > 0 + ? 'Check occupancy reporting' + : 'Let observations accumulate', + reason: + map.unknown > 0 + ? 'Captured status is unknown. Check a server’s telemetry or RCON diagnostics before choosing a hosting change.' + : 'This map has no elapsed observations in the selected window. Wait for another status check or try a wider range.', + outcome: 'Watch for known server time in the next report.', + tentative: true + }; + if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion) + return { + ...base, + kind: 'expand' as const, + title: 'Trial one additional lobby', + reason: 'Recorded lobbies were all full for at least 20% of known capacity time. Try one extra lobby during busy hours.', + outcome: + 'Compare all-full time and use of the additional lobby over the next comparable period.' + }; + if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion) + return { + ...base, + kind: 'expand' as const, + title: 'Trial an extra lobby during busy hours', + reason: 'Players were present for at least 80% of known server time. Capacity is unknown, so this experiment uses occupancy as a proxy, not evidence of saturation.', + outcome: + 'Check whether players use the extra lobby, and measure capacity if the server reports it.', + tentative: true + }; + if ( + utilization <= HOSTING_RULES.idleUtilization && + (pressure === null || pressure < HOSTING_RULES.quietPressure) + ) + return { + ...base, + kind: 'consolidate' as const, + title: 'Trial less idle hosting', + reason: 'Players were present for at most 10% of known time, with less than 1% measured pressure or unavailable capacity. Review one idle lobby or a shorter hosting window; preserve access to this map.', + outcome: + 'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.' + }; + if (pressure === null) + return { + ...base, + kind: 'capacity' as const, + title: 'Check capacity during busy hours', + reason: 'Occupancy is recorded, but player limits are unavailable. Check capacity reporting while keeping the current lineup.', + outcome: + 'Look for known-capacity time and all-full periods before the next hosting trial.' + }; + return { + ...base, + kind: 'watch' as const, + title: 'Keep the lineup and watch peaks', + reason: 'These observations do not trigger the expansion or idle-hosting rules. Keep the current lineup for another comparison period.', + outcome: 'Watch whether occupied time and measured capacity pressure change.' + }; + }) + .sort((a, b) => { + const priority = { expand: 0, consolidate: 1, capacity: 2, watch: 3 }; + return ( + priority[a.kind] - priority[b.kind] || + a.map.localeCompare(b.map) || + a.game.localeCompare(b.game) + ); + }); } diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index c325ad4..e66a748 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -22,11 +22,22 @@ export function parseMetricsQuery(params: URLSearchParams): MetricsQuery { const requestedGame = params.get('game')?.trim() ?? ''; const game = /^[a-z0-9_-]{1,32}$/.test(requestedGame) ? requestedGame : null; const requestedMap = params.get('map'); - const map = game && requestedMap && requestedMap.trim() && requestedMap.length <= 200 && !/[\u0000-\u001f\u007f]/.test(requestedMap) - ? requestedMap : null; + const map = + game && + requestedMap && + requestedMap.trim() && + requestedMap.length <= 200 && + // eslint-disable-next-line no-control-regex -- Reject control characters in requested map names. + !/[\u0000-\u001f\u007f]/.test(requestedMap) + ? requestedMap + : null; const requestedZone = params.get('timeZone'); - return { days: parseMetricsDays(params.get('days')), game, map, - timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE }; + return { + days: parseMetricsDays(params.get('days')), + game, + map, + timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE + }; } /** All durations are hours. Observed is known server time, not collection coverage. */ diff --git a/src/lib/navigation-context.ts b/src/lib/navigation-context.ts index 74c5c9a..ddfedaa 100644 --- a/src/lib/navigation-context.ts +++ b/src/lib/navigation-context.ts @@ -1,14 +1,25 @@ const origin = 'https://electric-cherry.invalid'; -const pagePath = /^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/; +const pagePath = + /^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/; /** Only return to application pages, never an external URL or a mutation/API endpoint. */ export function safeReturnHref(value: string | null | undefined, fallback: string): string { - if (!value || value.length > 12000 || !value.startsWith('/') || value.startsWith('//') || /[\\\x00-\x1f\x7f]/.test(value)) return fallback; + if ( + !value || + value.length > 12000 || + !value.startsWith('/') || + value.startsWith('//') || + // eslint-disable-next-line no-control-regex -- Reject control characters in redirect paths. + /[\\\x00-\x1f\x7f]/.test(value) + ) + return fallback; try { const url = new URL(value, origin); if (url.origin !== origin || !pagePath.test(url.pathname)) return fallback; return url.pathname + url.search + url.hash; - } catch { return fallback; } + } catch { + return fallback; + } } export function contextHref(destination: string, source: URL | string): string { @@ -25,11 +36,32 @@ export function returnLabel(href: string): string { if (path.startsWith('/servers/')) return 'Back to server'; if (path.startsWith('/leaderboard/players/')) return 'Back to player'; if (path.startsWith('/collections/launches/')) return 'Back to collection launch'; - if (path === '/collections' && url.searchParams.get('view') === 'active') return 'Back to active launches'; - if (path === '/collections' && url.searchParams.get('view') === 'history') return 'Back to launch history'; + if (path === '/collections' && url.searchParams.get('view') === 'active') + return 'Back to active launches'; + if (path === '/collections' && url.searchParams.get('view') === 'history') + return 'Back to launch history'; if (path.startsWith('/matches/')) return 'Back to match'; if (path === '/hosts/operations') return 'Back to host operation history'; if (path.startsWith('/hosts/')) return 'Back to host'; - const names: Record = { '': 'Servers', activity: 'Activity', operations: 'Activity', autoscaling: 'Autoscaling', collections: 'Collections', hosts: 'Hosts', capacity: 'Capacity', scripts: 'Scripts', configs: 'Configs', plugins: 'Plugins', 'game-files': 'Game Files', betas: 'Betas', campaigns: 'Campaigns', maintenance: 'Maintenance', provisioning: 'Server queues', leaderboard: 'Leaderboard', matches: 'Match History', metrics: 'Metrics' }; + const names: Record = { + '': 'Servers', + activity: 'Activity', + operations: 'Activity', + autoscaling: 'Autoscaling', + collections: 'Collections', + hosts: 'Hosts', + capacity: 'Capacity', + scripts: 'Scripts', + configs: 'Configs', + plugins: 'Plugins', + 'game-files': 'Game Files', + betas: 'Betas', + campaigns: 'Campaigns', + maintenance: 'Maintenance', + provisioning: 'Server queues', + leaderboard: 'Leaderboard', + matches: 'Match History', + metrics: 'Metrics' + }; return `Back to ${names[path.split('/')[1]] ?? 'previous page'}`; } diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 8e857be..29fdc08 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -1,52 +1,116 @@ import type { UserRole } from './auth.js'; -export interface NavigationItem { label: string; href: string; shortcut?: string; matches?: (path: string) => boolean } -export interface NavigationGroup { id: string; label: string; admin?: boolean; items: NavigationItem[] } +export interface NavigationItem { + label: string; + href: string; + shortcut?: string; + matches?: (path: string) => boolean; +} +export interface NavigationGroup { + id: string; + label: string; + admin?: boolean; + items: NavigationItem[]; +} export const navigationGroups: NavigationGroup[] = [ - { id: 'fleet', label: 'Fleet', items: [ - { label: 'Servers', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') }, - { label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) }, - { label: 'Capacity', href: '/capacity' }, - { label: 'Autoscaling', href: '/autoscaling' }, - { label: 'Collections', href: '/collections' } - ] }, - { id: 'releases', label: 'Releases', items: [ - { label: 'Scripts', href: '/scripts', shortcut: 's' }, - { label: 'Configs', href: '/configs', shortcut: 'c' }, - { label: 'Plugins', href: '/plugins', shortcut: 'p' }, - { label: 'Game files', href: '/game-files' }, - { label: 'Betas', href: '/betas', shortcut: 'b' }, - { label: 'Campaigns', href: '/campaigns', shortcut: 'a' } - ] }, - { id: 'activity', label: 'Activity', items: [ - { label: 'Activity log', href: '/activity', shortcut: 'd', matches: path => path === '/activity' || path === '/operations' || path === '/hosts/operations' }, - { label: 'Provisioning', href: '/provisioning', matches: path => /^\/(provisioning|deployments)(\/|$)/.test(path) }, - { label: 'Maintenance', href: '/maintenance', shortcut: 'w', matches: path => /^\/(maintenance|watchdog)(\/|$)/.test(path) } - ] }, - { id: 'players', label: 'Players', items: [ - { label: 'Leaderboard', href: '/leaderboard', shortcut: 'l' }, - { label: 'Matches', href: '/matches' }, - { label: 'Usage', href: '/metrics', shortcut: 'm' } - ] }, - { id: 'admin', label: 'Admin', admin: true, items: [ - { label: 'Users', href: '/settings/users' }, - { label: 'Keys', href: '/server-keys', shortcut: 'k' }, - { label: 'Settings', href: '/settings', shortcut: 'h', matches: path => path === '/settings' } - ] } + { + id: 'fleet', + label: 'Fleet', + items: [ + { + label: 'Servers', + href: '/', + shortcut: 'f', + matches: (path) => path === '/' || path.startsWith('/servers/') + }, + { + label: 'Hosts', + href: '/hosts', + shortcut: 'o', + matches: (path) => + path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) + }, + { label: 'Capacity', href: '/capacity' }, + { label: 'Autoscaling', href: '/autoscaling' }, + { label: 'Collections', href: '/collections' } + ] + }, + { + id: 'releases', + label: 'Releases', + items: [ + { label: 'Scripts', href: '/scripts', shortcut: 's' }, + { label: 'Configs', href: '/configs', shortcut: 'c' }, + { label: 'Plugins', href: '/plugins', shortcut: 'p' }, + { label: 'Game files', href: '/game-files' }, + { label: 'Betas', href: '/betas', shortcut: 'b' }, + { label: 'Campaigns', href: '/campaigns', shortcut: 'a' } + ] + }, + { + id: 'activity', + label: 'Activity', + items: [ + { + label: 'Activity log', + href: '/activity', + shortcut: 'd', + matches: (path) => + path === '/activity' || path === '/operations' || path === '/hosts/operations' + }, + { + label: 'Provisioning', + href: '/provisioning', + matches: (path) => /^\/(provisioning|deployments)(\/|$)/.test(path) + }, + { + label: 'Maintenance', + href: '/maintenance', + shortcut: 'w', + matches: (path) => /^\/(maintenance|watchdog)(\/|$)/.test(path) + } + ] + }, + { + id: 'players', + label: 'Players', + items: [ + { label: 'Leaderboard', href: '/leaderboard', shortcut: 'l' }, + { label: 'Matches', href: '/matches' }, + { label: 'Usage', href: '/metrics', shortcut: 'm' } + ] + }, + { + id: 'admin', + label: 'Admin', + admin: true, + items: [ + { label: 'Users', href: '/settings/users' }, + { label: 'Keys', href: '/server-keys', shortcut: 'k' }, + { + label: 'Settings', + href: '/settings', + shortcut: 'h', + matches: (path) => path === '/settings' + } + ] + } ]; export function visibleNavigation(role: UserRole): NavigationGroup[] { - return navigationGroups.filter(group => !group.admin || role === 'admin'); + return navigationGroups.filter((group) => !group.admin || role === 'admin'); } export function navigationItemActive(item: NavigationItem, path: string): boolean { - return item.matches ? item.matches(path) : path === item.href || path.startsWith(item.href + '/'); + return item.matches + ? item.matches(path) + : path === item.href || path.startsWith(item.href + '/'); } export function navigationLocation(path: string, role: UserRole) { for (const group of visibleNavigation(role)) { - const item = group.items.find(item => navigationItemActive(item, path)); + const item = group.items.find((item) => navigationItemActive(item, path)); if (item) return { group, item }; } return null; diff --git a/src/lib/operation-flow.ts b/src/lib/operation-flow.ts index 5e0da1a..be5d920 100644 --- a/src/lib/operation-flow.ts +++ b/src/lib/operation-flow.ts @@ -1,19 +1,42 @@ -export interface ActivityEvent { id: number; created_at: string; message: string } -export interface OperationStep { label: string; state: 'pending' | 'current' | 'done' | 'failed' } - -interface BetaServer { - stage: string; deployed_at: string | null; restarted_at: string | null; - healthy_at: string | null; match_seen_at: string | null; +export interface ActivityEvent { + id: number; + created_at: string; + message: string; +} +export interface OperationStep { + label: string; + state: 'pending' | 'current' | 'done' | 'failed'; } -export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] { +interface BetaServer { + stage: string; + deployed_at: string | null; + restarted_at: string | null; + healthy_at: string | null; + match_seen_at: string | null; +} + +export function betaFlow( + beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, + servers: BetaServer[] +): OperationStep[] { const stages = ['deploying', 'awaiting-empty', 'restarting']; const labels = ['Stage file changes', 'Wait for empty servers', 'Apply files and restart']; - if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); } - if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); } - if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); } - stages.push('ready'); labels.push('Review and promote'); - const positions = servers.map(s => { + if (beta.wait_for_health) { + stages.push('awaiting-health'); + labels.push('Verify health'); + } + if (beta.wait_for_match) { + stages.push('awaiting-match'); + labels.push('Observe a completed match'); + } + if (beta.testPlan) { + stages.push('observing'); + labels.push('Evaluate test requirements'); + } + stages.push('ready'); + labels.push('Review and promote'); + const positions = servers.map((s) => { if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage)); if (!s.deployed_at) return 0; if (!s.restarted_at) return 2; @@ -21,29 +44,67 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo if (beta.wait_for_match && !s.match_seen_at) return stages.indexOf('awaiting-match'); return stages.length - 1; }); - const current = beta.state === 'promoted' ? stages.length - : beta.state === 'ready-promote' ? stages.length - 1 - : positions.length ? Math.min(...positions) : 0; - return labels.map((label, i) => ({ label, state: i < current ? 'done' : i === current - ? beta.state === 'rolled-back' ? 'pending' : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) ? 'failed' : 'current' - : 'pending' })); + const current = + beta.state === 'promoted' + ? stages.length + : beta.state === 'ready-promote' + ? stages.length - 1 + : positions.length + ? Math.min(...positions) + : 0; + return labels.map((label, i) => ({ + label, + state: + i < current + ? 'done' + : i === current + ? beta.state === 'rolled-back' + ? 'pending' + : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) + ? 'failed' + : 'current' + : 'pending' + })); } -export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean, - files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] { - const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state)); - const positions = active.map(s => { +export function campaignFlow( + servers: { + server?: string; + restart_state: string; + restarted_at: string | null; + files_changed: number; + }[], + finished: boolean, + files: { server: string; snapshot_pending: number; apply_state: string }[] = [] +): OperationStep[] { + const active = servers.filter((s) => !['skipped', 'cancelled'].includes(s.restart_state)); + const positions = active.map((s) => { if (s.restart_state === 'done') return 5; if (s.restarted_at) return 4; - const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled'); - if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2; - if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3; + const changes = files.filter((f) => f.server === s.server && f.apply_state !== 'cancelled'); + if (changes.some((f) => f.apply_state === 'applying')) + return changes.some((f) => f.snapshot_pending) ? 1 : 2; + if (changes.length && changes.every((f) => f.apply_state === 'applied')) return 3; return 0; }); const current = positions.length ? Math.min(...positions) : finished ? 5 : 0; - return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({ - label, state: i < current ? 'done' : i !== current ? 'pending' - : active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed' - : active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current' + return [ + 'Wait for empty servers and restart capacity', + 'Capture rollback snapshots', + 'Apply file changes', + 'Restart servers', + 'Verify health' + ].map((label, i) => ({ + label, + state: + i < current + ? 'done' + : i !== current + ? 'pending' + : active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) + ? 'failed' + : active.some((s) => ['manual', 'held'].includes(s.restart_state)) + ? 'pending' + : 'current' })); } diff --git a/src/lib/plutonium.ts b/src/lib/plutonium.ts index 0e6c7c4..cf49c3f 100644 --- a/src/lib/plutonium.ts +++ b/src/lib/plutonium.ts @@ -52,9 +52,16 @@ export const PLUTONIUM_MANIFEST_URL = 'https://cdn.plutoniummod.com/updater/prod export function parsePlutoniumManifest(value: unknown): number { const manifest = value as { product?: unknown; revision?: unknown } | null; - if (!manifest || manifest.product !== 'plutonium-core-prod' || typeof manifest.revision !== 'number' - || !Number.isSafeInteger(manifest.revision) || manifest.revision < 1) { - throw new Error('The production updater manifest did not contain a valid Plutonium revision.'); + if ( + !manifest || + manifest.product !== 'plutonium-core-prod' || + typeof manifest.revision !== 'number' || + !Number.isSafeInteger(manifest.revision) || + manifest.revision < 1 + ) { + throw new Error( + 'The production updater manifest did not contain a valid Plutonium revision.' + ); } return manifest.revision; } @@ -66,12 +73,20 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT const [hour, minute] = time.split(':').map(Number); const target = hour * 60 + minute; const formatter = new Intl.DateTimeFormat('en-CA', { - timeZone, year: 'numeric', month: '2-digit', day: '2-digit', - hour: '2-digit', minute: '2-digit', hourCycle: 'h23' + timeZone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hourCycle: 'h23' }); function local(at: number) { - const parts = Object.fromEntries(formatter.formatToParts(at).map(p => [p.type, p.value])); - return { day: `${parts.year}-${parts.month}-${parts.day}`, minute: Number(parts.hour) * 60 + Number(parts.minute) }; + const parts = Object.fromEntries(formatter.formatToParts(at).map((p) => [p.type, p.value])); + return { + day: `${parts.year}-${parts.month}-${parts.day}`, + minute: Number(parts.hour) * 60 + Number(parts.minute) + }; } const today = local(now.getTime()).day; const start = Math.floor(now.getTime() / 60_000) * 60_000; @@ -83,9 +98,13 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT const wall = local(at); if (wall.day < today || wall.minute < target) continue; const previous = days.get(wall.day); - if (!previous || wall.minute < previous.minute) days.set(wall.day, { at, minute: wall.minute }); + if (!previous || wall.minute < previous.minute) + days.set(wall.day, { at, minute: wall.minute }); } - const next = [...days.values()].map(d => d.at).filter(at => at > now.getTime()).sort((a, b) => a - b)[0]; + const next = [...days.values()] + .map((d) => d.at) + .filter((at) => at > now.getTime()) + .sort((a, b) => a - b)[0]; if (next === undefined) throw new Error('Could not calculate the next maintenance time.'); return new Date(next).toISOString(); } @@ -94,5 +113,10 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT export function maintenanceTimeInZone(schedule: MaintenanceSchedule, timeZone: string): string { if (schedule.timeZone === timeZone) return schedule.time; const at = schedule.nextAt ?? nextMaintenanceAt(schedule.time, new Date(), schedule.timeZone); - return new Intl.DateTimeFormat('en-GB', { timeZone, hour: '2-digit', minute: '2-digit', hourCycle: 'h23' }).format(new Date(at)); + return new Intl.DateTimeFormat('en-GB', { + timeZone, + hour: '2-digit', + minute: '2-digit', + hourCycle: 'h23' + }).format(new Date(at)); } diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index c848330..f12bd4b 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -1,15 +1,32 @@ import type { ProvisioningAction } from './provisioning.js'; import type { LaunchRecipeSummary } from './deployment-plan.js'; -export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', - 'container-action', 'cleanup', 'verify-container', 'verify-game', - 'verify-map', 'verify-plutonium', 'verify-stopped'] as const; -export type ProvisioningPhase = typeof provisioningPhases[number]; +export const provisioningPhases = [ + 'requested', + 'eligibility', + 'startup-queue', + 'prepare', + 'container-action', + 'cleanup', + 'verify-container', + 'verify-game', + 'verify-map', + 'verify-plutonium', + 'verify-stopped' +] as const; +export type ProvisioningPhase = (typeof provisioningPhases)[number]; export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown'; export type EvidenceSource = 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null; export interface ProvisioningCheck { - key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped'; + key: + | 'container' + | 'game' + | 'map' + | 'plutonium-release' + | 'plutonium-version' + | 'occupancy' + | 'stopped'; label: string; result: EvidenceResult; source: EvidenceSource; @@ -46,7 +63,11 @@ export interface ProvisioningProgress { blocker: ProvisioningBlocker | null; checks: ProvisioningCheck[]; checkpoints: ProvisioningCheckpoint[]; - command: { state: 'attempted' | 'accepted' | 'rejected' | 'uncertain'; at: string; message: string } | null; + command: { + state: 'attempted' | 'accepted' | 'rejected' | 'uncertain'; + at: string; + message: string; + } | null; recovery: RecoveryObservation | null; recipeStatus: 'queued' | 'attempted' | null; } @@ -105,7 +126,14 @@ export interface ProvisioningDetail { cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null; backend?: 'native'; workerActivity?: WorkerActivity[]; - agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null; + agentOperation?: { + id: string; + hostId: string; + state: string; + updatedAt: string; + createdAt: string; + error: string | null; + } | null; deployment: ProvisioningDeployment; events: ProvisioningEvent[]; activity: { total: number; latestId: number | null; failureExcerpt: ProvisioningEvent | null }; @@ -122,7 +150,11 @@ export interface ProvisioningDetail { warningMessages: string[]; generatedAt: string; } -export interface WorkerActivity { key: string; label: string; startedAt: string } +export interface WorkerActivity { + key: string; + label: string; + startedAt: string; +} export interface ProvisioningEventPage { events: ProvisioningEvent[]; total: number; diff --git a/src/lib/provisioning-list.ts b/src/lib/provisioning-list.ts index 6fa14e1..b1d2ba1 100644 --- a/src/lib/provisioning-list.ts +++ b/src/lib/provisioning-list.ts @@ -2,42 +2,104 @@ import type { ProvisioningAction } from './provisioning.js'; import type { ProvisioningDeployment } from './provisioning-detail.js'; import type { ProvisioningSource } from './provisioning-detail.js'; -export const provisioningListStatuses = ['pending', 'running', 'awaiting-health', 'done', 'attention', 'failed', 'interrupted', 'warning', 'cancelled'] as const; -export const provisioningListSources = ['manual', 'campaign', 'beta', 'maintenance', 'recovery', 'autoscaling', 'rebalance', 'system'] as const; -export type ProvisioningListSource = typeof provisioningListSources[number]; +export const provisioningListStatuses = [ + 'pending', + 'running', + 'awaiting-health', + 'done', + 'attention', + 'failed', + 'interrupted', + 'warning', + 'cancelled' +] as const; +export const provisioningListSources = [ + 'manual', + 'campaign', + 'beta', + 'maintenance', + 'recovery', + 'autoscaling', + 'rebalance', + 'system' +] as const; +export type ProvisioningListSource = (typeof provisioningListSources)[number]; export const provisioningSourceLabels: Record = { - manual: 'Manual', campaign: 'Campaign', beta: 'Beta', maintenance: 'Maintenance', - recovery: 'Recovery', autoscaling: 'Autoscaling', rebalance: 'Rebalance', system: 'System' + manual: 'Manual', + campaign: 'Campaign', + beta: 'Beta', + maintenance: 'Maintenance', + recovery: 'Recovery', + autoscaling: 'Autoscaling', + rebalance: 'Rebalance', + system: 'System' }; export const provisioningSourceKinds = { - campaign: ['campaign'], beta: ['beta'], maintenance: ['maintenance'], - recovery: ['watchdog'], autoscaling: ['autoscaling'], rebalance: ['rebalance'] -} as const satisfies Record, readonly ProvisioningSource['kind'][]>; + campaign: ['campaign'], + beta: ['beta'], + maintenance: ['maintenance'], + recovery: ['watchdog'], + autoscaling: ['autoscaling'], + rebalance: ['rebalance'] +} as const satisfies Record< + Exclude, + readonly ProvisioningSource['kind'][] +>; /** Record titles and incident summaries belong in expanded details, not the source column. */ -export function provisioningSourceLabel(row: Pick): string { +export function provisioningSourceLabel( + row: Pick +): string { const kind = row.sources?.[0]?.kind; - return provisioningSourceLabels[kind === 'watchdog' ? 'recovery' : kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system')]; + return provisioningSourceLabels[ + kind === 'watchdog' + ? 'recovery' + : (kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system')) + ]; } -export function provisioningMatchesSource(row: Pick, source: ProvisioningListSource): boolean { +export function provisioningMatchesSource( + row: Pick, + source: ProvisioningListSource +): boolean { const sources = row.sources ?? []; - if (source === 'manual' || source === 'system') return !sources.length && (source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui'); + if (source === 'manual' || source === 'system') + return ( + !sources.length && + (source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui') + ); const kinds: readonly ProvisioningSource['kind'][] = provisioningSourceKinds[source]; - return sources.some(item => kinds.includes(item.kind)); + return sources.some((item) => kinds.includes(item.kind)); } -export type ProvisioningListRow = Pick & { - canCancelRemoval: number; canCancel?: number; sources?: ProvisioningSource[]; - phase?: string | null; phaseStartedAt?: string | null; waitCode?: string | null; - }; +export type ProvisioningListRow = Pick< + ProvisioningDeployment, + | 'id' + | 'server' + | 'server_deleted_at' + | 'action' + | 'state' + | 'stage' + | 'started_at' + | 'finished_at' + | 'triggered_by' + | 'requested_by' + | 'error' + | 'warnings' + | 'server_exists' +> & { + canCancelRemoval: number; + canCancel?: number; + sources?: ProvisioningSource[]; + phase?: string | null; + phaseStartedAt?: string | null; + waitCode?: string | null; +}; export interface ProvisioningListOptions { view?: 'active' | 'history' | 'all'; action?: ProvisioningAction; - status?: typeof provisioningListStatuses[number]; + status?: (typeof provisioningListStatuses)[number]; source?: ProvisioningListSource; server?: string; limit?: number; diff --git a/src/lib/provisioning.ts b/src/lib/provisioning.ts index 4de4ecc..4aaf29b 100644 --- a/src/lib/provisioning.ts +++ b/src/lib/provisioning.ts @@ -1,17 +1,34 @@ -export const provisioningActions = ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'] as const; -export type ProvisioningAction = typeof provisioningActions[number]; +export const provisioningActions = [ + 'create', + 'start', + 'restart', + 'rebuild', + 'stop', + 'destroy' +] as const; +export type ProvisioningAction = (typeof provisioningActions)[number]; export type RuntimeAction = Exclude; export const actionLabels: Record = { - create: 'Create', start: 'Start', restart: 'Restart', rebuild: 'Rebuild', stop: 'Stop', destroy: 'Destroy' + create: 'Create', + start: 'Start', + restart: 'Restart', + rebuild: 'Rebuild', + stop: 'Stop', + destroy: 'Destroy' }; export const actionCompleted: Record = { - create: 'Server creation completed', start: 'Server started', restart: 'Server restarted', - rebuild: 'Server rebuilt', stop: 'Server stopped', destroy: 'Server destroyed' + create: 'Server creation completed', + start: 'Server started', + restart: 'Server restarted', + rebuild: 'Server rebuilt', + stop: 'Server stopped', + destroy: 'Server destroyed' }; export const runtimeActions: RuntimeAction[] = ['start', 'restart', 'rebuild', 'stop']; export const runtimeDescriptions: Record = { start: 'Start the existing container and verify its health.', restart: 'Restart the container after the server is empty, then verify its health.', - rebuild: 'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.', + rebuild: + 'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.', stop: 'Stop the container after the server is empty. Its files, key, and registration are retained so you can start it again.' }; diff --git a/src/lib/rebalancing.ts b/src/lib/rebalancing.ts index 7aeb83c..748495f 100644 --- a/src/lib/rebalancing.ts +++ b/src/lib/rebalancing.ts @@ -4,15 +4,26 @@ export interface RebalanceSettings { maxMoves: number; } -export interface RebalanceHostOption { id: string; name: string; eligible: boolean; reason: string | null } +export interface RebalanceHostOption { + id: string; + name: string; + eligible: boolean; + reason: string | null; +} export interface RebalanceHostProjection { - id: string; name: string; + id: string; + name: string; before: { cpuPercent: number; memoryPercent: number; instances: number }; after: { cpuPercent: number; memoryPercent: number; instances: number }; } export interface RebalanceCandidate { - server: string; instanceId: string; generation: number; - sourceHostId: string; sourceHostName: string; targetHostId: string; targetHostName: string; + server: string; + instanceId: string; + generation: number; + sourceHostId: string; + sourceHostName: string; + targetHostId: string; + targetHostName: string; /** Estimated reduction in the source/destination pair's peak utilization, in percentage points. */ improvement: number; } @@ -23,26 +34,55 @@ export interface RebalancePlan { warnings: string[]; } export interface RebalancePreview extends RebalancePlan { - id: string; settings: RebalanceSettings; createdAt: string; expiresAt: string; + id: string; + settings: RebalanceSettings; + createdAt: string; + expiresAt: string; } export interface RebalanceMove extends RebalanceCandidate { - id: string; state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed'; - operationId: string | null; message: string; updatedAt: string; + id: string; + state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed'; + operationId: string | null; + message: string; + updatedAt: string; } export interface RebalanceRun { - id: string; settings: RebalanceSettings; + id: string; + settings: RebalanceSettings; state: 'running' | 'stopping' | 'done' | 'stopped' | 'failed'; - createdAt: string; updatedAt: string; message: string; moves: RebalanceMove[]; + createdAt: string; + updatedAt: string; + message: string; + moves: RebalanceMove[]; +} +export interface RebalanceReport { + hosts: RebalanceHostOption[]; + runs: RebalanceRun[]; } -export interface RebalanceReport { hosts: RebalanceHostOption[]; runs: RebalanceRun[] } export function rebalanceSettings(input: unknown): RebalanceSettings { - if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide rebalance settings.'); + if (!input || typeof input !== 'object' || Array.isArray(input)) + throw new Error('Provide rebalance settings.'); const value = input as Record; - if (!Array.isArray(value.hostIds) || value.hostIds.length < 2 || value.hostIds.length > 100 - || value.hostIds.some(id => typeof id !== 'string' || !id || id.length > 100) - || new Set(value.hostIds).size !== value.hostIds.length) throw new Error('Choose between 2 and 100 different hosts.'); - if (!['all', 't5', 't6'].includes(value.game as string)) throw new Error('Choose a supported game.'); - if (!Number.isInteger(value.maxMoves) || (value.maxMoves as number) < 1 || (value.maxMoves as number) > 20) throw new Error('Choose a move limit between 1 and 20.'); - return { hostIds: [...value.hostIds as string[]].sort(), game: value.game as RebalanceSettings['game'], maxMoves: value.maxMoves as number }; + if ( + !Array.isArray(value.hostIds) || + value.hostIds.length < 2 || + value.hostIds.length > 100 || + value.hostIds.some((id) => typeof id !== 'string' || !id || id.length > 100) || + new Set(value.hostIds).size !== value.hostIds.length + ) + throw new Error('Choose between 2 and 100 different hosts.'); + if (!['all', 't5', 't6'].includes(value.game as string)) + throw new Error('Choose a supported game.'); + if ( + !Number.isInteger(value.maxMoves) || + (value.maxMoves as number) < 1 || + (value.maxMoves as number) > 20 + ) + throw new Error('Choose a move limit between 1 and 20.'); + return { + hostIds: [...(value.hostIds as string[])].sort(), + game: value.game as RebalanceSettings['game'], + maxMoves: value.maxMoves as number + }; } diff --git a/src/lib/script-library.ts b/src/lib/script-library.ts index 7e75bf6..9a63989 100644 --- a/src/lib/script-library.ts +++ b/src/lib/script-library.ts @@ -2,20 +2,34 @@ import { T5_ZOMBIE_MAPS, T6_ZOMBIE_MAPS } from './map-aliases.js'; import { gameTitle } from './ui/server-labels.js'; export interface LibraryVersion { - id: number; version: number; sha256: string; size: number; - state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string; + id: number; + version: number; + sha256: string; + size: number; + state: 'untested' | 'tested' | 'test-failed'; + uploaded_at: string; } export interface LibraryScript { - id: number; name: string; target_rel_path: string; - scope_type: 'game' | 'map' | null; scope_value: string | null; + id: number; + name: string; + target_rel_path: string; + scope_type: 'game' | 'map' | null; + scope_value: string | null; current_version_id: number | null; - currentVersion: LibraryVersion | null; latestVersion: LibraryVersion | null; - versionCount: number; servers: string[]; runningBetaId: number | null; + currentVersion: LibraryVersion | null; + latestVersion: LibraryVersion | null; + versionCount: number; + servers: string[]; + runningBetaId: number | null; } export type ScriptStatus = 'untested' | 'testing' | 'tested' | 'failed' | 'promoted' | 'empty'; export const statusLabels: Record = { - untested: 'Needs testing', testing: 'Beta active', tested: 'Test passed', - failed: 'Test failed', promoted: 'Promoted', empty: 'No versions' + untested: 'Needs testing', + testing: 'Beta active', + tested: 'Test passed', + failed: 'Test failed', + promoted: 'Promoted', + empty: 'No versions' }; export function scriptStatus(s: LibraryScript): ScriptStatus { if (s.runningBetaId !== null) return 'testing'; @@ -24,17 +38,29 @@ export function scriptStatus(s: LibraryScript): ScriptStatus { return s.latestVersion.state === 'test-failed' ? 'failed' : s.latestVersion.state; } export function canTest(s: LibraryScript): boolean { - return s.runningBetaId === null && s.latestVersion !== null && s.latestVersion.id !== s.current_version_id; + return ( + s.runningBetaId === null && + s.latestVersion !== null && + s.latestVersion.id !== s.current_version_id + ); } export function mapLabel(canonical: string): string { - const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter(m => m.canonical === canonical); - return matches.length > 1 ? `${matches[0].friendly} + ${matches.length - 1} variants` : matches[0]?.friendly ?? canonical; + const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter((m) => m.canonical === canonical); + return matches.length > 1 + ? `${matches[0].friendly} + ${matches.length - 1} variants` + : (matches[0]?.friendly ?? canonical); } export function scopeLabel(s: Pick): string { - return s.scope_type === 'game' ? gameTitle(s.scope_value) - : s.scope_type === 'map' ? mapLabel(s.scope_value ?? '') : 'Unscoped'; + return s.scope_type === 'game' + ? gameTitle(s.scope_value) + : s.scope_type === 'map' + ? mapLabel(s.scope_value ?? '') + : 'Unscoped'; +} +export interface BetaSelection { + scriptId: number; + versionId: number; } -export interface BetaSelection { scriptId: number; versionId: number; } export function betaSelectionHref(selections: BetaSelection[]): string { const query = new URLSearchParams(); for (const s of selections) query.append('artifact', `${s.scriptId}:${s.versionId}`); @@ -42,13 +68,21 @@ export function betaSelectionHref(selections: BetaSelection[]): string { } export function parseBetaSelection(query: URLSearchParams): BetaSelection[] { const values = query.getAll('artifact'); - if (!values.length && query.has('pluginId')) values.push(`${query.get('pluginId')}:${query.get('versionId')}`); + if (!values.length && query.has('pluginId')) + values.push(`${query.get('pluginId')}:${query.get('versionId')}`); const seen = new Set(); - return values.map(value => { - if (!/^[1-9]\d*:[1-9]\d*$/.test(value)) throw new Error('The beta selection link is invalid. Choose artifacts below.'); + return values.map((value) => { + if (!/^[1-9]\d*:[1-9]\d*$/.test(value)) + throw new Error('The beta selection link is invalid. Choose artifacts below.'); const [scriptId, versionId] = value.split(':').map(Number); - if (!Number.isSafeInteger(scriptId) || !Number.isSafeInteger(versionId) || seen.has(scriptId)) { - throw new Error('The beta selection link contains invalid or repeated artifacts. Choose artifacts below.'); + if ( + !Number.isSafeInteger(scriptId) || + !Number.isSafeInteger(versionId) || + seen.has(scriptId) + ) { + throw new Error( + 'The beta selection link contains invalid or repeated artifacts. Choose artifacts below.' + ); } seen.add(scriptId); return { scriptId, versionId }; diff --git a/src/lib/server-capacity.ts b/src/lib/server-capacity.ts index fc979d6..f462b91 100644 --- a/src/lib/server-capacity.ts +++ b/src/lib/server-capacity.ts @@ -6,11 +6,20 @@ export interface PlayerCapacityObservation { } /** A newer roster must never renew the capacity reported by an older process. */ -export function currentCapacity(server: Partial & { telemetry_after?: string | null }, now: number): number | null { +export function currentCapacity( + server: Partial & { telemetry_after?: string | null }, + now: number +): number | null { const value = server.capacity; const observedAt = server.capacity_observed_at; - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0 - || !freshStatusTimestamp(observedAt ?? null, now)) return null; - if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after))) return null; + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value <= 0 || + !freshStatusTimestamp(observedAt ?? null, now) + ) + return null; + if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after))) + return null; return value; } diff --git a/src/lib/server-observation.ts b/src/lib/server-observation.ts index 6d2883e..1fe7fe4 100644 --- a/src/lib/server-observation.ts +++ b/src/lib/server-observation.ts @@ -36,51 +36,112 @@ export interface ServerObservation extends GameObservation { } /** Keep a first transient monitoring failure in diagnostics without demoting health. */ -export function rconHealthError(status: Pick): string | null { +export function rconHealthError( + status: Pick +): string | null { const reason = status.last_error?.split(':')[0]; - return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error; + return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') + ? null + : status.last_error; } -export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry { +export function freshTelemetry( + report: GameTelemetry | undefined, + now: number, + after?: string | null +): report is GameTelemetry { if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false; - if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false; - try { parseGameEvent(report); return true; } catch { return false; } + if ( + after && + (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after)) + ) + return false; + try { + parseGameEvent(report); + return true; + } catch { + return false; + } } -function playerMatchGuid(guid: string, game: string | null | undefined, source: 'rcon' | 'telemetry'): string { +function playerMatchGuid( + guid: string, + game: string | null | undefined, + source: 'rcon' | 'telemetry' +): string { if (game !== 't6') return guid; // T6 status prints hexadecimal account IDs; telemetry sends decimal IDs. // Keep integer precision for IDs beyond Number.MAX_SAFE_INTEGER. - if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid)) return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString(); + if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid)) + return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString(); if (source === 'telemetry' && /^\d+$/.test(guid)) return BigInt(guid).toString(); return guid; } /** Display policy: use one current roster/count source, keeping raw RCON for diagnostics/fallback. */ -export function selectServerObservation(server: T, report: GameTelemetry | undefined, now: number): T { - if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry; +export function selectServerObservation( + server: T, + report: GameTelemetry | undefined, + now: number +): T { + if ( + server.telemetry && + (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt)) + ) + report = server.telemetry; const raw = server.rcon_status ?? server; - const result: T = { ...server, ...raw, rcon_status: raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; + const result: T = { + ...server, + ...raw, + rcon_status: raw, + last_error: rconHealthError(raw), + telemetry: report, + status_source: raw.last_seen_at ? 'rcon' : null + }; const containerUnknown = server.container_state === 'unknown'; - if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) { - return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null }; + if ( + ['provisioning', 'starting'].includes(server.health) || + (server.container_state && server.container_state !== 'running' && !containerUnknown) || + (!server.container_state && server.health === 'offline') + ) { + return { + ...result, + health: server.health, + last_error: null, + last_players: null, + roster: null, + status_source: null + }; } const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at; - const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now); - const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); + const containerFresh = + server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now); + const after = + !server.telemetry_after || + Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error; - if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) { + if ( + containerFresh && + report?.server === server.name && + freshTelemetry(report, now, server.telemetry_after) + ) { const players = report.players.length; // Telemetry owns roster membership. Supplement missing pings only with // unambiguous RCON identities from the current map and server lifecycle. const pings = new Map(); - const guidKey = (guid: string, source: 'rcon' | 'telemetry') => playerMatchGuid(guid, server.game, source); + const guidKey = (guid: string, source: 'rcon' | 'telemetry') => + playerMatchGuid(guid, server.game, source); if (fresh && raw.last_map === report.map) { for (const player of raw.roster ?? []) { const ping = player.ping; if (player.guid) { const guid = guidKey(player.guid, 'rcon'); - pings.set(guid, pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0 ? null : ping); + pings.set( + guid, + pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0 + ? null + : ping + ); } } } @@ -89,23 +150,49 @@ export function selectServerObservation(server: T, const timing = report.timing?.server ?? { activeSince: server.active_since ?? report.receivedAt, inactiveSince: server.inactive_since ?? report.receivedAt, - gameStartedAt: report.event === 'game_start' ? report.receivedAt : server.game_started_at, - gameEndedAt: report.event === 'game_start' ? null : report.event === 'game_end' ? report.receivedAt : server.game_ended_at + gameStartedAt: + report.event === 'game_start' ? report.receivedAt : server.game_started_at, + gameEndedAt: + report.event === 'game_start' + ? null + : report.event === 'game_end' + ? report.receivedAt + : server.game_ended_at }; - return { ...result, health: 'online', status_source: 'telemetry', last_players: players, - last_map: report.map, last_seen_at: report.receivedAt, last_error: null, - roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null, - ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null, guid: player.guid, name: player.name })), + return { + ...result, + health: 'online', + status_source: 'telemetry', + last_players: players, + last_map: report.map, + last_seen_at: report.receivedAt, + last_error: null, + roster: report.players.map((player) => ({ + num: null, + score: 'score' in player ? player.score : null, + ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null, + guid: player.guid, + name: player.name + })), active_since: players > 0 ? timing.activeSince : null, inactive_since: players === 0 ? timing.inactiveSince : null, game_started_at: players === 0 ? null : timing.gameStartedAt, - game_ended_at: players === 0 ? null : timing.gameEndedAt }; + game_ended_at: players === 0 ? null : timing.gameEndedAt + }; } // A current authenticated RCON reply can establish game liveness when // inspection is unavailable, without claiming the container was inspected. const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh; const age = now - Date.parse(raw.last_seen_at ?? ''); - const health = fresh && !containerExpired ? 'online' - : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline'; - return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error }; + const health = + fresh && !containerExpired + ? 'online' + : after && age >= 0 && age <= 120_000 + ? 'stale' + : 'offline'; + return { + ...result, + health, + last_error: health === 'online' ? result.last_error : raw.last_error + }; } diff --git a/src/lib/server/activity.ts b/src/lib/server/activity.ts index 3a93bef..a710f77 100644 --- a/src/lib/server/activity.ts +++ b/src/lib/server/activity.ts @@ -1,6 +1,12 @@ import { hostOperationTitles } from '../host-operation-labels.js'; import type Database from 'better-sqlite3'; -import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js'; +import { + normalizeActivityQuery, + type ActivityPage, + type ActivityQuery, + type ActivityRow, + type ActivitySource +} from '../activity.js'; import { serverTitle } from '../ui/server-labels.js'; import { db } from './db/index.js'; import { ensureHostSchema } from './hosts/schema.js'; @@ -9,16 +15,22 @@ import { importantOperationSql } from './hosts/observations.js'; const initialized = new WeakSet(); const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; const labelCase = (column: string, labels: Record, fallback: string) => - `CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`; + `CASE ${column} ${Object.entries(labels) + .map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`) + .join(' ')} ELSE ${quote(fallback)} END`; // Old attempts may no longer be the native_provisioning pointer. Their stable // idempotency keys still identify the same retained controller operation. const lifecycleKey = `CASE WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6) - ${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0 + ${[':migration-create:', ':migrate-remove:'] + .map( + (marker) => `WHEN instr(o.idempotency_key,${quote(marker)})>0 AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>'' AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*' - THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')} + THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)` + ) + .join('\n')} ELSE o.idempotency_key END`; /** One row per user-level operation or finding; no command payloads or observations are read. */ @@ -128,19 +140,50 @@ normalized AS ( ) `; -type SqlRow = Omit & { sources: string; route: string; recordId: string; secondaryHostId: string | null }; -const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host']; +type SqlRow = Omit & { + sources: string; + route: string; + recordId: string; + secondaryHostId: string | null; +}; +const sourceOrder: ActivitySource[] = [ + 'maintenance', + 'recovery', + 'campaign', + 'beta', + 'autoscaling', + 'provisioning', + 'host' +]; // Use the same membership rule for Maintenance history and its exclusion from the main log. -const maintenanceMembershipSql = "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)"; +const maintenanceMembershipSql = + "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)"; function present(row: SqlRow): ActivityRow { - const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`)); - const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}` - : row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}` - : row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all` - : `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`; - return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources, - state: row.state, rawState: row.rawState, title: row.title, detail: row.detail, - server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href }; + const sources = sourceOrder.filter((source) => row.sources.includes(`|${source}|`)); + const href = + row.route === 'maintenance-job' + ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}` + : row.route === 'recovery' + ? `/maintenance/${encodeURIComponent(row.recordId)}` + : row.route === 'host' + ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all` + : `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`; + return { + id: row.id, + at: row.at, + type: row.type, + source: sources[0] ?? 'provisioning', + sources, + state: row.state, + rawState: row.rawState, + title: row.title, + detail: row.detail, + server: row.server, + serverDeletedAt: row.serverDeletedAt, + hostId: row.hostId, + hostName: row.hostName, + href + }; } /** All predicates, totals and pagination operate on the same retained SQL union. */ @@ -149,7 +192,9 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac const handle = db(); ensureHostSchema(handle); if (!initialized.has(handle)) { - handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : ''); + handle.function('activity_server_title', { deterministic: true }, (name: unknown) => + typeof name === 'string' ? serverTitle(name) : '' + ); initialized.add(handle); } const conditions: string[] = []; @@ -158,25 +203,44 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac if (query.excludeProvisioning) conditions.push("route <> 'provisioning' AND type <> 'restart'"); if (query.excludeMaintenance) conditions.push(`NOT ${maintenanceMembershipSql}`); const bindings: Record = {}; - for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; } - if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; } + for (const key of ['server', 'type', 'state'] as const) + if (query[key]) { + conditions.push(`${key}=@${key}`); + bindings[key] = query[key]; + } + if (query.host) { + conditions.push('(hostId=@host OR secondaryHostId=@host)'); + bindings.host = query.host; + } if (query.source) { // Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter. - conditions.push(query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0'); + conditions.push( + query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0' + ); if (query.source !== 'maintenance') bindings.source = `|${query.source}|`; } - if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); } + if (query.days) { + conditions.push('julianday(at)>=julianday(@since)'); + bindings.since = new Date(now - query.days * 86_400_000).toISOString(); + } if (query.q) { - conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0"); + conditions.push( + "instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0" + ); bindings.q = query.q.toLocaleLowerCase(); } const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; const limit = query.limit!; return handle.transaction(() => { - const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number }; + const { total } = handle + .prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`) + .get(bindings) as { total: number }; const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit); - const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId - FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`) + const rows = handle + .prepare( + `${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId + FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset` + ) .all({ ...bindings, limit, offset }) as SqlRow[]; return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } }; })(); diff --git a/src/lib/server/auth/access.ts b/src/lib/server/auth/access.ts index 7a81ffe..d094c7d 100644 --- a/src/lib/server/auth/access.ts +++ b/src/lib/server/auth/access.ts @@ -1,8 +1,10 @@ import type { UserRole } from '../../auth.js'; -export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin'; +export type RoutePermission = + 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin'; -const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/'); +const within = (route: string, base: string): boolean => + route === base || route.startsWith(base + '/'); const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD'; /** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname. @@ -15,32 +17,63 @@ export function routePermission(routeId: string | null, method: string): RoutePe if (routeId === null) return reading ? 'viewer' : 'operator'; // These cannot occur in this application's resolved route IDs. Fail closed // if a future caller accidentally supplies a raw or noncanonical pathname. - if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId) - || routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin'; + if ( + !routeId.startsWith('/') || + // eslint-disable-next-line no-control-regex -- Reject control characters in resolved route IDs. + /[%\\?#\x00-\x1f\x7f]/.test(routeId) || + (routeId !== '/' && + routeId + .split('/') + .slice(1) + .some((part) => !part || part === '.' || part === '..')) + ) + return 'admin'; if (within(routeId, '/api/agent')) return 'machine'; if (routeId === '/api/telemetry' && method === 'POST') return 'machine'; if (routeId === '/api/leaderboard' && reading) return 'public'; if ((routeId === '/login' || routeId === '/setup') && reading) return 'public'; - if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public'; - if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated'; + if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') + return 'public'; + if ( + routeId === '/account' || + ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId) + ) + return 'authenticated'; - if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/settings') || within(routeId, '/api/auth') - || within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin'; + if ( + within(routeId, '/api/users') || + within(routeId, '/settings') || + within(routeId, '/api/settings') || + within(routeId, '/api/auth') || + within(routeId, '/api/server-keys') || + within(routeId, '/server-keys') || + within(routeId, '/keys') + ) + return 'admin'; - if (!reading && ['/api/hosts', '/api/game-files'].some(base => within(routeId, base))) return 'admin'; + if (!reading && ['/api/hosts', '/api/game-files'].some((base) => within(routeId, base))) + return 'admin'; if (routeId === '/api/provisioning/[id]/override') return 'admin'; // These endpoints mix operational actions with global policy and schedule // changes. Per-incident/job actions remain available to operators. - if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin'; + if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) + return 'admin'; // Summaries remain visible to viewers; raw configuration, historical file // bytes, and unfiltered diagnostics may contain runtime credentials. - if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect' - || routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]' - || routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]' - || routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests' - || routeId === '/api/servers/[name]/rcon-diagnostics') return 'operator'; + if ( + routeId === '/configs/[id]' || + routeId === '/api/configs/[id]' || + routeId === '/api/configs/inspect' || + routeId === '/api/campaigns/[id]/file/[fileId]' || + routeId === '/api/deployments/[id]/file/[fileId]' || + routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]' || + routeId === '/api/servers/[name]/logs/[file]' || + routeId === '/api/servers/[name]/telemetry-requests' || + routeId === '/api/servers/[name]/rcon-diagnostics' + ) + return 'operator'; return reading ? 'viewer' : 'operator'; } @@ -49,7 +82,8 @@ export function routePermission(routeId: string | null, method: string): RoutePe export function roleAllows(role: UserRole, permission: RoutePermission): boolean { if (!['viewer', 'operator', 'admin'].includes(role)) return false; if (permission === 'machine') return false; - if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true; + if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') + return true; if (permission === 'operator') return role === 'operator' || role === 'admin'; return permission === 'admin' && role === 'admin'; } diff --git a/src/lib/server/auth/guard.ts b/src/lib/server/auth/guard.ts index 43085f7..1f7698e 100644 --- a/src/lib/server/auth/guard.ts +++ b/src/lib/server/auth/guard.ts @@ -1,112 +1,234 @@ import { error, json, redirect, type Handle } from '@sveltejs/kit'; import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js'; -import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js'; +import { + authBody, + authError, + requireSameOrigin, + requireUserTransport, + sessionCookie +} from './http.js'; import { roleAllows, routePermission, type RoutePermission } from './access.js'; import { withRequestUsername } from './request-context.js'; function publicAsset(routeId: string | null, path: string, method: string) { - return routeId === null && ['GET','HEAD'].includes(method) && - (path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/ec-agent-update.py','/agent/install.sh','/agent/manifest.json'].includes(path)); + return ( + routeId === null && + ['GET', 'HEAD'].includes(method) && + (path.startsWith('/_app/') || + path === '/favicon.svg' || + [ + '/agent/ec-agent.py', + '/agent/ec-agent-update.py', + '/agent/install.sh', + '/agent/manifest.json' + ].includes(path)) + ); } /** Streaming responses must lose access when the underlying session does. */ -export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response { +export function protectEventStream( + response: Response, + tokenHash: string, + permission: RoutePermission, + signal: AbortSignal +): Response { if (!response.body) return response; const reader = response.body.getReader(); let timer: ReturnType | undefined; let closed = false; let closeStream: () => void = () => {}; - const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); }; + const cleanup = () => { + if (timer) clearInterval(timer); + signal.removeEventListener('abort', closeStream); + }; const stream = new ReadableStream({ start(controller) { closeStream = () => { - if (closed) return; closed = true; cleanup(); - void reader.cancel().catch(() => {}); controller.close(); + if (closed) return; + closed = true; + cleanup(); + void reader.cancel().catch(() => {}); + controller.close(); }; timer = setInterval(() => { try { const current = readSessionHash(tokenHash); - if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream(); - } catch { closeStream(); } - },15_000); - signal.addEventListener('abort',closeStream,{once:true}); - if (signal.aborted) { closeStream(); return; } + if ( + !current || + current.user.mustChangePassword || + !roleAllows(current.user.role, permission) + ) + closeStream(); + } catch { + closeStream(); + } + }, 15_000); + signal.addEventListener('abort', closeStream, { once: true }); + if (signal.aborted) { + closeStream(); + return; + } }, async pull(controller) { try { const next = await reader.read(); if (closed) return; - if (next.done) { closed=true; cleanup(); controller.close(); } - else controller.enqueue(next.value); - } catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } } + if (next.done) { + closed = true; + cleanup(); + controller.close(); + } else controller.enqueue(next.value); + } catch (cause) { + if (!closed) { + closed = true; + cleanup(); + controller.error(cause); + } + } }, - cancel() { closed=true; cleanup(); return reader.cancel(); } + cancel() { + closed = true; + cleanup(); + return reader.cancel(); + } + }); + return new Response(stream, { + status: response.status, + statusText: response.statusText, + headers: response.headers }); - return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers}); } export const authenticationHandle: Handle = async ({ event, resolve }) => { - event.locals.user = null; event.locals.sessionHash = null; + event.locals.user = null; + event.locals.sessionHash = null; const routeId = event.route.id; const method = event.request.method.toUpperCase(); - if (publicAsset(routeId,event.url.pathname,method)) return resolve(event); - const permission = routePermission(routeId,method); - if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) { - return json({error:'Host credentials only authorize the host agent API.'},{status:403}); + if (publicAsset(routeId, event.url.pathname, method)) return resolve(event); + const permission = routePermission(routeId, method); + if ( + !routeId?.startsWith('/api/agent/') && + /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization') ?? '') + ) { + return json( + { error: 'Host credentials only authorize the host agent API.' }, + { status: 403 } + ); } if (permission === 'machine') return resolve(event); const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/'); try { requireUserTransport(event.url); - if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url); - } catch (cause) { if (api) return authError(cause); throw cause; } + if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) + requireSameOrigin(event.request, event.url); + } catch (cause) { + if (api) return authError(cause); + throw cause; + } const session = readSession(event.cookies.get(sessionCookie(event.url))); - event.locals.user = session?.user??null; - event.locals.sessionHash = session?.tokenHash??null; + event.locals.user = session?.user ?? null; + event.locals.sessionHash = session?.tokenHash ?? null; const setup = setupRequired(); if (permission !== 'public') { if (!session) { - if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}}); - redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`); + if (api) + return json( + { + error: setup + ? 'Complete administrator setup before using the management API.' + : 'Sign in to continue.', + setupRequired: setup + }, + { status: 401, headers: { 'Cache-Control': 'no-store' } } + ); + redirect( + 303, + setup + ? '/setup' + : `/login?next=${encodeURIComponent(event.url.pathname + event.url.search)}` + ); } if (session.user.mustChangePassword && permission !== 'authenticated') { - if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}}); - redirect(303,'/account'); + if (api) + return json( + { + error: 'Change your temporary password from Account before continuing.', + mustChangePassword: true + }, + { status: 403, headers: { 'Cache-Control': 'no-store' } } + ); + redirect(303, '/account'); } - if (!roleAllows(session.user.role,permission)) { - const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.'; - if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}}); - error(403,message); + if (!roleAllows(session.user.role, permission)) { + const message = + permission === 'admin' + ? 'Administrator access is required.' + : 'Operator access is required for this action.'; + if (api) + return json( + { error: message }, + { status: 403, headers: { 'Cache-Control': 'no-store' } } + ); + error(403, message); } // Launch APIs can also save credentials. Operators may launch with saved // keys, but only administrators may introduce or replace server keys. - if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) { + if ( + session.user.role === 'operator' && + method === 'POST' && + ['/api/servers', '/api/deployment-collections'].includes(routeId ?? '') + ) { try { - const body = await authBody(event.request.clone(),1024*1024); - if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) { - return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403}); + const body = await authBody(event.request.clone(), 1024 * 1024); + if ( + body.plutoKey || + (body.keys && + typeof body.keys === 'object' && + Object.values(body.keys).some(Boolean)) + ) { + return json( + { + error: 'An administrator must save server keys before an operator launches these servers.' + }, + { status: 403 } + ); } - } catch (cause) { return authError(cause); } + } catch (cause) { + return authError(cause); + } } - } else if (['/login','/setup'].includes(routeId??'')) { - if (session) redirect(303,session.user.mustChangePassword?'/account':'/'); - if (routeId==='/setup'&&!setup) redirect(303,'/login'); - if (routeId==='/login'&&setup) redirect(303,'/setup'); + } else if (['/login', '/setup'].includes(routeId ?? '')) { + if (session) redirect(303, session.user.mustChangePassword ? '/account' : '/'); + if (routeId === '/setup' && !setup) redirect(303, '/login'); + if (routeId === '/login' && setup) redirect(303, '/setup'); } let response = await withRequestUsername(session?.user.username ?? null, () => resolve(event)); - if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) { - recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname}); + if (session && !['GET', 'HEAD', 'OPTIONS'].includes(method) && response.status < 400) { + recordAudit(session.user.id, 'request', routeId, { + method, + status: response.status, + path: event.url.pathname + }); } if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) { - response = protectEventStream(response,session.tokenHash,permission,event.request.signal); + response = protectEventStream( + response, + session.tokenHash, + permission, + event.request.signal + ); } const headers = new Headers(response.headers); - headers.set('Cache-Control','no-store'); - headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', ')); - headers.set('Referrer-Policy','same-origin'); - headers.set('X-Content-Type-Options','nosniff'); - headers.set('X-Frame-Options','DENY'); - return new Response(response.body,{status:response.status,statusText:response.statusText,headers}); + headers.set('Cache-Control', 'no-store'); + headers.set('Vary', [headers.get('Vary'), 'Cookie'].filter(Boolean).join(', ')); + headers.set('Referrer-Policy', 'same-origin'); + headers.set('X-Content-Type-Options', 'nosniff'); + headers.set('X-Frame-Options', 'DENY'); + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers + }); }; diff --git a/src/lib/server/auth/http.ts b/src/lib/server/auth/http.ts index ed36c3b..33866c6 100644 --- a/src/lib/server/auth/http.ts +++ b/src/lib/server/auth/http.ts @@ -2,48 +2,102 @@ import { error, isHttpError, json, type Cookies } from '@sveltejs/kit'; import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js'; import type { AuthUser } from '../../auth.js'; -export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; } +export function sessionCookie(url: URL): string { + return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; +} export function requireUserTransport(url: URL): void { if (url.protocol === 'https:') return; - if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return; - error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.'); + if ( + process.env.EC_AUTH_ALLOW_HTTP === 'true' && + ['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname) + ) + return; + error( + 426, + 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.' + ); } export function requireSameOrigin(request: Request, url: URL): void { - if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') { + if ( + request.headers.get('origin') !== url.origin || + request.headers.get('sec-fetch-site') === 'cross-site' + ) { error(403, 'This action requires a same-origin request.'); } } export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) { - cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:', - sameSite: 'lax', expires: new Date(result.session.expiresAt) }); - return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } }); + cookies.set(sessionCookie(url), result.session.token, { + path: '/', + httpOnly: true, + secure: url.protocol === 'https:', + sameSite: 'lax', + expires: new Date(result.session.expiresAt) + }); + return json({ user: result.user }, { headers: { 'Cache-Control': 'no-store' } }); } export function requireUser(user: AuthUser | null): AuthUser { - if (!user) throw new AuthError('Sign in to continue.',401); + if (!user) throw new AuthError('Sign in to continue.', 401); return user; } -export async function authBody(request: Request, maxBytes = 8192): Promise> { - if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415); - if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413); - const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.'); - const chunks: Uint8Array[] = []; let length = 0; +export async function authBody( + request: Request, + maxBytes = 8192 +): Promise> { + if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) + throw new AuthError('Send a JSON request.', 415); + if (Number(request.headers.get('content-length')) > maxBytes) + throw new AuthError('Request is too large.', 413); + const reader = request.body?.getReader(); + if (!reader) throw new AuthError('A JSON object is required.'); + const chunks: Uint8Array[] = []; + let length = 0; try { for (;;) { - const { done, value } = await reader.read(); if (done) break; + const { done, value } = await reader.read(); + if (done) break; length += value.byteLength; - if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); } + if (length > maxBytes) { + void reader.cancel().catch(() => {}); + throw new AuthError('Request is too large.', 413); + } chunks.push(value); } - } finally { reader.releaseLock(); } + } finally { + reader.releaseLock(); + } let body: unknown; - try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); } - if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.'); + try { + body = JSON.parse(Buffer.concat(chunks, length).toString('utf8')); + } catch { + throw new AuthError('Invalid JSON.'); + } + if (!body || typeof body !== 'object' || Array.isArray(body)) + throw new AuthError('A JSON object is required.'); return body as Record; } export function authError(cause: unknown): Response { - if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} }); + if (isHttpError(cause)) + return json( + { error: cause.body.message }, + { status: cause.status, headers: { 'Cache-Control': 'no-store' } } + ); const message = cause instanceof Error ? cause.message : 'Authentication request failed.'; - const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500; - return json({ error: status === 500 ? 'The authentication request could not be completed.' : message }, - { status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} }); + const status = + cause instanceof AuthError + ? cause.status + : message.includes('busy') + ? 429 + : message.startsWith('Use a password') + ? 400 + : 500; + return json( + { error: status === 500 ? 'The authentication request could not be completed.' : message }, + { + status, + headers: { + 'Cache-Control': 'no-store', + ...(status === 429 ? { 'Retry-After': '900' } : {}) + } + } + ); } diff --git a/src/lib/server/auth/index.ts b/src/lib/server/auth/index.ts index aef358d..059f9df 100644 --- a/src/lib/server/auth/index.ts +++ b/src/lib/server/auth/index.ts @@ -8,179 +8,415 @@ export const SESSION_COOKIE = '__Host-ec_session'; export const SESSION_LIFETIME_MS = 12 * 60 * 60_000; export const SESSION_IDLE_MS = 30 * 60_000; const digest = (value: string) => createHash('sha256').update(value).digest('hex'); -function database() { const handle = db(); ensureAuthSchema(handle); return handle; } -type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number; - must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null }; +function database() { + const handle = db(); + ensureAuthSchema(handle); + return handle; +} +type UserRow = { + id: string; + username: string; + display_name: string; + password_hash: string; + role: UserRole; + disabled: number; + must_change_password: number; + created_at: number; + updated_at: number; + last_login_at: number | null; +}; export class AuthError extends Error { - constructor(message: string, public status = 400) { super(message); } + constructor( + message: string, + public status = 400 + ) { + super(message); + } } function present(row: UserRow): AuthUser { - return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled, - mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(), - updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() }; + return { + id: row.id, + username: row.username, + displayName: row.display_name, + role: row.role, + disabled: !!row.disabled, + mustChangePassword: !!row.must_change_password, + createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), + lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() + }; +} +function userRow(id: string) { + return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; +} +export function getUser(id: string): AuthUser | null { + const row = userRow(id); + return row ? present(row) : null; +} +export function listUsers(): AuthUser[] { + return ( + database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[] + ).map(present); +} +export function setupRequired(): boolean { + return !database().prepare('SELECT 1 FROM auth_users LIMIT 1').get(); } -function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; } -export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; } -export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); } -export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); } function username(value: unknown): string { - if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.'); + if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) + throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.'); const normalized = value.toLowerCase(); - if (normalized === systemUsername.toLowerCase()) throw new AuthError(`${systemUsername} is reserved for system actions. Choose another username.`); + if (normalized === systemUsername.toLowerCase()) + throw new AuthError( + `${systemUsername} is reserved for system actions. Choose another username.` + ); return normalized; } function displayName(value: unknown, fallback: string): string { const name = value === undefined ? fallback : value; - if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.'); + if ( + typeof name !== 'string' || + !name.trim() || + name.length > 100 || + // eslint-disable-next-line no-control-regex -- Reject control characters in display names. + /[\x00-\x1f\x7f]/.test(name) + ) + throw new AuthError('Display name must contain 1–100 printable characters.'); return name.trim(); } function role(value: unknown): UserRole { - if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.'); + if (!['admin', 'operator', 'viewer'].includes(value as string)) + throw new AuthError('Choose administrator, operator or viewer.'); return value as UserRole; } -export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record = {}): void { - database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)') +export function recordAudit( + actorId: string | null, + action: string, + targetId: string | null = null, + details: Record = {} +): void { + database() + .prepare( + 'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)' + ) .run(actorId, action, targetId, JSON.stringify(details), Date.now()); } export function listAuthAudit() { - return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt - FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all(); + return database() + .prepare( + `SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt + FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200` + ) + .all(); } /** Persistent fixed windows are consumed before hashing, including failed setup. */ export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void { - const now = Date.now(); const windowMs = 15 * 60_000; + const now = Date.now(); + const windowMs = 15 * 60_000; const ipKey = digest(`${kind}:ip:${clientAddress}`); const accountKey = digest(`${kind}:account:${account.toLowerCase()}`); - database().transaction(() => { - database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000); - for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) { - const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined; - if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429); - } - for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1) + database() + .transaction(() => { + database() + .prepare('DELETE FROM auth_rate_limits WHERE window_start < ?') + .run(now - 24 * 60 * 60_000); + for (const [key, limit] of [ + [ipKey, 30], + [accountKey, 10] + ] as const) { + const previous = database() + .prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?') + .get(key) as { window_start: number; attempts: number } | undefined; + if ( + previous && + previous.window_start > now - windowMs && + previous.attempts >= limit + ) + throw new AuthError('Too many attempts. Try again in 15 minutes.', 429); + } + for (const key of [ipKey, accountKey]) + database() + .prepare( + `INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1) ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END, - window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs); - }).immediate(); + window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END` + ) + .run(key, now, now - windowMs, now - windowMs); + }) + .immediate(); } -function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser { - if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409); - const id = randomUUID(); const now = Date.now(); - database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`) - .run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now); +function insertUser(input: { + username: string; + displayName: string; + role: UserRole; + passwordHash: string; + mustChangePassword: boolean; +}): AuthUser { + if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) + throw new AuthError('That username is already in use.', 409); + const id = randomUUID(); + const now = Date.now(); + database() + .prepare( + `INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)` + ) + .run( + id, + input.username, + input.displayName, + input.passwordHash, + input.role, + Number(input.mustChangePassword), + now, + now + ); return getUser(id)!; } -export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } } -export async function bootstrapAdmin(input: Record, clientAddress: string): Promise { +export interface AuthResult { + user: AuthUser; + session: { token: string; expiresAt: number }; +} +export async function bootstrapAdmin( + input: Record, + clientAddress: string +): Promise { consumeAuthAttempt(clientAddress, 'bootstrap', 'setup'); if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409); const expected = process.env.EC_SETUP_TOKEN; - if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503); - if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401); - const name = username(input.username); const label = displayName(input.displayName, name); + if (!expected || expected.length < 32) + throw new AuthError( + 'The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', + 503 + ); + if ( + typeof input.token !== 'string' || + !timingSafeEqual( + Buffer.from(digest(input.token), 'hex'), + Buffer.from(digest(expected), 'hex') + ) + ) + throw new AuthError('Invalid setup token.', 401); + const name = username(input.username); + const label = displayName(input.displayName, name); const passwordHash = await hashPassword(input.password); - return database().transaction(() => { - if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409); - const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false }); - recordAudit(user.id, 'setup', user.id); - return { user, session: createSession(user.id) }; - }).immediate(); + return database() + .transaction(() => { + if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409); + const user = insertUser({ + username: name, + displayName: label, + passwordHash, + role: 'admin', + mustChangePassword: false + }); + recordAudit(user.id, 'setup', user.id); + return { user, session: createSession(user.id) }; + }) + .immediate(); } -export async function authenticatePassword(input: Record, clientAddress: string): Promise { - const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : ''; +export async function authenticatePassword( + input: Record, + clientAddress: string +): Promise { + const name = + typeof input.username === 'string' ? input.username.slice(0, 64).toLowerCase() : ''; consumeAuthAttempt(clientAddress, name); - const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined; + const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as + UserRow | undefined; const valid = await verifyPassword(input.password, row?.password_hash ?? null); - const result = database().transaction(() => { - const current = row ? userRow(row.id) : undefined; - if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) { - recordAudit(null, 'login.failed'); return null; - } - database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id); - recordAudit(current.id, 'login', current.id); - return { user: getUser(current.id)!, session: createSession(current.id) }; - }).immediate(); + const result = database() + .transaction(() => { + const current = row ? userRow(row.id) : undefined; + if ( + !valid || + !current || + current.disabled || + current.password_hash !== row?.password_hash + ) { + recordAudit(null, 'login.failed'); + return null; + } + database() + .prepare('UPDATE auth_users SET last_login_at=? WHERE id=?') + .run(Date.now(), current.id); + recordAudit(current.id, 'login', current.id); + return { user: getUser(current.id)!, session: createSession(current.id) }; + }) + .immediate(); if (!result) throw new AuthError('Invalid username or password.', 401); return result; } export function createSession(userId: string): { token: string; expiresAt: number } { - const now = Date.now(); const user = userRow(userId); + const now = Date.now(); + const user = userRow(userId); if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401); - const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS; - database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS); - database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now); + const token = randomBytes(32).toString('base64url'); + const expiresAt = now + SESSION_LIFETIME_MS; + database() + .prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?') + .run(now, now - SESSION_IDLE_MS); + database() + .prepare( + 'INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)' + ) + .run(digest(token), userId, now, expiresAt, now); return { token, expiresAt }; } -export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null { +export function readSession( + token: string | undefined, + touch = true +): { user: AuthUser; tokenHash: string; expiresAt: number } | null { if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null; return readSessionHash(digest(token), touch); } -export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null { +export function readSessionHash( + tokenHash: string, + touch = false +): { user: AuthUser; tokenHash: string; expiresAt: number } | null { const now = Date.now(); - const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id - WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined; + const row = database() + .prepare( + `SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id + WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0` + ) + .get(tokenHash, now, now - SESSION_IDLE_MS) as + (UserRow & { expires_at: number; last_seen_at: number }) | undefined; if (!row) return null; - if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash); + if (touch && row.last_seen_at < now - 60_000) + database() + .prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?') + .run(now, tokenHash); return { user: present(row), tokenHash, expiresAt: row.expires_at }; } -export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); } +export function revokeSession(token: string | undefined): void { + if (token) + database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); +} export function revokeUserSessions(actorId: string, userId: string): void { database().transaction(() => { requireAdministrator(actorId); - if (!userRow(userId)) throw new AuthError('User not found.',404); + if (!userRow(userId)) throw new AuthError('User not found.', 404); database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId); - recordAudit(actorId,'sessions.revoked',userId); + recordAudit(actorId, 'sessions.revoked', userId); })(); } function requireAdministrator(actorId: string) { const actor = userRow(actorId); - if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403); + if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) + throw new AuthError('Administrator access is required.', 403); } -export async function createUser(actorId: string, input: Record): Promise { +export async function createUser( + actorId: string, + input: Record +): Promise { requireAdministrator(actorId); - const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role); + const name = username(input.username), + label = displayName(input.displayName, name), + userRole = role(input.role); const passwordHash = await hashPassword(input.password); - return database().transaction(() => { - requireAdministrator(actorId); - const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true }); - recordAudit(actorId,'user.created',user.id,{role:userRole}); return user; - }).immediate(); + return database() + .transaction(() => { + requireAdministrator(actorId); + const user = insertUser({ + username: name, + displayName: label, + role: userRole, + passwordHash, + mustChangePassword: true + }); + recordAudit(actorId, 'user.created', user.id, { role: userRole }); + return user; + }) + .immediate(); } -export async function updateUser(actorId: string, id: string, input: Record): Promise { +export async function updateUser( + actorId: string, + id: string, + input: Record +): Promise { requireAdministrator(actorId); - if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.'); - if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.'); + if (input.disabled !== undefined && typeof input.disabled !== 'boolean') + throw new AuthError('Disabled must be true or false.'); + if (input.password !== undefined && actorId === id) + throw new AuthError('Change your own password from Account using your current password.'); const passwordHash = input.password === undefined ? null : await hashPassword(input.password); - return database().transaction(() => { - requireAdministrator(actorId); - const current = userRow(id); if (!current) throw new AuthError('User not found.',404); - const nextRole = input.role === undefined ? current.role : role(input.role); - const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled); - const label = displayName(input.displayName,current.display_name); - if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) { - const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id); - if (!other) throw new AuthError('Keep at least one enabled administrator.',409); - } - database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`) - .run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id); - if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id); - recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash}); - return getUser(id)!; - }).immediate(); + return database() + .transaction(() => { + requireAdministrator(actorId); + const current = userRow(id); + if (!current) throw new AuthError('User not found.', 404); + const nextRole = input.role === undefined ? current.role : role(input.role); + const disabled = + input.disabled === undefined ? current.disabled : Number(input.disabled); + const label = displayName(input.displayName, current.display_name); + if ( + current.role === 'admin' && + !current.disabled && + (nextRole !== 'admin' || disabled) + ) { + const other = database() + .prepare( + "SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1" + ) + .get(id); + if (!other) throw new AuthError('Keep at least one enabled administrator.', 409); + } + database() + .prepare( + `UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?` + ) + .run( + label, + nextRole, + disabled, + passwordHash ?? current.password_hash, + passwordHash ? 1 : current.must_change_password, + Date.now(), + id + ); + if (passwordHash || disabled !== current.disabled || nextRole !== current.role) + database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id); + recordAudit(actorId, 'user.updated', id, { + role: nextRole, + disabled: !!disabled, + passwordReset: !!passwordHash + }); + return getUser(id)!; + }) + .immediate(); } -export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise { +export async function changePassword( + userId: string, + currentPassword: unknown, + newPassword: unknown +): Promise { const original = userRow(userId); - if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400); + if ( + !original || + original.disabled || + !(await verifyPassword(currentPassword, original.password_hash)) + ) + throw new AuthError('Current password is incorrect.', 400); if (currentPassword === newPassword) throw new AuthError('Choose a different password.'); const passwordHash = await hashPassword(newPassword); - return database().transaction(() => { - const current = userRow(userId); - if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409); - database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId); - database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId); - recordAudit(userId,'password.changed',userId); - return { user: getUser(userId)!, session: createSession(userId) }; - }).immediate(); + return database() + .transaction(() => { + const current = userRow(userId); + if (!current || current.disabled || current.password_hash !== original.password_hash) + throw new AuthError( + 'Account changed during password verification. Sign in again.', + 409 + ); + database() + .prepare( + 'UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?' + ) + .run(passwordHash, Date.now(), userId); + database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId); + recordAudit(userId, 'password.changed', userId); + return { user: getUser(userId)!, session: createSession(userId) }; + }) + .immediate(); } diff --git a/src/lib/server/auth/passwords.js b/src/lib/server/auth/passwords.js index 26152d9..b5ce762 100644 --- a/src/lib/server/auth/passwords.js +++ b/src/lib/server/auth/passwords.js @@ -15,12 +15,14 @@ export function validatePassword(password) { /** @param {string} password @param {string} salt @returns {Promise} */ function derive(password, salt) { - if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.')); + if (active >= 2) + return Promise.reject(new Error('Password service is busy. Try again shortly.')); active++; return new Promise((resolve, reject) => { scrypt(password, salt, 64, parameters, (error, result) => { active--; - if (error) reject(error); else resolve(result); + if (error) reject(error); + else resolve(result); }); }); } @@ -37,9 +39,17 @@ export async function hashPassword(password) { export async function verifyPassword(password, encoded) { if (typeof password !== 'string' || [...password].length > 128) return false; const parts = encoded?.split('$'); - const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1' - && /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]); + const valid = + parts?.length === 6 && + parts[0] === 'scrypt' && + parts[1] === '131072' && + parts[2] === '8' && + parts[3] === '1' && + /^[a-f0-9]{64}$/.test(parts[4]) && + /^[a-f0-9]{128}$/.test(parts[5]); // Missing accounts perform the same expensive operation as real accounts. const result = await derive(password, valid ? parts[4] : '0'.repeat(64)); - return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid; + return ( + timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid + ); } diff --git a/src/lib/server/auth/request-context.ts b/src/lib/server/auth/request-context.ts index 65a2ce4..20e7baf 100644 --- a/src/lib/server/auth/request-context.ts +++ b/src/lib/server/auth/request-context.ts @@ -3,11 +3,17 @@ import { AsyncLocalStorage } from 'node:async_hooks'; const requester = new AsyncLocalStorage<{ username: string | null; active: boolean }>(); /** Use only the authenticated session, and stop attribution when the request finishes. */ -export async function withRequestUsername(username: string | null, work: () => T | Promise): Promise { +export async function withRequestUsername( + username: string | null, + work: () => T | Promise +): Promise { const context = { username, active: true }; return requester.run(context, async () => { - try { return await work(); } - finally { context.active = false; } + try { + return await work(); + } finally { + context.active = false; + } }); } diff --git a/src/lib/server/auth/telemetry.ts b/src/lib/server/auth/telemetry.ts index 5698564..8adf991 100644 --- a/src/lib/server/auth/telemetry.ts +++ b/src/lib/server/auth/telemetry.ts @@ -1,24 +1,32 @@ import { createHash, timingSafeEqual } from 'node:crypto'; export type TelemetryAuthentication = - | { ok: true } - | { ok: false; status: 401 | 503; error: string }; + { ok: true } | { ok: false; status: 401 | 503; error: string }; -const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/; -const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value); +const tokenPattern = /^[A-Za-z0-9._~+/-]+=*$/; +const validToken = (value: string): boolean => + value.length >= 32 && value.length <= 512 && tokenPattern.test(value); const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest(); /** Machine telemetry has its own credential; browser sessions never authorize it. */ export function authenticateTelemetry(request: Request): TelemetryAuthentication { const expected = process.env.EC_TELEMETRY_TOKEN ?? ''; if (!validToken(expected)) { - return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' }; + return { + ok: false, + status: 503, + error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' + }; } const authorization = request.headers.get('authorization'); const custom = request.headers.get('x-ec-telemetry-token'); - const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1]; - const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' }); + const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+/-]+=*)$/i)?.[1]; + const denied = (): TelemetryAuthentication => ({ + ok: false, + status: 401, + error: 'A valid game telemetry token is required.' + }); // Reject malformed/ambiguous credentials rather than falling back to a // second header. A shared secret must never come from a URL, cookie or body. @@ -28,7 +36,8 @@ export function authenticateTelemetry(request: Request): TelemetryAuthentication const supplied = bearer ?? custom; if (!supplied) return denied(); const suppliedHash = hash(supplied); - if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied(); + if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) + return denied(); if (!timingSafeEqual(suppliedHash, hash(expected))) return denied(); return { ok: true }; } diff --git a/src/lib/server/autoscaling/admission.ts b/src/lib/server/autoscaling/admission.ts index 07dbf99..bb98bd0 100644 --- a/src/lib/server/autoscaling/admission.ts +++ b/src/lib/server/autoscaling/admission.ts @@ -4,29 +4,74 @@ import { scalingSnapshot } from './snapshot.js'; import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js'; import { deploymentStage } from '../provisioning/history.js'; -export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string } +export interface ScalingIntent { + policy_id: string; + revision: number; + decision: string; + operation_id: string; +} export function scalingIntent(operationId: string): ScalingIntent | undefined { - return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined; + return db() + .prepare( + 'SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0' + ) + .get(operationId) as ScalingIntent | undefined; } /** Rechecked before placement, after asynchronous preparation, and before dispatch. */ export function scalingAdmission(operationId: string): string | null { const intent = scalingIntent(operationId); if (!intent) return null; - const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined; + const row = db() + .prepare('SELECT * FROM autoscaling_policies WHERE id=?') + .get(intent.policy_id) as + | { + id: string; + settings: string; + mode: ScalingPolicy['mode']; + revision: number; + paused_reason: string | null; + owns_map: number; + } + | undefined; let reason: string | null = null; - const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1'; - if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.'; + const paused = + ( + db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as + { value: string } | undefined + )?.value === '1'; + if ( + !row || + !liveScaling(row.mode) || + !row.owns_map || + row.revision !== intent.revision || + row.paused_reason || + paused + ) + reason = 'Autoscaling policy is paused or changed.'; else { - const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' }; + const policy: ScalingPolicy = { + ...JSON.parse(row.settings), + id: row.id, + mode: row.mode, + revision: row.revision, + pausedReason: null, + updatedAt: '' + }; const requested = JSON.parse(intent.decision) as ScalingDecision; const snapshot = scalingSnapshot(policy, Date.now(), operationId); if (requested.action === 'create') { - snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server); - snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`); + snapshot.reservedNames = snapshot.reservedNames.filter((n) => n !== requested.server); + snapshot.blockers = snapshot.blockers.filter( + (b) => b !== `Unresolved native instance ${requested.server}.` + ); } const current = planScaling(policy, snapshot, Date.now()); - if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`; + if (current.action !== requested.action || current.server !== requested.server) + reason = `Autoscaling intent no longer eligible: ${current.reason}`; } - if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } }); + if (reason) + deploymentStage(operationId, reason, { + blocker: { code: 'autoscaling', message: reason, automatic: true } + }); return reason; } diff --git a/src/lib/server/autoscaling/authorization.ts b/src/lib/server/autoscaling/authorization.ts index ff952da..503a7cf 100644 --- a/src/lib/server/autoscaling/authorization.ts +++ b/src/lib/server/autoscaling/authorization.ts @@ -7,46 +7,106 @@ export const autoscalingRemovalCapability = 'instance.autoscaling-remove'; export function removalAuthorization(operationId: string, hostId: string): object | null { const intent = scalingIntent(operationId); if (!intent) return null; - if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.'); + if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) + throw new Error('Update the host agent before automatic removal.'); const reason = scalingAdmission(operationId); if (reason) throw new Error(reason); const expiresAt = Date.now() + 60_000; - db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)') + db() + .prepare( + 'INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)' + ) .run(operationId, intent.policy_id, intent.revision, expiresAt); return { policyId: intent.policy_id, revision: intent.revision, expiresAt }; } /** Claim is the cancellation boundary. Lost replies never authorize a second claim. */ -export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record): { authorized: boolean; validForMs?: number; reason?: string } { +export function claimRemovalAuthorization( + hostId: string, + commandId: string, + input: Record +): { authorized: boolean; validForMs?: number; reason?: string } { return db().transaction(() => { const now = Date.now(); - const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a + const row = db() + .prepare( + `SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation - WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as - { operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined; - if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at - || row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now - || row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' }; + WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL` + ) + .get(commandId, hostId) as + | { + operation_id: string; + policy_id: string; + revision: number; + expires_at: number; + claimed_at: number | null; + revoked_at: number | null; + deployment_state: string; + command_state: string; + } + | undefined; + if ( + !row || + input.policyId !== row.policy_id || + input.revision !== row.revision || + input.expiresAt !== row.expires_at || + row.claimed_at !== null || + row.revoked_at !== null || + row.expires_at <= now || + row.deployment_state !== 'running' || + !['pending', 'accepted', 'running'].includes(row.command_state) + ) + return { + authorized: false, + reason: 'Removal authorization expired, changed, or was already claimed.' + }; const reason = scalingAdmission(row.operation_id); - if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; } - db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id); + if (reason) { + db() + .prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?') + .run(now, row.operation_id); + return { authorized: false, reason }; + } + db() + .prepare( + 'UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL' + ) + .run(now, row.operation_id); // Only a short final occupancy check may occur after this boundary. return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) }; })(); } export function revokePolicyRemovals(policyId: string): void { - if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return; - db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId); + if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) + return; + db() + .prepare( + 'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL' + ) + .run(Date.now(), policyId); const at = new Date().toISOString(); - const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id + const commands = db() + .prepare( + `SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id - WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[]; + WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL` + ) + .all(policyId) as { id: string }[]; for (const command of commands) { const result = JSON.stringify({ autoscalingInvalidated: true }); - db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id); - db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at); + db() + .prepare( + "UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?" + ) + .run(result, at, command.id); + db() + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')" + ) + .run(command.id, at); } } diff --git a/src/lib/server/autoscaling/index.ts b/src/lib/server/autoscaling/index.ts index 37b6ef2..d48b502 100644 --- a/src/lib/server/autoscaling/index.ts +++ b/src/lib/server/autoscaling/index.ts @@ -1,6 +1,13 @@ import { createHash, randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; -import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingEvaluation, type ScalingReport, type ScalingView } from '../../autoscaling.js'; +import { + liveScaling, + scalingSettings, + type ScalingPolicy, + type ScalingEvaluation, + type ScalingReport, + type ScalingView +} from '../../autoscaling.js'; import { emptyObservation, planScaling } from './planner.js'; import { parseServerName } from '../../map-aliases.js'; import { scalingSnapshot } from './snapshot.js'; @@ -14,33 +21,98 @@ import { getHost } from '../hosts/index.js'; import { scalingOwner } from './ownership.js'; import { getServer } from '../fleet/servers.js'; -export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } } -interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string } -const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at }); -export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present); -export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); } -export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1'; +export class ScalingError extends Error { + constructor( + message: string, + public status = 400 + ) { + super(message); + } +} +interface PolicyRow { + id: string; + revision: number; + settings: string; + mode: ScalingPolicy['mode']; + paused_reason: string | null; + updated_at: string; +} +const present = (row: PolicyRow): ScalingPolicy => ({ + ...JSON.parse(row.settings), + id: row.id, + revision: row.revision, + mode: row.mode, + pausedReason: row.paused_reason, + updatedAt: row.updated_at +}); +export const listScalingPolicies = (): ScalingPolicy[] => + ( + db() + .prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug') + .all() as PolicyRow[] + ).map(present); +export function getScalingPolicy(id: string): ScalingPolicy | undefined { + const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as + PolicyRow | undefined; + return row && present(row); +} +export const scalingGloballyPaused = (): boolean => + ( + db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as + { value: string } | undefined + )?.value === '1'; export function evaluateScaling(input: unknown, id?: string, revision?: number): ScalingEvaluation { - const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined; + const settings = scalingSettings(input), + existing = id ? getScalingPolicy(id) : undefined; if (id && !existing) throw new ScalingError('Policy not found.', 404); - if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409); - if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.'); - const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'new-policy', revision: existing?.revision ?? 0, - pausedReason: null, updatedAt: existing?.updatedAt ?? '' }; + if (existing && existing.revision !== revision) + throw new ScalingError('Policy changed. Refresh before editing.', 409); + if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) + throw new ScalingError('A policy cannot change its game or map.'); + const policy: ScalingPolicy = { + ...settings, + id: existing?.id ?? 'new-policy', + revision: existing?.revision ?? 0, + pausedReason: null, + updatedAt: existing?.updatedAt ?? '' + }; const snapshot = scalingSnapshot(policy); const blockers: string[] = []; if (liveScaling(policy.mode)) { const owner = scalingOwner(policy.game, policy.mapSlug); - if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`); + if (owner && owner !== policy.id) + blockers.push( + `An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}` + ); blockers.push(...snapshot.blockers); - for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`); - if (policy.mode === 'create-destroy') for (const member of snapshot.members) { - const hostId = getServer(member.name)?.host_id; - if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`); + for (const member of snapshot.members) + if (!['occupied', 'empty'].includes(member.state)) + blockers.push( + `${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.` + ); + if (policy.mode === 'create-destroy') + for (const member of snapshot.members) { + const hostId = getServer(member.name)?.host_id; + if ( + !hostId || + !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability) + ) + blockers.push( + `Update the host agent for ${member.name} before enabling automatic removal.` + ); + } + if ( + db() + .prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0') + .get(policy.id) + ) + blockers.push('Settle the current autoscaling operation before editing a live policy.'); + try { + nativeLaunchSelection({ automatic: true }, policy.game); + } catch (error) { + blockers.push((error as Error).message); } - if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.'); - try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); } } const decision = planScaling(policy, snapshot, Date.now()); return { policy, snapshot, decision, blockers }; @@ -50,16 +122,48 @@ export function saveScaling(input: unknown, id?: string, revision?: number): Sca return db().transaction(() => { const evaluation = evaluateScaling(input, id, revision); if (evaluation.blockers.length) throw new ScalingError(evaluation.blockers[0]); - const policy = evaluation.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString(); + const policy = evaluation.policy, + settings = scalingSettings(policy), + nextId = id ?? randomUUID(), + at = new Date().toISOString(); if (id) revokePolicyRemovals(id); - if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409); - db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?) - ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at); + if ( + !id && + db() + .prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?') + .get(policy.game, policy.mapSlug) + ) + throw new ScalingError('A policy already exists for this scope.', 409); + db() + .prepare( + `INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at` + ) + .run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at); // An edit starts a new observation interval; earlier inactivity is not authorization. db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId); - db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId); - if (liveScaling(policy.mode)) for (const member of evaluation.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)') - .run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() })); + db() + .prepare( + 'UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?' + ) + .run(liveScaling(policy.mode) ? 1 : 0, nextId); + if (liveScaling(policy.mode)) + for (const member of evaluation.snapshot.members) + db() + .prepare( + 'INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)' + ) + .run( + nextId, + member.name, + member.identity, + JSON.stringify({ + identity: member.identity, + observedAt: member.observedAt, + emptySince: null, + evaluatedAt: Date.now() + }) + ); reconcileScaling(getScalingPolicy(nextId)!); return getScalingPolicy(nextId)!; })(); @@ -69,10 +173,18 @@ export function deleteScaling(id: string, revision: number): void { db().transaction(() => { const policy = getScalingPolicy(id); if (!policy) throw new ScalingError('Policy not found.', 404); - if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409); + if (policy.revision !== revision) + throw new ScalingError('Policy changed. Refresh before deleting.', 409); // Keep admission evidence and ownership while a host command may still act. - if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) { - throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409); + if ( + db() + .prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0') + .get(id) + ) { + throw new ScalingError( + 'Pause this policy and settle its outstanding Provisioning work before deleting it.', + 409 + ); } revokePolicyRemovals(id); db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id); @@ -85,39 +197,105 @@ export function deleteScaling(id: string, revision: number): void { export function setScalingGlobalPause(paused: boolean): void { db().transaction(() => { - db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0'); - if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); } + db() + .prepare( + "INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value" + ) + .run(paused ? '1' : '0'); + if (paused) + for (const policy of listScalingPolicies()) { + revokePolicyRemovals(policy.id); + reconcileScaling(policy); + } })(); } function reconcileScaling(policy: ScalingPolicy): void { - const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id + const intents = db() + .prepare( + `SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id - LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as - { id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[]; + LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0` + ) + .all(policy.id) as { + id: string; + revision: number; + operation_id: string; + action: string; + state: string; + submitted_at: string | null; + command_id: string | null; + }[]; for (const intent of intents) { - if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at - && (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false); + if ( + intent.action === 'create' && + ['queued', 'waiting', 'cancelled'].includes(intent.state) && + !intent.command_id && + !intent.submitted_at && + (scalingAdmission(intent.operation_id) || intent.state === 'cancelled') + ) + cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false); if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) { revokePolicyRemovals(policy.id); if (!intent.command_id) cancelRemoval(intent.operation_id, false); } - const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null }; - const terminal = ['done','warning','failed','cancelled'].includes(current.state); - const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined; - const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true; - const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed' - && !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id); - const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending; - if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) { - db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id); + const current = db() + .prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?') + .get(intent.operation_id) as { state: string; server_deleted_at: string | null }; + const terminal = ['done', 'warning', 'failed', 'cancelled'].includes(current.state); + const command = intent.command_id + ? (db() + .prepare('SELECT state,result FROM ec_host_operations WHERE id=?') + .get(intent.command_id) as { state: string; result: string | null } | undefined) + : undefined; + const invalidated = + command?.state === 'failed' && + command.result && + JSON.parse(command.result).autoscalingInvalidated === true; + const receiptPending = + current.state === 'cancelled' && + !current.server_deleted_at && + command?.state === 'completed' && + !!db() + .prepare( + 'SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL' + ) + .get(intent.operation_id); + const settled = + (!intent.command_id || + (!!command && ['completed', 'failed'].includes(command.state))) && + !receiptPending; + if ( + !invalidated && + (current.state === 'failed' || + current.state === 'interrupted' || + command?.state === 'unknown') && + liveScaling(policy.mode) + ) { + db() + .prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?') + .run( + `Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, + new Date().toISOString(), + policy.id + ); } if (terminal && settled) { db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id); - if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id); + if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) + db() + .prepare( + "UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?" + ) + .run(policy.id); } } - if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) { + if ( + !liveScaling(policy.mode) && + !db() + .prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0') + .get(policy.id) + ) { db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id); } } @@ -131,48 +309,152 @@ export async function runAutoscaling(): Promise { if (policy.mode === 'off') continue; db().transaction(() => { const snapshot = scalingSnapshot(policy, now); - if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); - const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined; + if (scalingGloballyPaused()) + snapshot.blockers.unshift('Autoscaling is globally paused.'); + const pending = db() + .prepare( + `SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0` + ) + .get(policy.id) as { id: string; stage: string } | undefined; if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`); const decision = planScaling(policy, snapshot, now); for (const member of snapshot.members) { // Keep the observation interval that justified a pending removal; // admission reads fresh evidence and revokes it on any new blocker. - if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue; - const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now); - db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?) - ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation)); + if ( + member.state === 'removing' && + db() + .prepare( + 'SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0' + ) + .get(policy.id, member.operationId) + ) + continue; + const observation = emptyObservation( + member, + { + identity: member.identity, + observedAt: member.observedAt, + emptySince: member.emptySince, + evaluatedAt: now + }, + now + ); + db() + .prepare( + `INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?) + ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation` + ) + .run(policy.id, member.name, member.identity, JSON.stringify(observation)); } // Evidence includes provenance and timestamps but never keys or player rosters. - const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers }); - const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex'); - const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined; - const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy'); + const evidence = JSON.stringify({ + members: snapshot.members, + reservations: snapshot.reservedNames, + keyPool: snapshot.keyPool, + blockers: snapshot.blockers + }); + const fingerprint = createHash('sha256') + .update( + JSON.stringify([ + policy.revision, + decision, + snapshot.members.map((m) => [m.name, m.identity, m.state]) + ]) + ) + .digest('hex'); + const last = db() + .prepare( + 'SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1' + ) + .get(policy.id) as { id: string; fingerprint: string } | undefined; + const execute = + liveScaling(policy.mode) && + (decision.action === 'create' || decision.action === 'destroy'); const decisionId = randomUUID(); - if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); - else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)') - .run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence); + if (!execute && last?.fingerprint === fingerprint) + db() + .prepare( + 'UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?' + ) + .run(now, evidence, last.id); + else + db() + .prepare( + 'INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)' + ) + .run( + decisionId, + policy.id, + policy.revision, + now, + now, + fingerprint, + JSON.stringify(decision), + evidence + ); if (execute) { - db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId); + db() + .prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?') + .run(decisionId); try { db().transaction(() => { - const result = decision.action === 'create' - ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug, - selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId }) - : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId); + const result = + decision.action === 'create' + ? queueNativeCreationIntent({ + server: decision.server!, + game: policy.game, + mapSlug: parseServerName(decision.server!)!.mapSlug, + selection: nativeLaunchSelection( + { automatic: true }, + policy.game + ), + createdBy: `autoscaling:${policy.id}`, + autoscalingDecisionId: decisionId + }) + : destroyServer( + decision.server!, + `autoscaling:${policy.id}`, + decisionId + ); if (!result.ok) throw new Error(result.message); - if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.'); - db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId); - db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server); + if (!result.deploymentId) + throw new Error('Provisioning did not return an operation identity.'); + db() + .prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?') + .run(result.deploymentId, decisionId); + db() + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)" + ) + .run(result.deploymentId, policy.id, decision.server); submitted = true; })(); } catch (error) { - db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId); - db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id); + db() + .prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?') + .run(decisionId); + db() + .prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?') + .run( + `Scaling action could not be queued: ${(error as Error).message}`, + policy.id + ); } } - db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`) - .run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit); + db() + .prepare( + `INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)` + ) + .run( + policy.id, + Math.floor(now / 15_000) * 15_000, + policy.mode, + decision.occupied, + decision.empty, + decision.unknown, + decision.deficit + ); })(); } return submitted; @@ -180,37 +462,96 @@ export async function runAutoscaling(): Promise { export function pruneAutoscaling(now = Date.now()): void { const cutoff = now - 31 * 86400_000; - db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at { + const policies: ScalingView[] = listScalingPolicies().map((policy) => { const snapshot = scalingSnapshot(policy); if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); const decision = planScaling(policy, snapshot, Date.now()); - const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state + const history = ( + db() + .prepare( + `SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id - WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as - { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[]) - .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state })); - const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled']; - const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map; + WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30` + ) + .all(policy.id) as { + id: string; + evaluated_at: number; + last_seen_at: number; + decision: string; + operation_id: string | null; + operation_state: string | null; + }[] + ).map((row) => ({ + id: row.id, + at: row.evaluated_at, + lastSeenAt: row.last_seen_at, + decision: JSON.parse(row.decision), + operationId: row.operation_id, + operationState: row.operation_state + })); + const unsettled = db() + .prepare( + `SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0` + ) + .all(policy.id) as ScalingView['unsettled']; + const ownsMap = !!( + db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { + owns_map: number; + } + ).owns_map; // Integrate only bounded intervals between adjacent samples; outages contribute no invented time. - const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration + const totals = db() + .prepare( + `WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes, COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours, COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes, - COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals']; - const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals, + COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000` + ) + .get(policy.id) as ScalingView['totals']; + const operations = db() + .prepare( + `SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals, COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed, COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples, AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds, MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id - WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations']; - return { policy, snapshot, decision, blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap }; + WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?` + ) + .get( + policy.id, + new Date(Date.now() - 31 * 86400_000).toISOString() + ) as ScalingView['operations']; + return { + policy, + snapshot, + decision, + blockers: [], + history, + lastEvaluatedAt: history[0]?.lastSeenAt ?? null, + totals, + operations, + unsettled, + ownsMap + }; }); - return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true }; + return { + policies, + globalPaused: scalingGloballyPaused(), + liveAvailable: true, + removalAvailable: true + }; } diff --git a/src/lib/server/autoscaling/ownership.ts b/src/lib/server/autoscaling/ownership.ts index 0e2abcc..6494a2a 100644 --- a/src/lib/server/autoscaling/ownership.ts +++ b/src/lib/server/autoscaling/ownership.ts @@ -4,18 +4,29 @@ import { liveScaling, type ScalingPolicy } from '../../autoscaling.js'; import { mapIdentity } from './snapshot.js'; function owningPolicy(game: string, mapSlug: string) { - return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1") - .get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined; + return db() + .prepare( + "SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1" + ) + .get(game, mapSlug, mapSlug) as + { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined; } /** Durable ownership prevents overlapping policies even while scaling is paused. */ export function scalingOwner(game: string, mapSlug: string): string | null { return owningPolicy(game, mapSlug)?.id ?? null; } -const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`; +const ownershipMessage = (owner: string) => + `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`; function manualLineupBlocker(game: string, mapSlug: string): string | null { const owner = owningPolicy(game, mapSlug); if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null; - if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null; + if ( + ( + db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as + { value: string } | undefined + )?.value === '1' + ) + return null; return ownershipMessage(owner.id); } export function scalingOwnershipBlocker(server: string): string | null { @@ -30,13 +41,30 @@ export function assertScalingOwnership(game: string, mapSlug: string, decisionId } const owner = scalingOwner(game, mapSlug); if (!owner) throw new Error('Autoscaling no longer owns this map.'); - if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return; + if ( + db() + .prepare( + 'SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL' + ) + .get(decisionId, owner) + ) + return; throw new Error(ownershipMessage(owner)); } /** Operator cancellation is a policy pause, not a fresh scaling signal. */ export function pauseScalingForOperation(operationId: string): void { - const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined; + const row = db() + .prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?') + .get(operationId) as { policy_id: string } | undefined; if (!row) return; - db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id); - db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id); + db() + .prepare( + "UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?" + ) + .run(new Date().toISOString(), row.policy_id); + db() + .prepare( + 'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL' + ) + .run(Date.now(), row.policy_id); } diff --git a/src/lib/server/autoscaling/planner.ts b/src/lib/server/autoscaling/planner.ts index 430d6f2..216d854 100644 --- a/src/lib/server/autoscaling/planner.ts +++ b/src/lib/server/autoscaling/planner.ts @@ -1,72 +1,170 @@ -import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js'; +import type { + ScalingDecision, + ScalingMember, + ScalingPolicy, + ScalingSnapshot +} from '../../autoscaling.js'; import { parseServerName, zombieMaps } from '../../map-aliases.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; -export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number } +export interface EmptyObservation { + identity: string; + observedAt: number | null; + emptySince: number | null; + evaluatedAt: number; +} /** Only distinct, continuous evidence can advance an empty interval. */ -export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation { +export function emptyObservation( + member: ScalingMember, + previous: EmptyObservation | undefined, + now: number +): EmptyObservation { const stamp = member.observedAt; - const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS; - const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null - && (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) - && previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS - && now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS; - return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now }; + const empty = + member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS; + const continuous = + empty && + previous?.identity === member.identity && + previous.emptySince !== null && + (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) && + previous.observedAt !== null && + stamp >= previous.observedAt && + stamp - previous.observedAt <= ONLINE_MAX_MS && + now >= previous.evaluatedAt && + now - previous.evaluatedAt <= ONLINE_MAX_MS; + return { + identity: member.identity, + observedAt: stamp, + emptySince: empty ? (continuous ? previous!.emptySince : stamp) : null, + evaluatedAt: now + }; } -export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision { +export function planScaling( + policy: ScalingPolicy, + snapshot: ScalingSnapshot, + now: number +): ScalingDecision { if (policy.mapSlug === '*') { const maps = zombieMaps(policy.game); - const decisions = maps.map(map => { - const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug; - return planScaling({ ...policy, mapSlug: map.slug }, { - ...snapshot, members: snapshot.members.filter(m => matches(m.name)), - reservedNames: snapshot.reservedNames.filter(matches) - }, now); + const decisions = maps.map((map) => { + const matches = (name: string) => + (snapshot.members.find((m) => m.name === name)?.mapSlug ?? + parseServerName(name)?.mapSlug) === map.slug; + return planScaling( + { ...policy, mapSlug: map.slug }, + { + ...snapshot, + members: snapshot.members.filter((m) => matches(m.name)), + reservedNames: snapshot.reservedNames.filter(matches) + }, + now + ); }); // Serialize game-wide work and prefer replenishing any map before trimming another. - const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing'); - const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy') - ?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0]; + const pending = snapshot.members.find( + (m) => m.state === 'creating' || m.state === 'removing' + ); + const selected = + decisions.find((d) => d.action === 'create') ?? + decisions.find((d) => d.action === 'destroy') ?? + decisions.find((d) => d.deficit > 0 || d.unknown > 0) ?? + decisions[0]; const map = maps[decisions.indexOf(selected)]; - return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server, - reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`, - desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0), - empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) }; + return { + ...selected, + action: pending ? 'wait' : selected.action, + server: pending ? null : selected.server, + reason: pending + ? (pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.`) + : `${map.friendly}: ${selected.reason}`, + desired: decisions.reduce((n, d) => n + d.desired, 0), + deficit: decisions.reduce((n, d) => n + d.deficit, 0), + occupied: decisions.reduce((n, d) => n + d.occupied, 0), + empty: decisions.reduce((n, d) => n + d.empty, 0), + pending: decisions.reduce((n, d) => n + d.pending, 0), + unknown: decisions.reduce((n, d) => n + d.unknown, 0) + }; } const { members } = snapshot; - const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length; - const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length; - const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length; + const occupied = members.filter((m) => m.state === 'occupied').length, + empty = members.filter((m) => m.state === 'empty').length; + const pending = members.filter((m) => m.state === 'creating' || m.state === 'removing').length; + const unknown = members.filter((m) => m.state === 'unknown' || m.state === 'stopped').length; const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares)); - const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision => - ({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown }); + const result = ( + action: ScalingDecision['action'], + reason: string, + server: string | null = null + ): ScalingDecision => ({ + action, + reason, + server, + desired, + deficit: Math.max(0, policy.spares - empty), + occupied, + empty, + pending, + unknown + }); if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.'); if (policy.pausedReason) return result('wait', policy.pausedReason); if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]); - const blocked = members.find(m => !['occupied', 'empty'].includes(m.state)); - if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`); - if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.'); + const blocked = members.find((m) => !['occupied', 'empty'].includes(m.state)); + if (blocked) + return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`); + if ( + members.some( + (m) => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS + ) + ) + return result('wait', 'Waiting for fresh occupancy evidence.'); if (occupied + empty < desired) { - if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.'); - if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total - ? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.` - : `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`); - const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`) - .find(name => !snapshot.reservedNames.includes(name)); - return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.'); + if (snapshot.reservedNames.length >= policy.maximum) + return result('wait', 'Maximum lobby reservations reached.'); + if (!snapshot.keyPool.available) + return result( + 'wait', + snapshot.keyPool.total + ? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.` + : `No Plutonium keys are saved for ${policy.game.toUpperCase()}.` + ); + const available = Array.from( + { length: 20 }, + (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}` + ).find((name) => !snapshot.reservedNames.includes(name)); + return available + ? result('create', 'Replenish empty lobby capacity.', available) + : result('wait', 'No unused lobby identity is available.'); } if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) { // Protect the lowest-numbered spares before checking removal timers. // Lobby 1 must survive even when a higher lobby has only just emptied. - const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) => - (parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - || a.name.localeCompare(b.name, 'en', { numeric: true })); + const orderedEmpty = members + .filter((m) => m.state === 'empty') + .sort( + (a, b) => + (parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - + (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) || + a.name.localeCompare(b.name, 'en', { numeric: true }) + ); const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied); - const candidates = orderedEmpty.slice(requiredEmpty).reverse() - .filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000); - if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name); + const candidates = orderedEmpty + .slice(requiredEmpty) + .reverse() + .filter( + (m) => + m.emptySince !== null && + m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000 + ); + if (candidates.length) + return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name); return result('wait', 'Waiting for a continuous empty interval.'); } - return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.'); + return result( + 'wait', + empty < policy.spares + ? 'Maximum reached; the empty spare target cannot be met.' + : 'Empty spare target is satisfied.' + ); } diff --git a/src/lib/server/autoscaling/schema.ts b/src/lib/server/autoscaling/schema.ts index db3eaf2..04eaeef 100644 --- a/src/lib/server/autoscaling/schema.ts +++ b/src/lib/server/autoscaling/schema.ts @@ -36,20 +36,28 @@ export function ensureAutoscalingSchema(handle: Database.Database): void { `); handle.transaction(() => { const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[]; - if (columns.some(column => column.name === 'next_action_at')) { + if (columns.some((column) => column.name === 'next_action_at')) { handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at'); } handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes') WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`); // Removed modes must not silently gain permission to destroy lobbies. // Keep ownership while delivered work settles through normal reconciliation. - handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=? + handle + .prepare( + `UPDATE autoscaling_authorizations SET revoked_at=? WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN - (SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now()); - handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'), + (SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))` + ) + .run(Date.now()); + handle + .prepare( + `UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'), revision=revision+1, paused_reason='This autoscaling mode was removed. Review and turn autoscaling On to resume.', owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0) THEN owns_map ELSE 0 END, updated_at=? - WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString()); + WHERE mode IN ('dry-run','create-only')` + ) + .run(new Date().toISOString()); })(); } diff --git a/src/lib/server/autoscaling/snapshot.ts b/src/lib/server/autoscaling/snapshot.ts index 4b74c49..7a80701 100644 --- a/src/lib/server/autoscaling/snapshot.ts +++ b/src/lib/server/autoscaling/snapshot.ts @@ -7,74 +7,218 @@ import { freshStatusTimestamp } from '../../server-freshness.js'; import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js'; import { emptyObservation, type EmptyObservation } from './planner.js'; -export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null { - const parsed = parseServerName(server.name), game = server.game ?? parsed?.game; +export function mapIdentity(server: { + name: string; + game?: string | null; + map_slug?: string | null; + map?: string | null; +}): { game: string; mapSlug: string } | null { + const parsed = parseServerName(server.name), + game = server.game ?? parsed?.game; if (!game) return null; - if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug }; + if (server.map_slug && zombieMaps(game).some((m) => m.slug === server.map_slug)) + return { game, mapSlug: server.map_slug }; if (server.map_slug) return null; - if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug }; - const matches = zombieMaps(game).filter(m => m.canonical === server.map); + if ( + parsed?.game === game && + parsed.mapCanonical && + (!server.map || parsed.mapCanonical === server.map) + ) + return { game, mapSlug: parsed.mapSlug }; + const matches = zombieMaps(game).filter((m) => m.canonical === server.map); return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null; } export function protectedWork(server: string, exceptOperation = ''): string | null { const handle = db(); - if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta'; - if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work'; - if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work'; - if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery'; + if ( + handle + .prepare( + `SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')` + ) + .get(server) + ) + return 'Active beta'; + if ( + handle + .prepare( + `SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')` + ) + .get(server) + ) + return 'Campaign work'; + if ( + handle + .prepare( + `SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')` + ) + .get(server) + ) + return 'Maintenance work'; + if ( + handle + .prepare( + `SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)` + ) + .get(server, exceptOperation) + ) + return 'Unresolved recovery'; return null; } -export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot { - const handle = db(), observe = observationReader(); +export function scalingSnapshot( + policy: ScalingPolicy, + now = Date.now(), + exceptOperation = '' +): ScalingSnapshot { + const handle = db(), + observe = observationReader(); const rows = listServers(); - const matches = (value: Parameters[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); }; + const matches = (value: Parameters[0]) => { + const identity = mapIdentity(value); + return ( + identity?.game === policy.game && + (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug) + ); + }; const blockers: string[] = []; - for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`); - const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[]; - const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j - JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[]; + for (const row of rows) + if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) + blockers.push(`Resolve ambiguous map identity for ${row.name}.`); + const operations = handle + .prepare( + `SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC` + ) + .all(exceptOperation) as { + id: string; + server: string; + action: string; + state: string; + stage: string; + }[]; + const reservations = handle + .prepare( + `SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j + JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?` + ) + .all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[]; const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => { - const observation = observe(row), operation = operations.find(o => o.server === row.name); + const observation = observe(row), + operation = operations.find((o) => o.server === row.name); const protection = protectedWork(row.name, exceptOperation); - const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined; + const host = row.host_id + ? (handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as + { mode: string } | undefined) + : undefined; const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`; - let state: ScalingMember['state'] = 'unknown', blocker: string | null = null; + let state: ScalingMember['state'] = 'unknown', + blocker: string | null = null; let players = observation.health === 'online' ? observation.last_players : null; - let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null; + let observedAt = + players !== null && observation.last_seen_at + ? Date.parse(observation.last_seen_at) + : null; let source = observation.status_source; // Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto. - if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now) - && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) { - players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon'; + if ( + (row.last_players ?? 0) > 0 && + freshStatusTimestamp(row.last_seen_at, now) && + (!observation.telemetry_after || + Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after)) + ) { + players = row.last_players; + observedAt = Date.parse(row.last_seen_at!); + source = 'rcon'; } - if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; } - else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; } - else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped'; - else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty'; - const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now); - return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null, - emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null }; + if (protection || !row.native_instance_id || host?.mode !== 'active') { + state = 'protected'; + blocker = protection ?? 'Native instance on an active host required'; + } else if (operation) { + state = + operation.action === 'create' + ? 'creating' + : operation.action === 'destroy' + ? 'removing' + : 'protected'; + blocker = `Provisioning ${operation.id}: ${operation.stage}`; + } else if ( + row.desired_state === 'stopped' || + ['stopped', 'exited', 'created', 'dead'].includes(row.container_state ?? '') + ) + state = 'stopped'; + else if ( + observation.health === 'online' && + players !== null && + observedAt !== null && + observedAt <= now && + freshStatusTimestamp(new Date(observedAt).toISOString(), now) + ) + state = players > 0 ? 'occupied' : 'empty'; + const transitions = [row.inactive_since, observation.inactive_since] + .map((at) => Date.parse(at ?? '')) + .filter((at) => Number.isFinite(at) && at <= now); + return { + name: row.name, + mapSlug: mapIdentity(row)!.mapSlug, + identity, + state, + players, + observedAt, + source: source ?? null, + emptySince: null, + emptyStartedAt: transitions.length ? Math.max(...transitions) : null, + blocker, + operationId: operation?.id ?? null + }; }); - for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) { - const operation = operations.find(o => o.server === row.name); - members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null }); - } - const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[]; + for (const row of reservations.filter(matches)) + if (!members.some((m) => m.name === row.name)) { + const operation = operations.find((o) => o.server === row.name); + members.push({ + name: row.name, + mapSlug: row.map_slug, + identity: 'reserved', + state: 'creating', + players: null, + observedAt: null, + source: null, + emptySince: null, + blocker: operation?.stage ?? 'Creation is pending.', + operationId: operation?.id ?? null + }); + } + const retained = handle + .prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?') + .all(policy.id) as { server: string; observation: string }[]; for (const member of members) { - const saved = retained.find(r => r.server === member.name); - const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined; + const saved = retained.find((r) => r.server === member.name); + const previous = saved ? (JSON.parse(saved.observation) as EmptyObservation) : undefined; member.emptySince = emptyObservation(member, previous, now).emptySince; } - const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]); - for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) { - reservedNames.add(row.name); - if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`); - } - for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name); - const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined; + const reservedNames = new Set([ + ...rows.filter(matches).map((r) => r.name), + ...reservations.filter(matches).map((r) => r.name) + ]); + for (const row of handle + .prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL') + .all() as { name: string }[]) + if (matches(row)) { + reservedNames.add(row.name); + if (!members.some((m) => m.name === row.name)) + blockers.push(`Unresolved native instance ${row.name}.`); + } + for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { + name: string; + }[]) + if (matches(row)) reservedNames.add(row.name); + const ownServer = exceptOperation + ? ( + handle + .prepare('SELECT server FROM server_deployments WHERE id=?') + .get(exceptOperation) as { server: string } | undefined + )?.server + : undefined; const { total, available } = serverKeyPool(policy.game, ownServer); return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers }; } diff --git a/src/lib/server/backends/files.ts b/src/lib/server/backends/files.ts index 4c0594b..73ad9f8 100644 --- a/src/lib/server/backends/files.ts +++ b/src/lib/server/backends/files.ts @@ -16,28 +16,65 @@ function remote(path: string) { } return null; } -function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); } +function missing(path: string): never { + throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); +} -export interface FileFingerprint { size: number; sha256: string } +export interface FileFingerprint { + size: number; + sha256: string; +} /** One scoped agent scan; null preserves compatibility with older agents and local files. */ -export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise | null> { +export async function nativeFileManifest( + root: string, + directories: string[], + ignoreNames: string[], + managedPaths: string[] +): Promise | null> { const target = remote(root); - if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null; - const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))]; + if ( + !target || + target.path !== '.' || + !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest') + ) + return null; + const roots = [ + ...new Set(directories.map((directory) => relative(root, directory).split(sep).join('/'))) + ]; // Preserve legacy configurations that scan the entire instance root. - if (roots.some(directory => !directory)) return null; - const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))]; - const result = await nativeRequest<{ files: Array }>(target.instance, 'files.manifest', { - roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name))) - }); - if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.'); + if (roots.some((directory) => !directory)) return null; + const paths = [ + ...new Set(managedPaths.map((path) => relative(root, path).split(sep).join('/'))) + ]; + const result = await nativeRequest<{ files: Array }>( + target.instance, + 'files.manifest', + { + roots, + paths, + ignoreNames: ignoreNames.filter( + (name) => !paths.some((path) => path.split('/').includes(name)) + ) + } + ); + if (!result || !Array.isArray(result.files) || result.files.length > 10_000) + throw new Error('Agent returned an invalid file manifest.'); const manifest = new Map(); for (const entry of result.files) { - if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0') - || posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.') - || !roots.some(directory => entry.path.startsWith(`${directory}/`)) - || !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) { + if ( + !entry || + typeof entry.path !== 'string' || + !entry.path || + entry.path.includes('\\') || + entry.path.includes('\0') || + posix.normalize(entry.path) !== entry.path || + entry.path.split('/').some((part) => part === '..' || part === '.') || + !roots.some((directory) => entry.path.startsWith(`${directory}/`)) || + !Number.isSafeInteger(entry.size) || + entry.size < 0 || + !/^[a-f0-9]{64}$/.test(entry.sha256) + ) { throw new Error('Agent returned an invalid file manifest entry.'); } const path = join(root, entry.path); @@ -50,44 +87,85 @@ export async function nativeFileManifest(root: string, directories: string[], ig export async function readFile(path: string): Promise { const target = remote(path); if (!target) return fs.readFile(path); - const result = await nativeRequest<{ contentSha256?: string; contentBase64?: string | null } | null>(target.instance, 'files.read', { path: target.path }); + const result = await nativeRequest<{ + contentSha256?: string; + contentBase64?: string | null; + } | null>(target.instance, 'files.read', { path: target.path }); // Stored receipts reference shared bytes; read them directly without a base64 round trip. if (typeof result?.contentSha256 === 'string') return getFileBlob(result.contentSha256); if (result?.contentBase64 === null) return missing(path); - if (typeof result?.contentBase64 === 'string') return Buffer.from(result.contentBase64, 'base64'); + if (typeof result?.contentBase64 === 'string') + return Buffer.from(result.contentBase64, 'base64'); throw new Error(`Agent returned an invalid response for files.read: ${target.path}`); } export async function exists(path: string): Promise { - try { await stat(path); return true; } - catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; } + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } } -export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> { +export async function stat( + path: string +): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> { const target = remote(path); if (!target) return fs.stat(path); - const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path }); + const result = await nativeRequest<{ + exists?: boolean; + size: number; + mtime: string | number; + isDirectory: boolean; + isFile: boolean; + } | null>(target.instance, 'files.stat', { path: target.path }); if (result === null || result.exists === false) return missing(path); - return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile }; + return { + size: result.size, + mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), + isDirectory: () => result.isDirectory, + isFile: () => result.isFile + }; } -export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> { +export async function readdir( + path: string, + options: { withFileTypes: true } +): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> { const target = remote(path); if (!target) return fs.readdir(path, options); - const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path }); - return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory })); + const result = await nativeRequest<{ + entries: { name: string; isDirectory: boolean; isFile?: boolean }[]; + }>(target.instance, 'files.list', { path: target.path }); + return result.entries.map((entry) => ({ + name: entry.name, + isDirectory: () => entry.isDirectory, + isFile: () => entry.isFile ?? !entry.isDirectory + })); } export async function nativeTail(path: string, lines: number) { const target = remote(path); if (!target) return undefined; - return nativeRequest(target.instance, 'logs.read', { path: target.path, lines }); + return nativeRequest( + target.instance, + 'logs.read', + { path: target.path, lines } + ); } export async function nativeLogChunk(path: string, cursor?: import('../fleet/logs.js').LogCursor) { const target = remote(path); if (!target) return undefined; - const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes('logs.read-offset'); - return nativeRequest(target.instance, 'logs.read', - { path: target.path, ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) }); + const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes( + 'logs.read-offset' + ); + return nativeRequest< + import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null + >(target.instance, 'logs.read', { + path: target.path, + ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) + }); } diff --git a/src/lib/server/backends/native.ts b/src/lib/server/backends/native.ts index 4101d5a..5e32445 100644 --- a/src/lib/server/backends/native.ts +++ b/src/lib/server/backends/native.ts @@ -6,13 +6,26 @@ import { trackActivity } from '../pipeline/activity.js'; import { latestGameSubmission } from '../fleet/lifecycle-evidence.js'; export interface NativeInstance { - id: string; server: string; host_id: string; generation: number; library_id: string; - image: string; bind_address: string; port: number; public_address: string | null; - public_port: number | null; created_at: string; retired_at: string | null; + id: string; + server: string; + host_id: string; + generation: number; + library_id: string; + image: string; + bind_address: string; + port: number; + public_address: string | null; + public_port: number | null; + created_at: string; + retired_at: string | null; } export function nativeInstance(server: string): NativeInstance | null { - return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null; + return ( + (db() + .prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL') + .get(server) as NativeInstance | undefined) ?? null + ); } /** Credentials come from the launch, never from the possibly modified config or @@ -21,72 +34,136 @@ export function nativeInstance(server: string): NativeInstance | null { export function runtimePasswordForServer(server: string): string | null { const instance = nativeInstance(server); if (!instance) return null; - const row = db().prepare(`SELECT + const row = db() + .prepare( + `SELECT json_extract(COALESCE(p.payload,o.payload),'$.rconPassword') AS password, json_extract(COALESCE(p.payload,o.payload),'$.env.PLUTAINER_RCON_PASSWORD') AS env_password FROM ec_host_operations o JOIN native_instances i ON ${runtimeLaunchSql} LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL - ORDER BY o.rowid DESC LIMIT 1`).get(instance.id) as { password: unknown; env_password: unknown } | undefined; - if (!row) throw new Error(`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`); + ORDER BY o.rowid DESC LIMIT 1` + ) + .get(instance.id) as { password: unknown; env_password: unknown } | undefined; + if (!row) + throw new Error( + `Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.` + ); const password = row.password || row.env_password; return typeof password === 'string' && password ? password : null; } export function nativeDeployment(id: string): boolean { - return !!db().prepare(`SELECT 1 FROM native_provisioning WHERE deployment_id = ? + return !!db() + .prepare( + `SELECT 1 FROM native_provisioning WHERE deployment_id = ? UNION ALL SELECT 1 FROM native_creation_intents WHERE deployment_id = ? UNION ALL SELECT 1 FROM native_creation_cancellations c - JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`).get(id, id, id); + JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1` + ) + .get(id, id, id); } export function unplacedNativeCreation(server: string): string | null { - const row = db().prepare(`SELECT d.id FROM server_deployments d + const row = db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN native_creation_intents i ON i.deployment_id=d.id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state='queued' - AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(server) as { id: string } | undefined; + AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1` + ) + .get(server) as { id: string } | undefined; return row?.id ?? null; } const observations = new Map>(); -const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']); +const readKinds = new Set([ + 'instance.inspect', + 'files.read', + 'files.stat', + 'files.list', + 'files.manifest', + 'logs.read' +]); /** Share overlapping observations only, never completed results or mutations. */ -export async function nativeRequest>(instance: NativeInstance, kind: string, payload: Record = {}, key?: string): Promise { +export async function nativeRequest>( + instance: NativeInstance, + kind: string, + payload: Record = {}, + key?: string +): Promise { const host = getHost(instance.host_id); - if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.'); - const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command))); + if (!host || host.connectionState !== 'online') + throw new Error('Host agent is offline; instance state is unknown.'); + const readOnly = + readKinds.has(kind) || + (kind === 'rcon.execute' && + ['status', 'getstatus', 'version'].includes(String(payload.command))); if (key || !readOnly) return executeRequest(instance, kind, payload, key); // A probe after a lifecycle submission must not join an old-process read. - const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server, - latestGameSubmission(instance.server), kind, payload]); + const identity = JSON.stringify([ + instance.host_id, + instance.id, + instance.generation, + instance.server, + latestGameSubmission(instance.server), + kind, + payload + ]); const pending = observations.get(identity); if (pending) return pending as Promise; - const request = executeRequest(instance, kind, payload).finally(() => { observations.delete(identity); }); + const request = executeRequest(instance, kind, payload).finally(() => { + observations.delete(identity); + }); observations.set(identity, request); return request; } /** Only the agent receives host paths, Docker access, or game credentials. */ -async function executeRequest(instance: NativeInstance, kind: string, payload: Record, key?: string): Promise { +async function executeRequest( + instance: NativeInstance, + kind: string, + payload: Record, + key?: string +): Promise { const operation = queueHostOperation(instance.host_id, kind, payload, { - instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID() + instanceId: instance.id, + generation: instance.generation, + idempotencyKey: key ?? randomUUID() }); - const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`, - () => waitHostOperationStatus(operation.id, 25_000)); - if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`); + const result = await trackActivity( + `agent:${operation.id}`, + `Waiting for ${kind} on ${instance.server}`, + () => waitHostOperationStatus(operation.id, 25_000) + ); + if (result.state !== 'completed') + throw new Error( + result.error || + `Agent operation ${result.id} is ${result.state}; its original ID is retained.` + ); // Do not apply replies from a retired generation to a replacement server. const current = nativeInstance(instance.server); - if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.'); + if (current?.id !== instance.id || current.generation !== instance.generation) + throw new Error('Instance was replaced during the operation.'); return result.result as T; } -export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> { +export async function nativeHealth( + server: string +): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> { const instance = nativeInstance(server); if (!instance) return { ok: false, reason: 'Native instance is not registered' }; try { - const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect'); - return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` }; - } catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; } + const result = await nativeRequest<{ state: string; healthy: boolean }>( + instance, + 'instance.inspect' + ); + return result.healthy + ? { ok: true, state: result.state } + : { ok: false, state: result.state, reason: `Container ${result.state}` }; + } catch (error) { + return { ok: false, state: 'unknown', reason: (error as Error).message }; + } } diff --git a/src/lib/server/backends/schema.ts b/src/lib/server/backends/schema.ts index 3366f1c..fa9e0bc 100644 --- a/src/lib/server/backends/schema.ts +++ b/src/lib/server/backends/schema.ts @@ -43,11 +43,21 @@ export function migrateNativeInstances(handle: Database.Database): void { ); CREATE INDEX IF NOT EXISTS idx_ec_host_migrations_instance ON ec_host_migrations(instance_id); `); - if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) { + if ( + !(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some( + (column) => column.name === 'runtime_password' + ) + ) { handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT'); } - if (!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(column => column.name === 'recovery_step')) { - handle.exec("ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))"); + if ( + !(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some( + (column) => column.name === 'recovery_step' + ) + ) { + handle.exec( + "ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))" + ); } if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) { handle.transaction(() => { @@ -56,29 +66,43 @@ export function migrateNativeInstances(handle: Database.Database): void { INSERT INTO settings(key,value) VALUES('native_file_plans_v1','1');`); })(); } - } /** Accepted native creations receive their host port only when provisioning places them. */ function migrateUnplacedCreationPort(handle: Database.Database): void { - const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { name: string; notnull: number }[]; - if (!columns.find(column => column.name === 'port')?.notnull) return; - const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'").get() as { sql: string }; + const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { + name: string; + notnull: number; + }[]; + if (!columns.find((column) => column.name === 'port')?.notnull) return; + const { sql } = handle + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'") + .get() as { sql: string }; const replacement = sql.replace(/\bport\s+INTEGER\s+NOT\s+NULL\b/i, 'port INTEGER'); if (replacement === sql) throw new Error('Could not migrate unplaced creation ports.'); - const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL") + const dependent = handle + .prepare( + "SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL" + ) .all() as { sql: string }[]; const foreignKeys = handle.pragma('foreign_keys', { simple: true }); handle.pragma('foreign_keys = OFF'); try { handle.transaction(() => { - handle.exec(replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced')); - const names = columns.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', '); + handle.exec( + replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced') + ); + const names = columns + .map((column) => `"${column.name.replaceAll('"', '""')}"`) + .join(', '); handle.exec(`INSERT INTO server_creation_jobs_unplaced(rowid,${names}) SELECT rowid,${names} FROM server_creation_jobs; DROP TABLE server_creation_jobs; ALTER TABLE server_creation_jobs_unplaced RENAME TO server_creation_jobs;`); for (const object of dependent) handle.exec(object.sql); - if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Unplaced creation migration found broken references.'); + if ((handle.pragma('foreign_key_check') as unknown[]).length) + throw new Error('Unplaced creation migration found broken references.'); })(); - } finally { handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); } + } finally { + handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); + } } diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index 3ceae46..9b77df7 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -17,13 +17,15 @@ function num(name: string, fallback: number): number { export const config = { serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'), - t6ServersRoot: env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'), + t6ServersRoot: + env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'), t6RelDestScript: env.T6_REL_DEST_SCRIPT ?? 'runtime/plutonium/storage/t6/scripts/zm', t6ConfigTargetFile: env.T6_CONFIG_TARGET_FILE ?? 'dedicated_zm.cfg', relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'), // Per-kind destination sub-paths under each server's directory. `relDest` // above stays as the script fallback so existing deploys keep working. - relDestScript: env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp', + relDestScript: + env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp', relDestConfig: env.REL_DEST_CONFIG ?? 'configs', relDestPlugin: env.REL_DEST_PLUGIN ?? 'runtime/plutonium/plugins', // Legacy T5 config filename. T6 has its own target above. @@ -54,19 +56,25 @@ export type ArtifactKind = 'script' | 'config' | 'plugin' | 'baseline'; export function relDestForKind(kind: ArtifactKind, game = 't5'): string { switch (kind) { - case 'script': return game === 't6' ? config.t6RelDestScript : config.relDestScript; - case 'config': return config.relDestConfig; - case 'baseline': return config.relDestConfig; - case 'plugin': return config.relDestPlugin; + case 'script': + return game === 't6' ? config.t6RelDestScript : config.relDestScript; + case 'config': + return config.relDestConfig; + case 'baseline': + return config.relDestConfig; + case 'plugin': + return config.relDestPlugin; } } // Server IDs keep the game prefix even while creation is queued and no registry // row exists yet. Legacy IDs retain the original T5 filesystem defaults. -export const gameOfServer = (server: string) => server.startsWith('t6_') ? 't6' : 't5'; -export const rootOfServer = (server: string) => gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot; +export const gameOfServer = (server: string) => (server.startsWith('t6_') ? 't6' : 't5'); +export const rootOfServer = (server: string) => + gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot; export const serverDir = (server: string) => `${rootOfServer(server)}/${server}`; -export const configFileForGame = (game: string | null) => game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile; +export const configFileForGame = (game: string | null) => + game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile; export const configFileOf = (server: string) => configFileForGame(gameOfServer(server)); export const destOfKind = (server: string, kind: ArtifactKind) => `${serverDir(server)}/${relDestForKind(kind, gameOfServer(server))}`; diff --git a/src/lib/server/db/connection.ts b/src/lib/server/db/connection.ts index f7da6fe..fcb64b9 100644 --- a/src/lib/server/db/connection.ts +++ b/src/lib/server/db/connection.ts @@ -26,8 +26,10 @@ export function migrationStep(name: string, work: () => T): T { console.info(`[database] migration starting: ${name}`); try { const result = work(); - console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started, - rssMiB: Math.round(process.memoryUsage().rss / 1048576) }); + console.info(`[database] migration finished: ${name}`, { + elapsedMs: Date.now() - started, + rssMiB: Math.round(process.memoryUsage().rss / 1048576) + }); return result; } catch (error) { console.error(`[database] migration failed: ${name}`, error); diff --git a/src/lib/server/db/file-blobs.ts b/src/lib/server/db/file-blobs.ts index afe1f8a..03171db 100644 --- a/src/lib/server/db/file-blobs.ts +++ b/src/lib/server/db/file-blobs.ts @@ -10,67 +10,103 @@ export function ensureFileBlobSchema(handle: Database.Database): void { CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`); } -export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null { +export function putFileBlob( + content: Uint8Array | null, + handle: Database.Database = db() +): string | null { if (content === null) return null; const sha256 = createHash('sha256').update(content).digest('hex'); - handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content); + handle + .prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)') + .run(sha256, content.byteLength, content); return sha256; } export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer { - const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined; + const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as + { content: Buffer } | undefined; if (!row) throw new Error('File content is unavailable or has expired.'); return row.content; } /** SQL fragments are for fixed schema identifiers, never user input. */ -export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`; +export const blobContent = (column: string) => + `(SELECT content FROM file_blobs WHERE sha256=${column})`; export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`; -export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable, handle: Database.Database = db()): void { - handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId); - const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)'); +export function referenceBlobs( + ownerType: string, + ownerId: string, + hashes: Iterable, + handle: Database.Database = db() +): void { + handle + .prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?') + .run(ownerType, ownerId); + const insert = handle.prepare( + 'INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)' + ); for (const hash of hashes) insert.run(ownerType, ownerId, hash); } /** Immutable file bytes in JSON are replaced with hash references, including before images. */ -export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set()): { value: unknown; hashes: Set } { - if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes }; +export function packFileJson( + value: unknown, + handle: Database.Database = db(), + hashes = new Set() +): { value: unknown; hashes: Set } { + if (Array.isArray(value)) + return { value: value.map((item) => packFileJson(item, handle, hashes).value), hashes }; if (!value || typeof value !== 'object') return { value, hashes }; const result: Record = {}; for (const [key, item] of Object.entries(value)) { - if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') { + if ( + (key === 'contentBase64' || + (key === 'content' && 'path' in value && 'sha256' in value)) && + typeof item === 'string' + ) { const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!; result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash; hashes.add(hash); } else { result[key] = packFileJson(item, handle, hashes).value; - if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item); + if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') + hashes.add(item); } } return { value: result, hashes }; } +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Unpacked JSON retains the caller-specific stored object shape. export function unpackFileJson(value: unknown, handle: Database.Database = db()): any { - if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle)); + if (Array.isArray(value)) return value.map((item) => unpackFileJson(item, handle)); if (!value || typeof value !== 'object') return value; const result: Record = {}; for (const [key, item] of Object.entries(value)) { if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') { - result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64'); + result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob( + item, + handle + ).toString('base64'); } else result[key] = unpackFileJson(item, handle); } return result; } export const blobColumns = [ - ['script_versions', 'content'], ['staged_file_changes', 'content'], - ['deployment_files', 'content_before'], ['deployment_files', 'content_after'], - ['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content'] + ['script_versions', 'content'], + ['staged_file_changes', 'content'], + ['deployment_files', 'content_before'], + ['deployment_files', 'content_after'], + ['beta_file_snapshots', 'prior_content'], + ['beta_revision_files', 'content'] ] as const; /** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */ -export function* migrationRows(handle: Database.Database, sql: string): Generator { +export function* migrationRows( + handle: Database.Database, + sql: string +): Generator { const select = handle.prepare(sql); let cursor = -Number.MAX_SAFE_INTEGER; for (;;) { @@ -84,31 +120,53 @@ export function* migrationRows(handle: Database.Database, sql: string): Gener /** Backfill one row at a time so large libraries do not exhaust startup memory. */ export function migrateFileBlobs(handle: Database.Database): void { for (const [table, column] of blobColumns) { - if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue; - migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => { - handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`); - const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`); - for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) { - update.run(putFileBlob(row.content, handle), row.rid); - } - handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); - })()); + if ( + !(handle.pragma(`table_info(${table})`) as { name: string }[]).some( + (item) => item.name === column + ) + ) + continue; + migrationStep(`shared files: ${table}.${column}`, () => + handle.transaction(() => { + handle.exec( + `ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)` + ); + const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`); + for (const row of migrationRows<{ content: Buffer | null }>( + handle, + `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1` + )) { + update.run(putFileBlob(row.content, handle), row.rid); + } + handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); + })() + ); } - for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`); + for (const [table, column] of blobColumns) + handle.exec( + `CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)` + ); if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) { - migrationStep('shared files: launch recipes', () => handle.transaction(() => { - for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle, - 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) { - const packed = packFileJson(JSON.parse(row.recipe), handle); - handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id); - referenceBlobs('recipe', row.deployment_id, packed.hashes, handle); - } - handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')"); - })()); + migrationStep('shared files: launch recipes', () => + handle.transaction(() => { + for (const row of migrationRows<{ deployment_id: string; recipe: string }>( + handle, + 'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1' + )) { + const packed = packFileJson(JSON.parse(row.recipe), handle); + handle + .prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?') + .run(JSON.stringify(packed.value), row.deployment_id); + referenceBlobs('recipe', row.deployment_id, packed.hashes, handle); + } + handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')"); + })() + ); } handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`); - for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'} + for (const action of ['INSERT', 'UPDATE OF recipe']) + handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'} AFTER ${action} ON server_launch_recipes BEGIN DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id; INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) @@ -117,8 +175,12 @@ export function migrateFileBlobs(handle: Database.Database): void { } export function pruneFileBlobs(limit = 100): number { - return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b + return db() + .prepare( + `DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256) ${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')} - LIMIT ?)`).run(limit).changes; + LIMIT ?)` + ) + .run(limit).changes; } diff --git a/src/lib/server/db/fleet-settings.ts b/src/lib/server/db/fleet-settings.ts index 291c0a6..53b8c8f 100644 --- a/src/lib/server/db/fleet-settings.ts +++ b/src/lib/server/db/fleet-settings.ts @@ -1,26 +1,36 @@ import type Database from 'better-sqlite3'; -import { defaultFleetSettings, validateFleetSettings, type FleetSettings } from '../../fleet-settings.js'; +import { + defaultFleetSettings, + validateFleetSettings, + type FleetSettings +} from '../../fleet-settings.js'; export const FLEET_SETTINGS_KEY = 'fleet_settings'; /** Import legacy environment values once; persisted settings win on every later boot. */ export function initializeFleetSettings(handle: Database.Database): void { - const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined; + const saved = handle + .prepare('SELECT value FROM settings WHERE key = ?') + .get(FLEET_SETTINGS_KEY) as { value: string } | undefined; if (saved) { const settings = JSON.parse(saved.value); const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings }); - handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY); + handle + .prepare('UPDATE settings SET value = ? WHERE key = ?') + .run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY); return; } const values = { ...defaultFleetSettings }; const legacy: Partial> = { - deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY', + deployKeepDays: 'DEPLOY_KEEP_DAYS', + restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY', createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY' }; for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) { const raw = process.env[legacy[key]!]; if (raw) values[key] = Number(raw); } - handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)') + handle + .prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)') .run(FLEET_SETTINGS_KEY, JSON.stringify(validateFleetSettings(values))); } diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index aaa1468..ff948a5 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -28,9 +28,15 @@ let optimizeTimer: NodeJS.Timeout | undefined; // IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't // backfill columns; this can. const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ - { name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' }, + { + name: 'container_checked_at', + ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' + }, { name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' }, - { name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" }, + { + name: 'desired_state', + ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" + }, // Base columns that shipped in the very first schema stay here for parity // with fresh installs — no ALTER is issued if they're already present. { name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' }, @@ -43,7 +49,10 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ { name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' }, { name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' }, { name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' }, - { name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' }, + { + name: 'rcon_failures', + ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' + }, { name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' }, // Script-scope columns — static per server, populated via server-edit UI. { name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' }, @@ -78,7 +87,10 @@ function migrateServers(handle: Database.Database): void { const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']); handle.transaction(() => { for (const column of rows) { - if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`); + if (retired.has(column.name)) + handle.exec( + `ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"` + ); } })(); } @@ -98,8 +110,14 @@ const SCRIPT_VERSIONS_COLUMNS: Array<{ name: string; ddl: string }> = [ // the authoritative "promoted" signal), and any version already gated // through a ready-promote beta will be re-marked next time that state // machine fires. - { name: 'state', ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'" }, - { name: 'kind', ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" } + { + name: 'state', + ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'" + }, + { + name: 'kind', + ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" + } ]; const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [ @@ -107,18 +125,40 @@ const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [ ]; const DEPLOYMENT_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [ - { name: 'kind', ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" } + { + name: 'kind', + ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" + } ]; const DEPLOYMENTS_COLUMNS: Array<{ name: string; ddl: string }> = [ - { name: 'script_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)' }, - { name: 'script_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)' }, - { name: 'prior_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)' }, - { name: 'beta_id', ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)' }, - { name: 'rolled_back_by', ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)' } + { + name: 'script_id', + ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)' + }, + { + name: 'script_version_id', + ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)' + }, + { + name: 'prior_version_id', + ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)' + }, + { + name: 'beta_id', + ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)' + }, + { + name: 'rolled_back_by', + ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)' + } ]; -function migrateTable(handle: Database.Database, table: string, cols: Array<{ name: string; ddl: string }>): void { +function migrateTable( + handle: Database.Database, + table: string, + cols: Array<{ name: string; ddl: string }> +): void { const rows = handle.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; const existing = new Set(rows.map((r) => r.name)); for (const col of cols) { @@ -169,14 +209,16 @@ function backfillMultiScriptJoins(handle: Database.Database): void { // operator can promote/rollback in the old app before upgrading. function dropLegacyBetaTables(handle: Database.Database): void { const legacy = handle - .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')") + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')" + ) .all() as Array<{ name: string }>; if (legacy.length === 0) return; const legacyNames = legacy.map((r) => r.name); // The old `betas` table had a `slug` column; the new one has `id` + `script_id`. // If we see `slug`, we know it's the old shape. - const betasCols = handle.prepare("PRAGMA table_info(betas)").all() as Array<{ name: string }>; + const betasCols = handle.prepare('PRAGMA table_info(betas)').all() as Array<{ name: string }>; const isLegacyShape = betasCols.some((c) => c.name === 'slug'); if (!isLegacyShape) return; @@ -205,8 +247,7 @@ function dropLegacyBetaTables(handle: Database.Database): void { // active betas aren't blocked. function assertSnapshotSchemeSafe(handle: Database.Database): void { const done = handle.prepare("SELECT value FROM settings WHERE key = 'snapshots_v1'").get() as - | { value: string } - | undefined; + { value: string } | undefined; if (done) return; const active = handle .prepare("SELECT id FROM betas WHERE state IN ('running','ready-promote') LIMIT 1") @@ -227,19 +268,36 @@ function migrateGameTelemetry(handle: Database.Database): void { handle.transaction(() => { const convert = (raw: string): string => { const plan = JSON.parse(raw) as BetaPlan; - plan.requirements = plan.requirements.map(r => { + plan.requirements = plan.requirements.map((r) => { const { event: _event, ...rest } = r as BetaRequirement & { event?: string }; return ['feature', 'restart-budget', 'clean-logs'].includes(r.kind) - ? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } : rest; + ? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } + : rest; }); return JSON.stringify(plan); }; - const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ beta_id: number; plan: string }>; - for (const row of plans) handle.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?').run(convert(row.plan), row.beta_id); - const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ name: string; plan: string }>; - for (const row of presets) handle.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?').run(convert(row.plan), row.name); - handle.prepare(`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL - AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`).run(JSON.stringify(emptyMetrics())); + const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ + beta_id: number; + plan: string; + }>; + for (const row of plans) + handle + .prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?') + .run(convert(row.plan), row.beta_id); + const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ + name: string; + plan: string; + }>; + for (const row of presets) + handle + .prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?') + .run(convert(row.plan), row.name); + handle + .prepare( + `UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL + AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))` + ) + .run(JSON.stringify(emptyMetrics())); handle.exec(`UPDATE beta_test_plans SET outcome = 'active', reason = 'Gameplay telemetry updated; collecting fresh script events' WHERE outcome IN ('active','passed') AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote')); UPDATE script_versions SET state = 'untested' WHERE state = 'tested' @@ -254,54 +312,96 @@ function migrateGameTelemetry(handle: Database.Database): void { } function migrateProvisioningActions(handle: Database.Database): void { - const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string }; + const table = handle + .prepare( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'" + ) + .get() as { sql: string }; if (table.sql.includes("'rebuild'")) return; // Rebuild the CHECK constraint without cascading into events, recipes, or // collection members. Preserve rowids: they order same-server operations. handle.pragma('foreign_keys = OFF'); try { handle.transaction(() => { - handle.exec(table.sql.replace('server_deployments', 'server_deployments_new') - .replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'")); - const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[]) - .map(column => `"${column.name.replaceAll('"', '""')}"`).join(', '); + handle.exec( + table.sql + .replace('server_deployments', 'server_deployments_new') + .replace( + "'create', 'destroy'", + "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'" + ) + ); + const columns = ( + handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[] + ) + .map((column) => `"${column.name.replaceAll('"', '""')}"`) + .join(', '); handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns}) SELECT rowid, ${columns} FROM server_deployments; DROP TABLE server_deployments; ALTER TABLE server_deployments_new RENAME TO server_deployments;`); - if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references'); + if ((handle.pragma('foreign_key_check') as unknown[]).length) + throw new Error('Provisioning migration found broken references'); })(); - } finally { handle.pragma('foreign_keys = ON'); } + } finally { + handle.pragma('foreign_keys = ON'); + } } /** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */ function migrateWatchdogConstraints(handle: Database.Database): void { for (const name of ['provisioning_sources', 'deployment_servers']) { - const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string }; - const replacement = name === 'provisioning_sources' - ? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'")) - : sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL'); + const { sql } = handle + .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(name) as { sql: string }; + const replacement = + name === 'provisioning_sources' + ? sql.includes("'autoscaling'") + ? sql + : sql.replace( + /'campaign','beta','maintenance'(?:,'watchdog')?/, + "'campaign','beta','maintenance','watchdog','autoscaling'" + ) + : sql.replace( + /server\s+TEXT NOT NULL REFERENCES servers\(name\)/, + 'server TEXT NOT NULL' + ); if (sql === replacement) continue; - const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL") + const dependent = handle + .prepare( + "SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL" + ) .all(name) as { sql: string }[]; handle.pragma('foreign_keys = OFF'); try { handle.transaction(() => { handle.exec(replacement.replace(name, name + '_watchdog_new')); - handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${ - (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ') - }) SELECT rowid, ${ - (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ') - } FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`); + handle.exec( + `INSERT INTO ${name}_watchdog_new (rowid, ${( + handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[] + ) + .map((col) => col.name) + .join(', ')}) SELECT rowid, ${( + handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[] + ) + .map((col) => col.name) + .join( + ', ' + )} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};` + ); for (const object of dependent) handle.exec(object.sql); - if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references'); + if ((handle.pragma('foreign_key_check') as unknown[]).length) + throw new Error('Watchdog migration found broken references'); })(); - } finally { handle.pragma('foreign_keys = ON'); } + } finally { + handle.pragma('foreign_keys = ON'); + } } } function backfillDesiredState(handle: Database.Database): void { - if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return; + if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) + return; handle.transaction(() => { // A failed stop may have reached the host agent. Preserve that explicit intent. handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = ( @@ -331,35 +431,80 @@ export function db(): Database.Database { migrateProvisioningActions(handle); migrateWatchdogConstraints(handle); migrateTable(handle, 'deployment_files', [ - { name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' } + { + name: 'snapshot_pending', + ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' + } + ]); + migrateTable(handle, 'restart_rollout', [ + { + name: 'operation_id', + ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' + } ]); - migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]); migrateTable(handle, 'server_deployments', [ - { name: 'requested_by', ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT' }, - { name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' }, - { name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' }, + { + name: 'requested_by', + ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT' + }, + { + name: 'submitted_at', + ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' + }, + { + name: 'checked_at', + ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' + }, { name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' }, { name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' }, - { name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' }, - { name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' }, - { name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' } + { + name: 'phase_started_at', + ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' + }, + { + name: 'observed_at', + ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' + }, + { + name: 'observation_json', + ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' + } ]); migrateTable(handle, 'server_deployment_events', [ { name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' }, - { name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' }, - { name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' } + { + name: 'phase_key', + ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' + }, + { + name: 'details_json', + ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' + } ]); for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) { - migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]); + migrateTable(handle, table, [ + { + name: 'provisioning_id', + ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` + } + ]); } migrateTable(handle, 'maintenance_jobs', [ - { name: 'kind', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'" }, - { name: 'origin', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'" } + { + name: 'kind', + ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'" + }, + { + name: 'origin', + ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'" + } ]); migrateServers(handle); // Old snippets may contain player addresses, including hex-encoded bytes. // Discard them once; newly retained diagnostics are sanitized failure replies only. - if (!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()) { + if ( + !handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get() + ) { handle.transaction(() => { handle.exec(`UPDATE servers SET last_reply_snippet=NULL; UPDATE servers SET last_error='bad-reply:unparseable print reply' @@ -374,25 +519,52 @@ export function db(): Database.Database { migrateTable(handle, 'scripts', SCRIPTS_COLUMNS); migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS); migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS); - migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]); + migrateTable(handle, 'beta_test_runs', [ + { + name: 'revision', + ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' + } + ]); migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS); migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS); // Older databases gain rolled_back_by above, after schema.sql runs. - handle.exec('CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)'); + handle.exec( + 'CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)' + ); migrateTable(handle, 'server_launch_collections', [ - { name: 'cancelled_at', ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT' }, - { name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, - { name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" } + { + name: 'cancelled_at', + ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT' + }, + { + name: 'mode', + ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" + }, + { + name: 'retained_servers', + ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" + } ]); migrateTable(handle, 'server_launch_recipes', [ - { name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' } + { + name: 'recipe_summary', + ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' + } ]); migrateServerHistory(handle); migrateTable(handle, 'server_collection_previews', [ - { name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, - { name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' } + { + name: 'mode', + ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" + }, + { + name: 'fleet_snapshot', + ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' + } + ]); + migrateTable(handle, 'server_telemetry', [ + { name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' } ]); - migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]); // Kind-referencing indexes: safe here because migrateTable above just // ALTER'd the column into existence if it was missing. handle.exec( @@ -403,14 +575,27 @@ export function db(): Database.Database { backfillMultiScriptJoins(handle); assertSnapshotSchemeSafe(handle); migrateGameTelemetry(handle); - migrateTable(handle, 'leaderboard_matches', [{ name: 'start_players', ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' }]); + migrateTable(handle, 'leaderboard_matches', [ + { + name: 'start_players', + ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' + } + ]); migrateTable(handle, 'leaderboard_match_players', [ - { name: 'first_observed_round', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER' }, - { name: 'present_at_start', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER' } + { + name: 'first_observed_round', + ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER' + }, + { + name: 'present_at_start', + ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER' + } ]); backfillLeaderboard(handle); initializeFleetSettings(handle); - migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle)); + migrationStep('native instance schema and file plans', () => + migrateNativeInstances(handle) + ); migrationStep('per-game server key pools', () => migrateServerKeys(handle)); migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle)); migrationStep('rebalancing schema', () => ensureRebalancingSchema(handle)); @@ -436,12 +621,18 @@ export function db(): Database.Database { ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`); installPerformanceIndexes(handle); installReportRevisions(handle); - migrationStep('retention clocks and indexes', () => retentionClocks(handle, { - deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'], - maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'], - leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'], - betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at'] - })); + migrationStep('retention clocks and indexes', () => + retentionClocks(handle, { + deployments: ['started_at', 'finished_at'], + server_deployments: ['started_at', 'finished_at'], + maintenance_jobs: ['requested_at', 'finished_at'], + watchdog_incidents: ['first_seen_at', 'finished_at'], + leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'], + betas: ['opened_at', 'closed_at'], + script_versions: ['uploaded_at'], + retention_controls: ['resolved_at'] + }) + ); // Existing databases require one rebuild to gain the incremental-vacuum page map. enableIncrementalVacuum(handle); handle.pragma('optimize = 0x10002'); @@ -472,8 +663,11 @@ export function db(): Database.Database { _db = handle; optimizeTimer = setInterval(() => { - try { handle.pragma('optimize'); } - catch (error) { console.error('[database] optimize failed:', error); } + try { + handle.pragma('optimize'); + } catch (error) { + console.error('[database] optimize failed:', error); + } }, 60 * 60_000); optimizeTimer.unref(); return _db; diff --git a/src/lib/server/db/retention-clocks.ts b/src/lib/server/db/retention-clocks.ts index 203c1d2..710dbbb 100644 --- a/src/lib/server/db/retention-clocks.ts +++ b/src/lib/server/db/retention-clocks.ts @@ -3,30 +3,59 @@ import { migrationStep, tuneMigrationConnection } from './connection.js'; /** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */ export function retentionClocks(handle: Database.Database, clocks: Record): void { - handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)'); + handle.exec( + 'CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)' + ); for (const [table, columns] of Object.entries(clocks)) { const marker = `iso-clocks:${table}:v1`; - if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) { + if ( + !handle + .prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?') + .get(marker) + ) { handle.transaction(() => { - for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column}) + for (const column of columns) + handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column}) WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`); - handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker); + handle + .prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)') + .run(marker); })(); } - const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[]; - const indexed = new Set(indexes.filter(index => !index.partial).map(index => - (handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name)); - for (const column of columns) if (!indexed.has(column)) { - handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`); - } + const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { + name: string; + partial: number; + }[]; + const indexed = new Set( + indexes + .filter((index) => !index.partial) + .map( + (index) => + ( + handle + .prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`) + .get() as { name: string } + ).name + ) + ); + for (const column of columns) + if (!indexed.has(column)) { + handle.exec( + `CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})` + ); + } } } /** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */ export function enableIncrementalVacuum(handle: Database.Database): void { if (handle.pragma('auto_vacuum', { simple: true }) !== 2) { - handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`); - if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) { + handle.exec( + `CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)` + ); + if ( + handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get() + ) { handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`); @@ -42,7 +71,9 @@ export function enableIncrementalVacuum(handle: Database.Database): void { }; try { tuneMigrationConnection(handle); - migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM')); + migrationStep('incremental vacuum page map (disk-backed rebuild)', () => + handle.exec('VACUUM') + ); } finally { handle.pragma('shrink_memory'); handle.pragma(`temp_store = ${previous.temp}`); @@ -51,9 +82,16 @@ export function enableIncrementalVacuum(handle: Database.Database): void { handle.pragma(`journal_size_limit = ${previous.journal}`); } } - if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) { + if ( + handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get() + ) { handle.transaction(() => { - if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(` + if ( + handle + .prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'") + .get() + ) + handle.exec(` UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE(( SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`); diff --git a/src/lib/server/db/server-history.ts b/src/lib/server/db/server-history.ts index 419d06e..929b046 100644 --- a/src/lib/server/db/server-history.ts +++ b/src/lib/server/db/server-history.ts @@ -1,17 +1,34 @@ import type Database from 'better-sqlite3'; -const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs']; +const historyTables = [ + 'beta_servers', + 'deployment_servers', + 'server_deployments', + 'leaderboard_matches', + 'watchdog_incidents', + 'maintenance_jobs' +]; const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')"; /** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */ function detachBetaHistory(handle: Database.Database): void { for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) { - const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string }; - const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL'); + const { sql } = handle + .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(name) as { sql: string }; + const replacement = sql.replace( + /server\s+TEXT NOT NULL REFERENCES servers\(name\)/, + 'server TEXT NOT NULL' + ); if (replacement === sql) continue; - const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL") + const dependents = handle + .prepare( + "SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL" + ) .all(name) as { sql: string }[]; - const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', '); + const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]) + .map((col) => col.name) + .join(', '); handle.pragma('foreign_keys = OFF'); try { handle.transaction(() => { @@ -19,9 +36,12 @@ function detachBetaHistory(handle: Database.Database): void { handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`); for (const dependent of dependents) handle.exec(dependent.sql); - if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references'); + if ((handle.pragma('foreign_key_check') as unknown[]).length) + throw new Error('Server history migration found broken references'); })(); - } finally { handle.pragma('foreign_keys = ON'); } + } finally { + handle.pragma('foreign_keys = ON'); + } } } @@ -40,12 +60,19 @@ export function migrateServerHistory(handle: Database.Database): void { detachBetaHistory(handle); handle.transaction(() => { for (const table of historyTables) { - const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[]; - if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`); + const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { + name: string; + }[]; + if (!columns.some((column) => column.name === 'server_deleted_at')) + handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`); } - const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[]; - if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) { - handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'"); + const collectionColumns = handle + .prepare('PRAGMA table_info(server_launch_collections)') + .all() as { name: string }[]; + if (!collectionColumns.some((column) => column.name === 'retained_server_deletions')) { + handle.exec( + "ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'" + ); } // Historical unfinished work cannot claim the next server with this name. handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending; @@ -96,15 +123,17 @@ export function migrateServerHistory(handle: Database.Database): void { OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')) AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`; for (const table of historyTables) { - const exception = table === 'server_deployments' - ? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS ( + const exception = + table === 'server_deployments' + ? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS ( SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning'))) AND id NOT IN (${liveRebuilds})` - : table === 'deployment_servers' - ? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))` - : table === 'watchdog_incidents' - ? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : ''; + : table === 'deployment_servers' + ? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))` + : table === 'watchdog_incidents' + ? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` + : ''; handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`); } diff --git a/src/lib/server/db/server-keys.ts b/src/lib/server/db/server-keys.ts index b8fde45..08415a7 100644 --- a/src/lib/server/db/server-keys.ts +++ b/src/lib/server/db/server-keys.ts @@ -22,79 +22,131 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_le /** Preserve every occupied legacy assignment, even duplicated credentials. */ export function migrateServerKeys(handle: Database.Database): void { - handle.transaction(() => { - const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[]) - .some(column => column.name === 'server_id'); - if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys'); - handle.exec(keySchema + leaseSchema); - if (!legacy) return; - const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as - { server_id: string; pluto_key: string; updated_at: string }[]; - const occupied = handle.prepare(`SELECT 1 WHERE + handle + .transaction(() => { + const legacy = ( + handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[] + ).some((column) => column.name === 'server_id'); + if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys'); + handle.exec(keySchema + leaseSchema); + if (!legacy) return; + const rows = handle + .prepare( + `SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id` + ) + .all() as { server_id: string; pluto_key: string; updated_at: string }[]; + const occupied = handle.prepare(`SELECT 1 WHERE EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL) OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL) OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`); - const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?'); - const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')"); - const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)'); - const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)'); - const assignments = new Map(); - function save(game: string, credential: string, label: string, updatedAt: string): string { - const existing = (find.get(game, credential) as { id: string } | undefined)?.id; - if (existing) return existing; - const id = randomUUID(); - insert.run(id, game, label, credential, updatedAt); - return id; - } - for (const row of rows) { - const game = /^(t5|t6)_/.exec(row.server_id)?.[1] - ?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game; - if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`); - const id = save(game, row.pluto_key, row.server_id, row.updated_at); - if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]); - } - // The old form could overwrite a saved key without changing the running - // container. Durable create/rebuild payloads are stronger evidence of - // its actual credential. Keep ambiguous delivered attempts as additional - // holds until the instance is destroyed; never free a possibly live key. - const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name); - if (hasTable('ec_host_operations')) { - const payload = hasTable('ec_host_operation_payloads') - ? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload'; - const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`; - const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at, + const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?'); + const serverGame = handle.prepare( + "SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')" + ); + const insert = handle.prepare( + 'INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)' + ); + const lease = handle.prepare( + 'INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)' + ); + const assignments = new Map(); + function save( + game: string, + credential: string, + label: string, + updatedAt: string + ): string { + const existing = (find.get(game, credential) as { id: string } | undefined)?.id; + if (existing) return existing; + const id = randomUUID(); + insert.run(id, game, label, credential, updatedAt); + return id; + } + for (const row of rows) { + const game = + /^(t5|t6)_/.exec(row.server_id)?.[1] ?? + (serverGame.get(row.server_id) as { game: string } | undefined)?.game; + if (!game) + throw new Error( + `Cannot determine the game for saved server key ${row.server_id}` + ); + const id = save(game, row.pluto_key, row.server_id, row.updated_at); + if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]); + } + // The old form could overwrite a saved key without changing the running + // container. Durable create/rebuild payloads are stronger evidence of + // its actual credential. Keep ambiguous delivered attempts as additional + // holds until the instance is destroyed; never free a possibly live key. + const hasTable = (name: string) => + !!handle + .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?") + .get(name); + if (hasTable('ec_host_operations')) { + const payload = hasTable('ec_host_operation_payloads') + ? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' + : 'o.payload'; + const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`; + const launched = handle + .prepare( + `SELECT n.server,o.state,o.updated_at, json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential, json_extract(${command},'$.game') AS game, json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed' AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation) - ORDER BY o.rowid DESC`).all() as - { server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[]; - const recovered = new Map(); - const complete = new Set(); - for (const row of launched) { - if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue; - const game = row.game === 't5' || row.game === 't6' ? row.game - : row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6' - : /^(t5|t6)_/.exec(row.server)?.[1] - ?? (serverGame.get(row.server) as { game: string } | undefined)?.game; - if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`); - const ids = recovered.get(row.server) ?? []; - const id = save(game, row.credential, row.server, row.updated_at); - if (!ids.includes(id)) ids.push(id); - recovered.set(row.server, ids); - if (row.state === 'completed') complete.add(row.server); + ORDER BY o.rowid DESC` + ) + .all() as { + server: string; + state: string; + updated_at: string; + credential: unknown; + game: unknown; + game_id: unknown; + }[]; + const recovered = new Map(); + const complete = new Set(); + for (const row of launched) { + if ( + complete.has(row.server) || + typeof row.credential !== 'string' || + !row.credential + ) + continue; + const game = + row.game === 't5' || row.game === 't6' + ? row.game + : row.game_id === 't5sp' + ? 't5' + : row.game_id === 't6zm' + ? 't6' + : (/^(t5|t6)_/.exec(row.server)?.[1] ?? + (serverGame.get(row.server) as { game: string } | undefined) + ?.game); + if (!game) + throw new Error( + `Cannot determine the game for launched server ${row.server}` + ); + const ids = recovered.get(row.server) ?? []; + const id = save(game, row.credential, row.server, row.updated_at); + if (!ids.includes(id)) ids.push(id); + recovered.set(row.server, ids); + if (row.state === 'completed') complete.add(row.server); + } + for (const [server, ids] of recovered) { + // If no completed launch receipt survives, its saved assignment + // remains a possible prior credential while the attempt resolves. + if (!complete.has(server)) + for (const id of assignments.get(server) ?? []) + if (!ids.includes(id)) ids.push(id); + assignments.set(server, ids); + } } - for (const [server, ids] of recovered) { - // If no completed launch receipt survives, its saved assignment - // remains a possible prior credential while the attempt resolves. - if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id); - assignments.set(server, ids); - } - } - for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0)); - handle.exec('DROP TABLE legacy_server_keys'); - }).immediate(); + for (const [server, ids] of assignments) + ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0)); + handle.exec('DROP TABLE legacy_server_keys'); + }) + .immediate(); } diff --git a/src/lib/server/db/slow-statements.ts b/src/lib/server/db/slow-statements.ts index f78c91f..7191561 100644 --- a/src/lib/server/db/slow-statements.ts +++ b/src/lib/server/db/slow-statements.ts @@ -1,19 +1,31 @@ import type Database from 'better-sqlite3'; /** Log query shapes, never bound values or SQL string literals. Disabled by default. */ -export function timeStatement(statement: Database.Statement, sql: string, thresholdMs: number): Database.Statement { +export function timeStatement( + statement: Database.Statement, + sql: string, + thresholdMs: number +): Database.Statement { if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) return statement; - const shape = sql.replace(/'(?:''|[^'])*'/g, '?').replace(/\s+/g, ' ').trim().slice(0, 200); + const shape = sql + .replace(/'(?:''|[^'])*'/g, '?') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 200); for (const method of ['run', 'get', 'all'] as const) { const original = statement[method].bind(statement) as (...args: unknown[]) => unknown; - Object.defineProperty(statement, method, { value: (...args: unknown[]) => { - const started = performance.now(); - try { return original(...args); } - finally { - const ms = performance.now() - started; - if (ms >= thresholdMs) console.warn('[sql]', { ms: Math.round(ms), method, sql: shape }); + Object.defineProperty(statement, method, { + value: (...args: unknown[]) => { + const started = performance.now(); + try { + return original(...args); + } finally { + const ms = performance.now() - started; + if (ms >= thresholdMs) + console.warn('[sql]', { ms: Math.round(ms), method, sql: shape }); + } } - } }); + }); } return statement; } diff --git a/src/lib/server/db/version-layout.ts b/src/lib/server/db/version-layout.ts index 2a05161..0eaf01f 100644 --- a/src/lib/server/db/version-layout.ts +++ b/src/lib/server/db/version-layout.ts @@ -3,25 +3,43 @@ import type Database from 'better-sqlite3'; /** Keep metadata ahead of overflow pages, preserving IDs and all references. */ export function migrateVersionLayout(handle: Database.Database): void { const columns = handle.pragma('table_info(script_versions)') as { name: string }[]; - if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return; - const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string }; + if (!columns.some((column) => column.name === 'content') || columns.at(-1)?.name === 'content') + return; + const { sql } = handle + .prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'") + .get() as { sql: string }; const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i; if (!content.test(sql)) throw new Error('Unrecognized script version content layout'); - const replacement = sql.replace(content, '').replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(') + const replacement = sql + .replace(content, '') + .replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(') .replace('script_versions', 'script_versions_metadata'); - const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL") + const dependents = handle + .prepare( + "SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL" + ) .all() as { sql: string }[]; - const sequence = handle.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'").get() as { seq: number } | undefined; + const sequence = handle + .prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'") + .get() as { seq: number } | undefined; handle.pragma('foreign_keys = OFF'); try { handle.transaction(() => { handle.exec(replacement); - const names = columns.map(column => `"${column.name}"`).join(','); + const names = columns.map((column) => `"${column.name}"`).join(','); handle.exec(`INSERT INTO script_versions_metadata (${names}) SELECT ${names} FROM script_versions; DROP TABLE script_versions; ALTER TABLE script_versions_metadata RENAME TO script_versions;`); - if (sequence) handle.prepare("UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'").run(sequence.seq); + if (sequence) + handle + .prepare( + "UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'" + ) + .run(sequence.seq); for (const dependent of dependents) handle.exec(dependent.sql); - if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Version layout migration found broken references'); + if ((handle.pragma('foreign_key_check') as unknown[]).length) + throw new Error('Version layout migration found broken references'); })(); - } finally { handle.pragma('foreign_keys = ON'); } + } finally { + handle.pragma('foreign_keys = ON'); + } } diff --git a/src/lib/server/fleet/beta-evaluation.ts b/src/lib/server/fleet/beta-evaluation.ts index d0b6e35..4498fcd 100644 --- a/src/lib/server/fleet/beta-evaluation.ts +++ b/src/lib/server/fleet/beta-evaluation.ts @@ -19,14 +19,30 @@ export interface TestMetrics { outages: number; streaks: Record; } -export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} }); +export const emptyMetrics = (): TestMetrics => ({ + values: {}, + manual: {}, + outages: 0, + streaks: {} +}); /** Gameplay comes only from the script. No reporter IDs or beta handshake. */ -export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void { +export function applyGameEvent( + plan: BetaPlan, + metrics: TestMetrics, + event: GameEvent, + now: number +): void { const players = event.players.length; if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) { - metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined, - lastEvent: now, players, ended: false, rounds: {} }; + metrics.match = { + map: event.map, + started: event.event === 'game_start' ? now : undefined, + lastEvent: now, + players, + ended: false, + rounds: {} + }; metrics.streaks = {}; } const match = metrics.match; @@ -34,10 +50,12 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game const minutes = Math.max(0, now - match.lastEvent) / 60_000; for (const r of plan.requirements) { const current = metrics.values[r.id] ?? 0; - const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers); + const populated = + players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers); if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players); if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes; - if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60; + if (r.kind === 'player-hours' && populated) + metrics.values[r.id] = current + (minutes * Math.min(players, match.players)) / 60; if (r.kind === 'sustained-population') { metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0; metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]); @@ -46,10 +64,14 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round); if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round); } - if (event.event === 'game_end' && r.kind === 'matches' - && event.round >= MIN_RECORDED_ROUND - && (match.rounds[r.id] ?? -1) >= r.minRound - && (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) { + if ( + event.event === 'game_end' && + r.kind === 'matches' && + event.round >= MIN_RECORDED_ROUND && + (match.rounds[r.id] ?? -1) >= r.minRound && + (r.minMinutes === 0 || + (match.started !== undefined && now - match.started >= r.minMinutes * 60_000)) + ) { metrics.values[r.id] = current + 1; } } @@ -66,30 +88,63 @@ export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0; if (!healthy && m.wasHealthy === true) m.outages++; for (const r of plan.requirements) { - if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0; + if (r.kind === 'healthy-minutes') + m.values[r.id] = + healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0; } - m.lastSample = at; m.wasHealthy = healthy; + m.lastSample = at; + m.wasHealthy = healthy; } -export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] { - return plan.requirements.map(r => { +export function evaluateRequirements( + plan: BetaPlan, + metrics: TestMetrics, + phase: number +): RequirementProgress[] { + return plan.requirements.map((r) => { const budget = r.kind === 'failure-budget'; - const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id]) - : r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0; - return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target, + const current = budget + ? metrics.outages + : r.kind === 'manual' + ? Number(!!metrics.manual[r.id]) + : r.kind === 'recovery' + ? phase + : (metrics.values[r.id] ?? 0); + return { + id: r.id, + label: r.label, + current, + target: r.target, + passed: budget ? current <= r.target : current >= r.target, failed: budget && current > r.target, - detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off' - : budget ? `${current} / ${r.target} allowed; monitored until promotion` - : `${Math.round(current * 100) / 100} / ${r.target}` }; + detail: + r.kind === 'manual' + ? (metrics.manual[r.id] ?? 'Awaiting tester sign-off') + : budget + ? `${current} / ${r.target} allowed; monitored until promotion` + : `${Math.round(current * 100) / 100} / ${r.target}` + }; }); } -export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean { +export function coveragePassed( + plan: BetaPlan, + servers: Array<{ passed: boolean; game: string | null; map: string | null }> +): boolean { if (!servers.length) return false; - if (plan.scope === 'all') return servers.every(s => s.passed); - if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum; - const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map - : s.game && s.map ? `${s.game}/${s.map}` : null; + if (plan.scope === 'all') return servers.every((s) => s.passed); + if (plan.scope === 'quorum') return servers.filter((s) => s.passed).length >= plan.quorum; + const key = (s: (typeof servers)[number]) => + plan.scope === 'game' + ? s.game + : plan.scope === 'map' + ? s.map + : s.game && s.map + ? `${s.game}/${s.map}` + : null; const groups = new Set(servers.map(key)); - return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group)); + return ( + !groups.has(null) && + [...groups].every((group) => servers.some((s) => s.passed && key(s) === group)) + ); } diff --git a/src/lib/server/fleet/beta-pages.ts b/src/lib/server/fleet/beta-pages.ts index 7a3b87e..2665678 100644 --- a/src/lib/server/fleet/beta-pages.ts +++ b/src/lib/server/fleet/beta-pages.ts @@ -8,13 +8,21 @@ const positive = (value: string | null, fallback: number) => { }; export function betaListPage(params: URLSearchParams) { - const state = params.get('state'), view = params.get('view'), search = (params.get('q') ?? '').slice(0, 200).trim(); - const predicates: string[] = [], args: (string | number)[] = []; - if (state) { predicates.push('b.state=?'); args.push(state); } - else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')"); + const state = params.get('state'), + view = params.get('view'), + search = (params.get('q') ?? '').slice(0, 200).trim(); + const predicates: string[] = [], + args: (string | number)[] = []; + if (state) { + predicates.push('b.state=?'); + args.push(state); + } else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')"); else if (view === 'history') predicates.push("b.state IN ('promoted','rolled-back')"); const status = params.get('status'); - if (status && ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)) { + if ( + status && + ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status) + ) { // Match betaStatus: closed lifecycle wins over the final test outcome. predicates.push(`(CASE WHEN b.state IN ('promoted','rolled-back') THEN b.state WHEN EXISTS (SELECT 1 FROM beta_servers m WHERE m.beta_id=b.id AND (m.server_deleted_at IS NOT NULL OR m.stage='failed')) @@ -24,8 +32,15 @@ export function betaListPage(params: URLSearchParams) { args.push(status); } if (search) { - const names = (db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[]) - .filter(row => `${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())).map(row => row.server); + const names = ( + db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[] + ) + .filter((row) => + `${row.server} ${serverTitle(row.server)}` + .toLowerCase() + .includes(search.toLowerCase()) + ) + .map((row) => row.server); predicates.push(`(instr(lower('#'||b.id||' '||COALESCE(b.notes,'')),lower(?))>0 OR EXISTS ( SELECT 1 FROM beta_scripts bs JOIN scripts s ON s.id=bs.script_id WHERE bs.beta_id=b.id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0)) OR EXISTS (SELECT 1 FROM scripts s WHERE s.id=b.script_id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0)) @@ -33,21 +48,33 @@ export function betaListPage(params: URLSearchParams) { args.push(search, search, search, search, search, JSON.stringify(names)); } const where = predicates.length ? ' WHERE ' + predicates.join(' AND ') : ''; - const total = (db().prepare('SELECT COUNT(*) n FROM betas b' + where).get(...args) as { n: number }).n; + const total = ( + db() + .prepare('SELECT COUNT(*) n FROM betas b' + where) + .get(...args) as { n: number } + ).n; const pageSize = Math.min(100, positive(params.get('limit'), 25)); const totalPages = Math.max(1, Math.ceil(total / pageSize)); const page = Math.min(totalPages, positive(params.get('page'), 1)); - const betas = db().prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?') + const betas = db() + .prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?') .all(...args, pageSize, (page - 1) * pageSize) as BetaRow[]; - const counts = db().prepare(`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active, - COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`).get() as { active: number; history: number }; + const counts = db() + .prepare( + `SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active, + COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas` + ) + .get() as { active: number; history: number }; return { betas, page, pageSize, total, totalPages, counts }; } export function betaActivityPage(id: number, params = new URLSearchParams()) { const before = positive(params.get('before'), Number.MAX_SAFE_INTEGER); const limit = Math.min(100, positive(params.get('limit'), 50)); - const rows = db().prepare('SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id limit; const events = rows.slice(0, limit).reverse(); diff --git a/src/lib/server/fleet/beta-revisions.ts b/src/lib/server/fleet/beta-revisions.ts index 349eb95..a182c68 100644 --- a/src/lib/server/fleet/beta-revisions.ts +++ b/src/lib/server/fleet/beta-revisions.ts @@ -2,29 +2,67 @@ import { db } from '../db/index.js'; import type { BetaRevision } from '../../beta-revision.js'; export function currentBetaRevision(id: number): { revision: number; started_at: string } { - return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as - { revision: number; started_at: string } | undefined - ?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at }; + return ( + (db() + .prepare( + `SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1` + ) + .get(id) as { revision: number; started_at: string } | undefined) ?? { + revision: 1, + started_at: ( + db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { + opened_at: string; + } + ).opened_at + } + ); } /** Called in the revision transaction, before replacing the pinned bundle. */ -export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void { - db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)') +export function archiveBetaRevision( + id: number, + revision: number, + startedAt: string, + notes: string | null +): void { + db() + .prepare( + 'INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)' + ) .run(id, revision, startedAt, notes); - db().prepare(`INSERT OR IGNORE INTO beta_revision_files + db() + .prepare( + `INSERT OR IGNORE INTO beta_revision_files (beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash) SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id - WHERE bs.beta_id = ?`).run(revision, id); + WHERE bs.beta_id = ?` + ) + .run(revision, id); } export function betaRevisionHistory(id: number): BetaRevision[] { - const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC') - .all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>; - const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256 - FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array; - return rows.map(({ results_json, ...row }) => ({ ...row, + const rows = db() + .prepare( + 'SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC' + ) + .all(id) as Array<{ + revision: number; + started_at: string; + notes: string | null; + results_json: string | null; + }>; + const files = db() + .prepare( + `SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256 + FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id` + ) + .all(id) as Array; + return rows.map(({ results_json, ...row }) => ({ + ...row, results: results_json ? JSON.parse(results_json) : null, - files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file) + files: files + .filter((file) => file.revision === row.revision) + .map(({ revision, ...file }) => file) })); } diff --git a/src/lib/server/fleet/beta-selection.ts b/src/lib/server/fleet/beta-selection.ts index ff87a1b..7682c38 100644 --- a/src/lib/server/fleet/beta-selection.ts +++ b/src/lib/server/fleet/beta-selection.ts @@ -7,7 +7,7 @@ import { observationReader } from './status.js'; import type { ScriptRow } from './scripts.js'; const games = ['t5', 't6'] as const; -type Game = typeof games[number]; +type Game = (typeof games)[number]; type Counts = Partial>; export function parseBetaInstanceCounts(value: unknown): Counts { @@ -16,7 +16,12 @@ export function parseBetaInstanceCounts(value: unknown): Counts { } const counts: Counts = {}; for (const [game, count] of Object.entries(value)) { - if (!games.includes(game as Game) || !Number.isSafeInteger(count) || count < 1 || count > 1000) { + if ( + !games.includes(game as Game) || + !Number.isSafeInteger(count) || + count < 1 || + count > 1000 + ) { throw new Error('Instance counts must be integers from 1 to 1000 for t5 or t6'); } counts[game as Game] = count; @@ -25,79 +30,137 @@ export function parseBetaInstanceCounts(value: unknown): Counts { } function matches(script: ScriptRow, server: ServerRow): boolean { - return !script.scope_type || !script.scope_value - || (script.scope_type === 'game' ? server.game : server.map) === script.scope_value; + return ( + !script.scope_type || + !script.scope_value || + (script.scope_type === 'game' ? server.game : server.map) === script.scope_value + ); } /** Choose at submission time using the same fresh observations shown in Fleet. */ export function selectBetaServers(scripts: ScriptRow[], instancesPerGame: unknown): string[] { - const counts = parseBetaInstanceCounts(instancesPerGame), observe = observationReader(), now = Date.now(); - const hosts = new Map(listHosts().map(host => [host.id, host])); - const busy = new Set((db().prepare(`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL + const counts = parseBetaInstanceCounts(instancesPerGame), + observe = observationReader(), + now = Date.now(); + const hosts = new Map(listHosts().map((host) => [host.id, host])); + const busy = new Set( + ( + db() + .prepare( + `SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND (state IN ('queued','waiting','running','awaiting-health','interrupted') OR (action IN ('destroy','rebuild') AND state = 'failed' - AND stage IN ('Stopping and removing container','Removing server files and registry')))`) - .all() as { server: string }[]).map(row => row.server)); - const candidates = listServers().flatMap(server => { - if (!counts[server.game as Game] || server.desired_state === 'stopped' || busy.has(server.name) - || !scripts.some(script => matches(script, server)) || protectedWork(server.name)) return []; - const host = server.host_id ? hosts.get(server.host_id) : null; - if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) return []; - const observation = observe(server); - if (observation.health !== 'online' || observation.last_players === null - || !freshStatusTimestamp(observation.last_seen_at, now)) return []; - // An occupied RCON observation must not rank as empty just because a - // competing telemetry report claims zero players. - const rconPlayers = freshStatusTimestamp(server.last_seen_at, now) - && (!observation.telemetry_after || Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after)) - ? server.last_players ?? 0 : 0; - return [{ server, players: Math.max(observation.last_players, rconPlayers) }]; - }).sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name)); + AND stage IN ('Stopping and removing container','Removing server files and registry')))` + ) + .all() as { server: string }[] + ).map((row) => row.server) + ); + const candidates = listServers() + .flatMap((server) => { + if ( + !counts[server.game as Game] || + server.desired_state === 'stopped' || + busy.has(server.name) || + !scripts.some((script) => matches(script, server)) || + protectedWork(server.name) + ) + return []; + const host = server.host_id ? hosts.get(server.host_id) : null; + if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) + return []; + const observation = observe(server); + if ( + observation.health !== 'online' || + observation.last_players === null || + !freshStatusTimestamp(observation.last_seen_at, now) + ) + return []; + // An occupied RCON observation must not rank as empty just because a + // competing telemetry report claims zero players. + const rconPlayers = + freshStatusTimestamp(server.last_seen_at, now) && + (!observation.telemetry_after || + Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after)) + ? (server.last_players ?? 0) + : 0; + return [{ server, players: Math.max(observation.last_players, rconPlayers) }]; + }) + .sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name)); for (const game of games) { - const available = candidates.filter(candidate => candidate.server.game === game).length; + const available = candidates.filter((candidate) => candidate.server.game === game).length; if ((counts[game] ?? 0) > available) { - throw new Error(`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`); + throw new Error( + `${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.` + ); } } for (const script of scripts) { - if (!candidates.some(candidate => matches(script, candidate.server))) { - throw new Error(`${script.name}: no available server in the requested games matches this file's scope.`); + if (!candidates.some((candidate) => matches(script, candidate.server))) { + throw new Error( + `${script.name}: no available server in the requested games matches this file's scope.` + ); } } // Map scopes need a representative even when the least populated servers // all run the same map. Reserve enough remaining slots for every map. - const requiredMaps = new Set(scripts.filter(script => script.scope_type === 'map').map(script => script.scope_value!)); + const requiredMaps = new Set( + scripts.filter((script) => script.scope_type === 'map').map((script) => script.scope_value!) + ); function canComplete(remaining: typeof candidates, slots: Counts, maps: Set): boolean { const options = new Map>(); for (const game of games) { - if (remaining.filter(candidate => candidate.server.game === game).length < (slots[game] ?? 0)) return false; + if ( + remaining.filter((candidate) => candidate.server.game === game).length < + (slots[game] ?? 0) + ) + return false; } for (const map of maps) { - const choices = new Set(remaining.filter(candidate => candidate.server.map === map && (slots[candidate.server.game as Game] ?? 0) > 0) - .map(candidate => candidate.server.game as Game)); + const choices = new Set( + remaining + .filter( + (candidate) => + candidate.server.map === map && + (slots[candidate.server.game as Game] ?? 0) > 0 + ) + .map((candidate) => candidate.server.game as Game) + ); if (!choices.size) return false; options.set(map, choices); } // There are two supported games: single-game maps reserve that game's // slots, and maps available in either game use any remaining slot. - return games.every(game => [...options.values()].filter(choices => choices.size === 1 && choices.has(game)).length <= (slots[game] ?? 0)) - && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0); + return ( + games.every( + (game) => + [...options.values()].filter( + (choices) => choices.size === 1 && choices.has(game) + ).length <= (slots[game] ?? 0) + ) && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0) + ); } if (!canComplete(candidates, counts, requiredMaps)) { - throw new Error('Increase the instance counts to cover every selected file\'s map scope, or choose servers in advanced options.'); + throw new Error( + "Increase the instance counts to cover every selected file's map scope, or choose servers in advanced options." + ); } - const selected: string[] = [], slots = { ...counts }; + const selected: string[] = [], + slots = { ...counts }; let maps = requiredMaps; for (let index = 0; index < candidates.length; index++) { - const { server } = candidates[index], game = server.game as Game; + const { server } = candidates[index], + game = server.game as Game; if (!slots[game]) continue; - const nextSlots = { ...slots, [game]: slots[game]! - 1 }, nextMaps = new Set(maps); + const nextSlots = { ...slots, [game]: slots[game]! - 1 }, + nextMaps = new Set(maps); if (server.map) nextMaps.delete(server.map); if (!canComplete(candidates.slice(index + 1), nextSlots, nextMaps)) continue; - selected.push(server.name); slots[game] = nextSlots[game]; maps = nextMaps; - if (games.every(game => !slots[game])) break; + selected.push(server.name); + slots[game] = nextSlots[game]; + maps = nextMaps; + if (games.every((game) => !slots[game])) break; } return selected; } diff --git a/src/lib/server/fleet/beta-tests.ts b/src/lib/server/fleet/beta-tests.ts index 6b7b4c3..378afb1 100644 --- a/src/lib/server/fleet/beta-tests.ts +++ b/src/lib/server/fleet/beta-tests.ts @@ -4,52 +4,120 @@ import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js'; import type { GameTelemetry } from '../../telemetry.js'; -import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js'; +import { + applySample, + applyGameEvent, + emptyMetrics, + evaluateRequirements, + type TestMetrics +} from './beta-evaluation.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { currentBetaRevision } from './beta-revisions.js'; import { invalidateBeta } from '../pipeline/bus.js'; -export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string } -export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null } +export interface TestRun { + id: string; + beta_id: number; + server: string; + phase: number; + created_at: string; + observing_at: string | null; + metrics: string; +} +export interface StoredPlan { + beta_id: number; + plan: string; + outcome: string; + reason: string | null; +} const nowIso = () => new Date().toISOString(); -const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers - WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server); -export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined; -export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined; -export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); invalidateBeta(id); } +const activeParticipant = (id: number, server: string) => + !!db() + .prepare( + `SELECT 1 FROM beta_servers + WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL` + ) + .get(id, server); +export const storedPlan = (id: number) => + db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as + StoredPlan | undefined; +export const currentRun = (id: number, server: string) => + db() + .prepare( + 'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL' + ) + .get(id, server) as TestRun | undefined; +export function activity(id: number, message: string) { + db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); + invalidateBeta(id); +} export function setOutcome(id: number, outcome: string, reason: string | null): void { const previous = storedPlan(id); if (!previous || (previous.outcome === outcome && previous.reason === reason)) return; - db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id); + db() + .prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?') + .run(outcome, reason, id); activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`); } -export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; } -export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); invalidateBeta(run.beta_id); } +export function readMetrics(run: TestRun): TestMetrics { + return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; +} +export function saveMetrics(run: TestRun, metrics: TestMetrics): void { + db() + .prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?') + .run(JSON.stringify(metrics), run.id); + invalidateBeta(run.beta_id); +} export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean { - return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)') - .run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0; + return ( + db() + .prepare( + 'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)' + ) + .run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0 + ); } /** Internal progress record; never sent to or required from the game script. */ export async function prepareRun(betaId: number, server: string, phase = 0): Promise { - if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta'); + if (!activeParticipant(betaId, server)) + throw new Error('Original test server was deleted; start a new beta'); db().transaction(() => { - db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server); - db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)') - .run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision); + db() + .prepare( + 'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL' + ) + .run(nowIso(), betaId, server); + db() + .prepare( + 'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)' + ) + .run( + randomUUID(), + betaId, + server, + phase, + nowIso(), + JSON.stringify(emptyMetrics()), + currentBetaRevision(betaId).revision + ); })(); return currentRun(betaId, server)!; } /** Route ordinary server events to every deployed, active beta automatically. */ export function recordGameEvent(event: GameTelemetry): void { - const runs = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id + const runs = db() + .prepare( + `SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id JOIN beta_test_plans p ON p.beta_id = b.id JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote') AND s.server_deleted_at IS NULL AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready') - AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[]; + AND s.restarted_at IS NOT NULL` + ) + .all(event.server) as TestRun[]; for (const run of runs) { const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan)); const metrics = readMetrics(run); @@ -64,17 +132,27 @@ export async function observeRun(run: TestRun, plan: BetaPlan): Promise { const m = readMetrics(run); const row = getServer(run.server); const server = row ? serverObservation(row) : null; - const timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at; + const timestamp = + server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at; const sample = Date.parse(timestamp ?? ''); - const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at) - && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS; + const freshSample = + Number.isFinite(sample) && + sample >= Date.parse(run.observing_at ?? run.created_at) && + Date.now() - sample >= 0 && + Date.now() - sample <= ONLINE_MAX_MS; const healthy = freshSample && server?.health === 'online'; const at = freshSample && server?.container_state === 'running' ? sample : Date.now(); const before = m.wasHealthy; const lastSample = m.lastSample; applySample(plan, m, at, healthy); - if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source: server?.status_source }); - if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null }); + if (freshSample && sample !== lastSample) + evidence(run, `sample:${sample}`, 'status-sample', { + healthy, + checkedAt: timestamp, + source: server?.status_source + }); + if (before !== m.wasHealthy) + evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null }); saveMetrics(run, m); } @@ -82,22 +160,53 @@ export function testView(betaId: number): BetaTestView | null { const stored = storedPlan(betaId); if (!stored) return null; const plan = parseBetaPlan(JSON.parse(stored.plan)); - const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r + const runs = db() + .prepare( + `SELECT r.*, s.server_deleted_at FROM beta_test_runs r JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server - WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array; - return { plan, outcome: stored.outcome, reason: stored.reason, - servers: runs.map(run => { + WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server` + ) + .all(betaId) as Array; + return { + plan, + outcome: stored.outcome, + reason: stored.reason, + servers: runs.map((run) => { const metrics = readMetrics(run); const progress = evaluateRequirements(plan, metrics, run.phase); - const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined; + const latest = db() + .prepare( + "SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1" + ) + .get(run.id) as { received_at: string; kind: string; payload: string } | undefined; const event = latest ? JSON.parse(latest.payload) : null; - return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at, - observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null, - players: metrics.match?.players ?? null, round: event?.round ?? null, - matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false, - healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } }; + return { + server: run.server, + server_deleted_at: run.server_deleted_at, + phase: run.phase, + passed: progress.every((r) => r.passed), + progress, + observingAt: run.observing_at, + observation: { + lastGameplayAt: latest?.received_at ?? null, + lastEvent: latest?.kind ?? null, + players: metrics.match?.players ?? null, + round: event?.round ?? null, + matchStarted: metrics.match?.started !== undefined, + gameEnded: metrics.match?.ended ?? false, + healthy: metrics.wasHealthy ?? null, + healthCheckedAt: metrics.lastSample + ? new Date(metrics.lastSample).toISOString() + : null + } + }; }), - evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e + evidence: db() + .prepare( + `SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server - WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] }; + WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200` + ) + .all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] + }; } diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 8e62628..e55605e 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -9,9 +9,14 @@ import type { BetaPromotionPreview } from '../../beta-promotion.js'; const operations = new Set(); export const betaOperationInFlight = (id: number) => operations.has(id); async function exclusive(id: number, action: () => Promise): Promise { - if (operations.has(id)) throw new BetaError('A beta operation is already running', 'invalid-state'); + if (operations.has(id)) + throw new BetaError('A beta operation is already running', 'invalid-state'); operations.add(id); - try { return await action(); } finally { operations.delete(id); } + try { + return await action(); + } finally { + operations.delete(id); + } } import { mkdir, rm, writeFile } from 'node:fs/promises'; @@ -33,7 +38,6 @@ import { serversMatchingScope, setScope, uploadNewVersion, - type ScopeType, type ScriptRow, type ScriptVersionRow } from './scripts.js'; @@ -41,7 +45,12 @@ import { composeServerCfg } from './config-compose.js'; import { applyRuntimePassword } from './runtime-config.js'; import { runtimePasswordForServer } from '../backends/native.js'; import { removeScript } from './inventory.js'; -import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js'; +import { + cancelStagedFiles, + isFileApplied, + queueFileChange, + stagedFileChanges +} from './staged-files.js'; import { emit } from '../pipeline/bus.js'; import { cancelProvisioningJob } from '../provisioning/cancellation.js'; import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js'; @@ -54,11 +63,6 @@ function isCollapsingKind(kind: ArtifactKind): boolean { return kind === 'config' || kind === 'baseline'; } -// The path we should write to for the composed cfg on this server. -function composedCfgPath(server: string): string { - return join(destOfKind(server, 'config'), configFileOf(server)); -} - // Compose the effective dedicated_sp.cfg for a given beta+server, taking // into account: // • baseline: from the beta if present, else the currently-promoted @@ -137,10 +141,22 @@ function composeCfgForServer( // In-beta overlays scoped to this server. const inBetaScopedOverlays = inputs.inBetaOverlays .filter(({ script }) => scopeAppliesToServer(script, { game: serverGame, map: serverMap })) - .map(({ script, version }) => ({ id: script.id, text: getVersionContent(version.id).toString('utf8') })); + .map(({ script, version }) => ({ + id: script.id, + text: getVersionContent(version.id).toString('utf8') + })); - const composed = composeServerCfg(baselineText, [...promotedOverlays, ...inBetaScopedOverlays], serverGame, serverMap, getServer(serverName)?.map_slug); - return { text: applyRuntimePassword(composed, runtimePasswordForServer(serverName)), baselineScriptId }; + const composed = composeServerCfg( + baselineText, + [...promotedOverlays, ...inBetaScopedOverlays], + serverGame, + serverMap, + getServer(serverName)?.map_slug + ); + return { + text: applyRuntimePassword(composed, runtimePasswordForServer(serverName)), + baselineScriptId + }; } export interface BetaRow { @@ -163,8 +179,8 @@ export interface BetaServerRow { beta_id: number; server: string; server_deleted_at: string | null; - had_original: number; // deprecated; per-script data lives in beta_server_scripts - prior_version_id: number | null; // deprecated + had_original: number; // deprecated; per-script data lives in beta_server_scripts + prior_version_id: number | null; // deprecated stage: BetaStage; error: string | null; deployed_at: string | null; @@ -218,19 +234,35 @@ export class BetaError extends Error { } function assertNativeActivationSettled(betaId: number): void { - const pending = db().prepare(`SELECT n.operation_id FROM native_provisioning n + const pending = db() + .prepare( + `SELECT n.operation_id FROM native_provisioning n JOIN provisioning_sources s ON s.operation_id=n.deployment_id WHERE s.kind='beta' AND s.source_id=? AND n.operation_id IS NOT NULL - AND n.files_json IS NOT NULL AND n.applied_at IS NULL`).all(String(betaId)) as { operation_id: string }[]; - if (pending.some(row => getHostOperation(row.operation_id)?.state !== 'failed')) throw new BetaError('The host is still reconciling file activation. Wait for its durable before images before promoting or rolling back.', 'conflict'); + AND n.files_json IS NOT NULL AND n.applied_at IS NULL` + ) + .all(String(betaId)) as { operation_id: string }[]; + if (pending.some((row) => getHostOperation(row.operation_id)?.state !== 'failed')) + throw new BetaError( + 'The host is still reconciling file activation. Wait for its durable before images before promoting or rolling back.', + 'conflict' + ); } function nowIso(): string { return new Date().toISOString(); } -function unresolvedCampaignWrites(server: string): Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }> { - return db().prepare(`SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path +function unresolvedCampaignWrites(server: string): Array<{ + server: string; + deploymentId: string; + state: string; + kind: ArtifactKind; + rel_path: string; +}> { + return db() + .prepare( + `SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path FROM staged_file_changes f JOIN deployment_servers ds ON ds.deployment_id = f.source_id AND ds.server = f.server LEFT JOIN server_deployments operation ON operation.id = ds.provisioning_id @@ -238,8 +270,15 @@ function unresolvedCampaignWrites(server: string): Array<{ server: string; deplo AND ds.restart_state IN ('pending','failed','held','manual') AND (f.state = 'applying' OR (f.state = 'pending' AND (ds.restart_state IN ('held','manual') OR operation.id IS NULL - OR operation.state IN ('queued','waiting','running','awaiting-health'))))`) - .all(server) as Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }>; + OR operation.state IN ('queued','waiting','running','awaiting-health'))))` + ) + .all(server) as Array<{ + server: string; + deploymentId: string; + state: string; + kind: ArtifactKind; + rel_path: string; + }>; } // ---------- Reads ---------- @@ -250,7 +289,9 @@ export function getBeta(id: number): BetaRow | undefined { export function listBetas(state?: BetaRow['state']): BetaRow[] { if (state) { - return db().prepare('SELECT * FROM betas WHERE state = ? ORDER BY opened_at DESC').all(state) as BetaRow[]; + return db() + .prepare('SELECT * FROM betas WHERE state = ? ORDER BY opened_at DESC') + .all(state) as BetaRow[]; } return db().prepare('SELECT * FROM betas ORDER BY opened_at DESC').all() as BetaRow[]; } @@ -263,22 +304,33 @@ export function listBetaServers(betaId: number): BetaServerRow[] { /** History retains names, so operational callers must also check the participant's identity. */ export function isActiveBetaServer(betaId: number, server: string): boolean { - return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server - WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`) + return !!db() + .prepare( + `SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server + WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL` + ) .get(betaId, server); } export function listBetaScripts(betaId: number, server?: string): BetaScriptRow[] { return db() - .prepare(`SELECT bs.* FROM beta_scripts bs WHERE bs.beta_id = ? + .prepare( + `SELECT bs.* FROM beta_scripts bs WHERE bs.beta_id = ? ${server === undefined ? '' : 'AND EXISTS (SELECT 1 FROM beta_server_scripts bss WHERE bss.beta_id = bs.beta_id AND bss.script_id = bs.script_id AND bss.server = ?)'} - ORDER BY bs.script_id`) + ORDER BY bs.script_id` + ) .all(betaId, ...(server === undefined ? [] : [server])) as BetaScriptRow[]; } -export function getBetaServerScript(betaId: number, server: string, scriptId: number): BetaServerScriptRow | undefined { +export function getBetaServerScript( + betaId: number, + server: string, + scriptId: number +): BetaServerScriptRow | undefined { return db() - .prepare('SELECT * FROM beta_server_scripts WHERE beta_id = ? AND server = ? AND script_id = ?') + .prepare( + 'SELECT * FROM beta_server_scripts WHERE beta_id = ? AND server = ? AND script_id = ?' + ) .get(betaId, server, scriptId) as BetaServerScriptRow | undefined; } @@ -320,26 +372,40 @@ export interface OpenBetaResult { export async function openBeta(input: OpenBetaInput): Promise { const automatic = input.instancesPerGame !== undefined; - if (automatic && input.servers !== undefined) throw new BetaError('Choose automatic instance counts or explicit servers, not both', 'invalid-state'); - const plan = automatic ? parseBetaPlan({ ...(input.testPlan ?? defaultPlan()), scope: 'all' }) - : input.testPlan ? parseBetaPlan(input.testPlan) : null; + if (automatic && input.servers !== undefined) + throw new BetaError( + 'Choose automatic instance counts or explicit servers, not both', + 'invalid-state' + ); + const plan = automatic + ? parseBetaPlan({ ...(input.testPlan ?? defaultPlan()), scope: 'all' }) + : input.testPlan + ? parseBetaPlan(input.testPlan) + : null; if (!input.scripts || input.scripts.length === 0) { throw new BetaError('at least one script is required', 'no-scripts'); } // Resolve every (script, version) pair and validate belonging. - interface Resolved { script: ScriptRow; version: ScriptVersionRow } + interface Resolved { + script: ScriptRow; + version: ScriptVersionRow; + } const resolved: Resolved[] = []; const seenScriptIds = new Set(); for (const s of input.scripts) { if (seenScriptIds.has(s.scriptId)) { - throw new BetaError(`script ${s.scriptId} listed twice in one beta`, 'duplicate-script'); + throw new BetaError( + `script ${s.scriptId} listed twice in one beta`, + 'duplicate-script' + ); } seenScriptIds.add(s.scriptId); const script = getScript(s.scriptId); if (!script) throw new BetaError(`no such script (id=${s.scriptId})`, 'no-such-script'); const version = getVersion(s.scriptVersionId); - if (!version) throw new BetaError(`no such version (id=${s.scriptVersionId})`, 'no-such-version'); + if (!version) + throw new BetaError(`no such version (id=${s.scriptVersionId})`, 'no-such-version'); if (version.script_id !== script.id) { throw new BetaError( `version ${s.scriptVersionId} does not belong to script ${s.scriptId}`, @@ -351,51 +417,91 @@ export async function openBeta(input: OpenBetaInput): Promise { let selectedServers = input.servers; if (automatic) { - try { selectedServers = selectBetaServers(resolved.map(row => row.script), input.instancesPerGame); } - catch (error) { throw new BetaError((error as Error).message, 'invalid-state'); } + try { + selectedServers = selectBetaServers( + resolved.map((row) => row.script), + input.instancesPerGame + ); + } catch (error) { + throw new BetaError((error as Error).message, 'invalid-state'); + } } - if (!selectedServers?.length) throw new BetaError('at least one server is required', 'no-servers'); - if (new Set(selectedServers).size !== selectedServers.length) throw new BetaError('Select each server only once', 'invalid-state'); - if (plan?.scope === 'quorum' && plan.quorum > selectedServers.length) throw new BetaError('Quorum exceeds selected servers', 'invalid-state'); + if (!selectedServers?.length) + throw new BetaError('at least one server is required', 'no-servers'); + if (new Set(selectedServers).size !== selectedServers.length) + throw new BetaError('Select each server only once', 'invalid-state'); + if (plan?.scope === 'quorum' && plan.quorum > selectedServers.length) + throw new BetaError('Quorum exceeds selected servers', 'invalid-state'); // Look up server rows once. const serverRows: Array<{ name: string; game: string | null; map: string | null }> = []; for (const name of selectedServers) { const row = getServer(name); - if (!row || row.destroyed_at) throw new BetaError(`unknown server: ${name}`, 'unknown-server'); + if (!row || row.destroyed_at) + throw new BetaError(`unknown server: ${name}`, 'unknown-server'); // Once teardown is queued or in flight, opening a beta cannot safely // claim the instance. A failed remote request may still be running. - const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL + const removal = db() + .prepare( + `SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health') OR (state IN ('failed', 'interrupted') AND stage IN ( 'Stopping and removing container', 'Removing server files and registry'))) - LIMIT 1`).get(name); - if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict'); + LIMIT 1` + ) + .get(name); + if (removal) + throw new BetaError( + `${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, + 'conflict' + ); serverRows.push({ name, game: row.game, map: row.map }); } if (plan && ['map', 'game', 'map-game'].includes(plan.scope)) { - const missing = serverRows.some(s => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map)); - if (missing) throw new BetaError('Coverage requires game/map metadata on every selected server', 'invalid-state'); + const missing = serverRows.some( + (s) => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map) + ); + if (missing) + throw new BetaError( + 'Coverage requires game/map metadata on every selected server', + 'invalid-state' + ); } // Each server receives only matching bundle members. Unscoped files apply // to every selected server; every file must be exercised by at least one. - const perServer = new Map(serverRows.map(server => [server.name, resolved.filter(({ script }) => scopeAppliesToServer(script, server))])); + const perServer = new Map( + serverRows.map((server) => [ + server.name, + resolved.filter(({ script }) => scopeAppliesToServer(script, server)) + ]) + ); for (const [server, members] of perServer) { - if (!members.length) throw new BetaError(`${server}: no bundled files apply to this server`, 'out-of-scope'); + if (!members.length) + throw new BetaError(`${server}: no bundled files apply to this server`, 'out-of-scope'); const seenTargets = new Map(); for (const { script } of members) { if (script.kind === 'config') continue; - const target = script.kind === 'baseline' ? 'baseline' : `${script.kind}:${script.target_rel_path}`; + const target = + script.kind === 'baseline' + ? 'baseline' + : `${script.kind}:${script.target_rel_path}`; const prior = seenTargets.get(target); - if (prior !== undefined) throw new BetaError(`${server}: files ${prior} and ${script.id} share target ${script.target_rel_path}`, 'duplicate-target'); + if (prior !== undefined) + throw new BetaError( + `${server}: files ${prior} and ${script.id} share target ${script.target_rel_path}`, + 'duplicate-target' + ); seenTargets.set(target, script.id); } } for (const { script } of resolved) { - if (!serverRows.some(server => scopeAppliesToServer(script, server))) { - throw new BetaError(`${script.name}: select at least one server matching ${script.scope_type}=${script.scope_value}`, 'out-of-scope'); + if (!serverRows.some((server) => scopeAppliesToServer(script, server))) { + throw new BetaError( + `${script.name}: select at least one server matching ${script.scope_type}=${script.scope_value}`, + 'out-of-scope' + ); } } @@ -404,10 +510,14 @@ export async function openBeta(input: OpenBetaInput): Promise { // Legacy campaigns have no staged payload and retain their restart-only priority. const queuedCampaigns = selectedServers.flatMap(unresolvedCampaignWrites); if (queuedCampaigns.length) { - const interrupted = queuedCampaigns.some(row => row.state === 'applying'); - throw new BetaError(interrupted - ? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.' - : 'Finish or resume queued file campaigns before opening a beta on these servers.', 'conflict', queuedCampaigns); + const interrupted = queuedCampaigns.some((row) => row.state === 'applying'); + throw new BetaError( + interrupted + ? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.' + : 'Finish or resume queued file campaigns before opening a beta on these servers.', + 'conflict', + queuedCampaigns + ); } // Per-(server, script) conflict check. heldByBeta covers "another running @@ -421,7 +531,8 @@ export async function openBeta(input: OpenBetaInput): Promise { if (owner) conflicts.push({ server: name, scriptId: script.id, heldByBetaId: owner }); } } - if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts); + if (conflicts.length > 0) + throw new BetaError('target already held by another beta', 'conflict', conflicts); // DB writes in one transaction. const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols @@ -451,30 +562,35 @@ export async function openBeta(input: OpenBetaInput): Promise { input.notes ?? null ); betaId = Number(res.lastInsertRowid); - if (plan) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(plan)); + if (plan) + db() + .prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)') + .run(betaId, JSON.stringify(plan)); for (const r of resolved) { insertBetaScript.run(betaId, r.script.id, r.version.id, r.script.kind); } for (const name of selectedServers) { insertBetaServer.run(betaId, name, nowIso()); if (plan) { - const selected = serverRows.find(s => s.name === name)!; - db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map); + const selected = serverRows.find((s) => s.name === name)!; + db() + .prepare( + 'INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)' + ) + .run(betaId, name, selected.game, selected.map); } for (const r of perServer.get(name)!) { - insertBetaServerScript.run( - betaId, - name, - r.script.id, - 0, - null - ); + insertBetaServerScript.run(betaId, name, r.script.id, 0, null); } } })(); - if (plan?.waveSize) db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId); - await deployBetaServers(betaId, plan?.waveSize ? selectedServers.slice(0, plan.waveSize) : selectedServers); + if (plan?.waveSize) + db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId); + await deployBetaServers( + betaId, + plan?.waveSize ? selectedServers.slice(0, plan.waveSize) : selectedServers + ); emit({ type: 'beta-opened', @@ -490,49 +606,102 @@ export async function openBeta(input: OpenBetaInput): Promise { }; } -export interface BetaRevisionUpload { scriptId: number; content: Buffer; releaseVersion?: string } +export interface BetaRevisionUpload { + scriptId: number; + content: Buffer; + releaseVersion?: string; +} /** Replace existing bundle members; commit files, deadline and evidence reset together. */ -export async function reviseBeta(betaId: number, expectedRevision: number, uploads: BetaRevisionUpload[], notes = ''): Promise<{ revision: number }> { +export async function reviseBeta( + betaId: number, + expectedRevision: number, + uploads: BetaRevisionUpload[], + notes = '' +): Promise<{ revision: number }> { // Drain a runner already observing the previous revision before resetting it. const { betaTick } = await import('../pipeline/beta-runner.js'); await betaTick(); return exclusive(betaId, async () => { const beta = getBeta(betaId); if (!beta) throw new BetaError('Beta not found', 'no-such-beta'); - if (!['running', 'ready-promote'].includes(beta.state)) throw new BetaError('Beta is closed', 'invalid-state'); + if (!['running', 'ready-promote'].includes(beta.state)) + throw new BetaError('Beta is closed', 'invalid-state'); const current = currentBetaRevision(betaId); - if (current.revision !== expectedRevision) throw new BetaError('The beta changed. Reload and review the current revision.', 'preview-changed'); - if (!uploads.length || new Set(uploads.map(file => file.scriptId)).size !== uploads.length) { + if (current.revision !== expectedRevision) + throw new BetaError( + 'The beta changed. Reload and review the current revision.', + 'preview-changed' + ); + if ( + !uploads.length || + new Set(uploads.map((file) => file.scriptId)).size !== uploads.length + ) { throw new BetaError('Choose at least one replacement, once per file.', 'invalid-state'); } - if (notes.length > 1000) throw new BetaError('Change notes must be at most 1000 characters.', 'invalid-state'); + if (notes.length > 1000) + throw new BetaError('Change notes must be at most 1000 characters.', 'invalid-state'); const bundle = listBetaScripts(betaId); for (const file of uploads) { - const member = bundle.find(row => row.script_id === file.scriptId); - if (!member) throw new BetaError('Updates can only replace files already in this beta.', 'invalid-state'); - if (!file.content.length) throw new BetaError('Choose non-empty replacement files.', 'invalid-state'); + const member = bundle.find((row) => row.script_id === file.scriptId); + if (!member) + throw new BetaError( + 'Updates can only replace files already in this beta.', + 'invalid-state' + ); + if (!file.content.length) + throw new BetaError('Choose non-empty replacement files.', 'invalid-state'); if (getVersion(member.script_version_id)!.sha256 === sha256Buffer(file.content)) { - throw new BetaError(`${getScript(file.scriptId)!.name}: the replacement is identical to the current version.`, 'invalid-state'); + throw new BetaError( + `${getScript(file.scriptId)!.name}: the replacement is identical to the current version.`, + 'invalid-state' + ); } } const servers = listBetaServers(betaId); - if (!servers.length || servers.some(server => !isActiveBetaServer(betaId, server.server))) { - throw new BetaError('An original beta server is missing. Roll back and open a new beta.', 'invalid-state'); + if ( + !servers.length || + servers.some((server) => !isActiveBetaServer(betaId, server.server)) + ) { + throw new BetaError( + 'An original beta server is missing. Roll back and open a new beta.', + 'invalid-state' + ); } ensureHostSchema(db()); assertNativeActivationSettled(betaId); - if (stagedFileChanges('beta', String(betaId)).some(row => row.state === 'applying')) { - throw new BetaError('A file write was interrupted. Resolve it before updating the beta.', 'conflict'); + if (stagedFileChanges('beta', String(betaId)).some((row) => row.state === 'applying')) { + throw new BetaError( + 'A file write was interrupted. Resolve it before updating the beta.', + 'conflict' + ); } - const jobs = db().prepare(`SELECT DISTINCT d.id, d.state, d.submitted_at, n.operation_id FROM server_deployments d + const jobs = db() + .prepare( + `SELECT DISTINCT d.id, d.state, d.submitted_at, n.operation_id FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id LEFT JOIN native_provisioning n ON n.deployment_id = d.id - WHERE s.kind = 'beta' AND s.source_id = ?`).all(String(betaId)) as - Array<{ id: string; state: string; submitted_at: string | null; operation_id: string | null }>; - if (jobs.some(job => ['running', 'awaiting-health'].includes(job.state) - || (job.state === 'interrupted' && servers.some(server => server.provisioning_id === job.id)) - || (['waiting', 'queued'].includes(job.state) && (job.submitted_at || job.operation_id)))) { - throw new BetaError('Wait for the current activation and recovery to finish before updating this beta.', 'conflict'); + WHERE s.kind = 'beta' AND s.source_id = ?` + ) + .all(String(betaId)) as Array<{ + id: string; + state: string; + submitted_at: string | null; + operation_id: string | null; + }>; + if ( + jobs.some( + (job) => + ['running', 'awaiting-health'].includes(job.state) || + (job.state === 'interrupted' && + servers.some((server) => server.provisioning_id === job.id)) || + (['waiting', 'queued'].includes(job.state) && + (job.submitted_at || job.operation_id)) + ) + ) { + throw new BetaError( + 'Wait for the current activation and recovery to finish before updating this beta.', + 'conflict' + ); } const stored = storedPlan(betaId); const plan = stored ? parseBetaPlan(JSON.parse(stored.plan)) : null; @@ -540,70 +709,148 @@ export async function reviseBeta(betaId: number, expectedRevision: number, uploa const at = nowIso(); db().transaction(() => { archiveBetaRevision(betaId, current.revision, current.started_at, null); - db().prepare('UPDATE beta_revisions SET results_json = ? WHERE beta_id = ? AND revision = ?') - .run(JSON.stringify({ state: beta.state, test: testView(betaId), servers }), betaId, current.revision); + db() + .prepare( + 'UPDATE beta_revisions SET results_json = ? WHERE beta_id = ? AND revision = ?' + ) + .run( + JSON.stringify({ state: beta.state, test: testView(betaId), servers }), + betaId, + current.revision + ); for (const upload of uploads) { - const version = uploadNewVersion(upload.scriptId, upload.content, notes.trim() || undefined, upload.releaseVersion); - db().prepare('UPDATE beta_scripts SET script_version_id = ? WHERE beta_id = ? AND script_id = ?') + const version = uploadNewVersion( + upload.scriptId, + upload.content, + notes.trim() || undefined, + upload.releaseVersion + ); + db() + .prepare( + 'UPDATE beta_scripts SET script_version_id = ? WHERE beta_id = ? AND script_id = ?' + ) .run(version.id, betaId, upload.scriptId); - db().prepare('UPDATE betas SET script_version_id = ? WHERE id = ? AND script_id = ?') + db() + .prepare( + 'UPDATE betas SET script_version_id = ? WHERE id = ? AND script_id = ?' + ) .run(version.id, betaId, upload.scriptId); } // Preflight all targets before cancelling work. Errors roll back the // uploads and history as well as the beta changes. - const changes = new Map(servers.map(server => [server.server, revisionFileChanges(betaId, server.server)])); + const changes = new Map( + servers.map((server) => [server.server, revisionFileChanges(betaId, server.server)]) + ); // Betas created before staged activation may only have deployed_at // and snapshots. Record their first writes before replacing staging. - for (const server of servers.filter(server => server.deployed_at)) { + for (const server of servers.filter((server) => server.deployed_at)) { for (const change of changes.get(server.server)!) { - if (isFileApplied('beta', String(betaId), server.server, change.kind, change.relPath) === null) { - db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)') + if ( + isFileApplied( + 'beta', + String(betaId), + server.server, + change.kind, + change.relPath + ) === null + ) { + db() + .prepare( + 'INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)' + ) .run(betaId, server.server, change.relPath); } } } - for (const job of jobs.filter(job => ['waiting', 'queued'].includes(job.state))) { + for (const job of jobs.filter((job) => ['waiting', 'queued'].includes(job.state))) { cancelProvisioningJob(job.id, `Superseded by beta #${betaId} revision ${revision}`); } - for (const server of servers) releaseRestartRequest(server.server, { kind: 'beta', id: String(betaId) }); + for (const server of servers) + releaseRestartRequest(server.server, { kind: 'beta', id: String(betaId) }); // New IDs prevent historical host receipts from resolving to a newer // revision's desired bytes. Original before images stay intact. - db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId)); - db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(at, betaId); - db().prepare(`UPDATE beta_servers SET stage = 'queued', error = NULL, deployed_at = NULL, - restarted_at = NULL, healthy_at = NULL, match_seen_at = NULL, updated_at = ? WHERE beta_id = ?`).run(at, betaId); + db() + .prepare( + "DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = ?" + ) + .run(String(betaId)); + db() + .prepare( + 'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL' + ) + .run(at, betaId); + db() + .prepare( + `UPDATE beta_servers SET stage = 'queued', error = NULL, deployed_at = NULL, + restarted_at = NULL, healthy_at = NULL, match_seen_at = NULL, updated_at = ? WHERE beta_id = ?` + ) + .run(at, betaId); db().prepare("UPDATE betas SET state = 'running' WHERE id = ?").run(betaId); setOutcome(betaId, 'active', null); archiveBetaRevision(betaId, revision, at, notes.trim() || null); // The original first wave follows selection order; later waves are // ordered by the runner. Preserve that canary selection on revision. - const firstWave = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ? ORDER BY rowid LIMIT ?') + const firstWave = db() + .prepare('SELECT server FROM beta_servers WHERE beta_id = ? ORDER BY rowid LIMIT ?') .all(betaId, plan?.waveSize || servers.length) as { server: string }[]; for (const server of firstWave) { for (const change of changes.get(server.server)!) queueFileChange(change); - db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty' WHERE beta_id = ? AND server = ?").run(betaId, server.server); + db() + .prepare( + "UPDATE beta_servers SET stage = 'awaiting-empty' WHERE beta_id = ? AND server = ?" + ) + .run(betaId, server.server); } - activity(betaId, `Revision ${revision} submitted; fresh tests and testing window${notes.trim() ? `: ${notes.trim()}` : ''}`); + activity( + betaId, + `Revision ${revision} submitted; fresh tests and testing window${notes.trim() ? `: ${notes.trim()}` : ''}` + ); })(); emit({ type: 'beta-state', betaId, state: 'running' }); return { revision }; }); } -function revisionFileChanges(betaId: number, name: string): Parameters[0][] { - const resolved = listBetaScripts(betaId, name).map(row => ({ script: getScript(row.script_id)!, version: getVersion(row.script_version_id)! })); +function revisionFileChanges( + betaId: number, + name: string +): Parameters[0][] { + const resolved = listBetaScripts(betaId, name).map((row) => ({ + script: getScript(row.script_id)!, + version: getVersion(row.script_version_id)! + })); const server = getServer(name)!; - const changes: Parameters[0][] = resolved.filter(row => !isCollapsingKind(row.script.kind)) - .map(({ script, version }) => ({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: script.kind, - relPath: script.target_rel_path, content: getVersionContent(version.id), scriptVersionId: version.id, inventorySource: `beta:${betaId}` })); - if (resolved.some(row => isCollapsingKind(row.script.kind))) { + const changes: Parameters[0][] = resolved + .filter((row) => !isCollapsingKind(row.script.kind)) + .map(({ script, version }) => ({ + sourceKind: 'beta', + sourceId: String(betaId), + server: name, + kind: script.kind, + relPath: script.target_rel_path, + content: getVersionContent(version.id), + scriptVersionId: version.id, + inventorySource: `beta:${betaId}` + })); + if (resolved.some((row) => isCollapsingKind(row.script.kind))) { const composed = composeCfgForServer(name, server.game, server.map, { - inBetaBaseline: resolved.find(row => row.script.kind === 'baseline')?.version, - inBetaOverlays: resolved.filter(row => row.script.kind === 'config') + inBetaBaseline: resolved.find((row) => row.script.kind === 'baseline')?.version, + inBetaOverlays: resolved.filter((row) => row.script.kind === 'config') + }); + if (!composed) + throw new BetaError( + `no baseline available for ${name} — upload one before running config overlays`, + 'invalid-state' + ); + changes.push({ + sourceKind: 'beta', + sourceId: String(betaId), + server: name, + kind: 'config', + relPath: configFileOf(name), + content: Buffer.from(composed.text), + inventorySource: `beta:${betaId}` }); - if (!composed) throw new BetaError(`no baseline available for ${name} — upload one before running config overlays`, 'invalid-state'); - changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: 'config', relPath: configFileOf(name), - content: Buffer.from(composed.text), inventorySource: `beta:${betaId}` }); } return changes; } @@ -614,9 +861,16 @@ export async function deployBetaServers(betaId: number, names: string[]): Promis } async function deployBetaServersInternal(betaId: number, names: string[]): Promise { for (const name of names) { - if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server'); + if (!isActiveBetaServer(betaId, name)) + throw new BetaError( + `${name}: original beta server was deleted; start a new beta`, + 'unknown-server' + ); } - const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! })); + const resolved = listBetaScripts(betaId).map((r) => ({ + script: getScript(r.script_id)!, + version: getVersion(r.script_version_id)! + })); const nonCollapsing = resolved.filter((r) => !isCollapsingKind(r.script.kind)); // Script IDs distinguish same-named files for different games in archives. @@ -633,20 +887,42 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi if (!isActiveBetaServer(betaId, name)) continue; try { const changes = revisionFileChanges(betaId, name); - const configChange = changes.find(change => change.kind === 'config'); - if (configChange?.content) await writeFile(join(archive, `${name}.${configFileOf(name)}`), configChange.content); + const configChange = changes.find((change) => change.kind === 'config'); + if (configChange?.content) + await writeFile( + join(archive, `${name}.${configFileOf(name)}`), + configChange.content + ); if (!isActiveBetaServer(betaId, name)) continue; db().transaction(() => { for (const change of changes) queueFileChange(change); - db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") + db() + .prepare( + "UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL" + ) .run(nowIso(), betaId, name); })(); - emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null }); + emit({ + type: 'beta-server-stage', + betaId, + server: name, + stage: 'awaiting-empty', + error: null + }); } catch (error) { const message = `deploy: ${(error as Error).message}`; - db().prepare("UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") + db() + .prepare( + "UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL" + ) .run(message, nowIso(), betaId, name); - emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: message }); + emit({ + type: 'beta-server-stage', + betaId, + server: name, + stage: 'failed', + error: message + }); } } } @@ -661,10 +937,16 @@ export interface RollbackResult { warnings: string[]; } -export async function rollbackBeta(betaId: number, options: { restart?: 'manual' | 'when-empty' } = {}): Promise { +export async function rollbackBeta( + betaId: number, + options: { restart?: 'manual' | 'when-empty' } = {} +): Promise { return exclusive(betaId, () => rollbackBetaInternal(betaId, options)); } -async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise { +async function rollbackBetaInternal( + betaId: number, + options: { restart?: 'manual' | 'when-empty' } +): Promise { const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'running' && beta.state !== 'ready-promote') { @@ -681,12 +963,23 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual ensureHostSchema(db()); db().transaction(() => { - const jobs = db().prepare(`SELECT s.operation_id FROM provisioning_sources s WHERE s.kind='beta' AND s.source_id=?`) + const jobs = db() + .prepare( + `SELECT s.operation_id FROM provisioning_sources s WHERE s.kind='beta' AND s.source_id=?` + ) .all(String(betaId)) as { operation_id: string }[]; for (const job of jobs) { - const undelivered = db().prepare(`SELECT 1 FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id - WHERE n.deployment_id=? AND o.state='pending' AND o.last_delivered_at IS NULL`).get(job.operation_id); - if (undelivered) cancelProvisioningJob(job.operation_id, `Superseded by rollback of beta #${betaId}`); + const undelivered = db() + .prepare( + `SELECT 1 FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id + WHERE n.deployment_id=? AND o.state='pending' AND o.last_delivered_at IS NULL` + ) + .get(job.operation_id); + if (undelivered) + cancelProvisioningJob( + job.operation_id, + `Superseded by rollback of beta #${betaId}` + ); reconcileNativeProvisioningReceipt(job.operation_id); } // Before images remain mandatory for a mutation already delivered to a host. @@ -703,8 +996,16 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual WHERE beta_id = ? AND server = ? AND rel_path = ?` ); - const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = []; - const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'); + const restorations: Array<{ + server: string; + kind: ArtifactKind; + relPath: string; + content: Buffer | null; + versionId: number | null; + }> = []; + const changed = db().prepare( + 'SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?' + ); const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => { // A queued revision may supersede an applied one before its own activation. if (changed.get(betaId, bs.server, path)) return true; @@ -715,31 +1016,47 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual }; for (const bs of servers) { if (!isActiveBetaServer(betaId, bs.server)) { - warnings.push(`${bs.server}: original server was deleted; retained history was skipped`); + warnings.push( + `${bs.server}: original server was deleted; retained history was skipped` + ); continue; } - const members = bundle.filter(({ script }) => getBetaServerScript(betaId, bs.server, script.id)); + const members = bundle.filter(({ script }) => + getBetaServerScript(betaId, bs.server, script.id) + ); for (const { script } of members) { - if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) continue; + if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) + continue; const bss = getBetaServerScript(betaId, bs.server, script.id); const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as - | { prior_content: Buffer; prior_sha256: string; prior_size: number } - | undefined; + { prior_content: Buffer; prior_sha256: string; prior_size: number } | undefined; if (bss?.had_original && !snap) { warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`); continue; } - restorations.push({ server: bs.server, kind: script.kind, relPath: script.target_rel_path, - content: snap?.prior_content ?? null, versionId: bss?.prior_version_id ?? null }); + restorations.push({ + server: bs.server, + kind: script.kind, + relPath: script.target_rel_path, + content: snap?.prior_content ?? null, + versionId: bss?.prior_version_id ?? null + }); (snap ? restored : dropped).add(bs.server); } const targetRel = configFileOf(bs.server); - if (members.some(({ script }) => isCollapsingKind(script.kind)) && applied(bs, 'config', targetRel)) { + if ( + members.some(({ script }) => isCollapsingKind(script.kind)) && + applied(bs, 'config', targetRel) + ) { const snap = getSnapshot.get(betaId, bs.server, targetRel) as - | { prior_content: Buffer; prior_sha256: string; prior_size: number } - | undefined; - restorations.push({ server: bs.server, kind: 'config', relPath: targetRel, - content: snap?.prior_content ?? null, versionId: null }); + { prior_content: Buffer; prior_sha256: string; prior_size: number } | undefined; + restorations.push({ + server: bs.server, + kind: 'config', + relPath: targetRel, + content: snap?.prior_content ?? null, + versionId: null + }); (snap ? restored : dropped).add(bs.server); } } @@ -748,41 +1065,80 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual db().transaction(() => { for (const bs of servers) { if (!isActiveBetaServer(betaId, bs.server)) continue; - const pending = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).all(bs.server) as { id: string }[]; - for (const job of pending) cancelProvisioningJob(job.id, `Superseded by rollback of beta #${betaId}`); + const pending = db() + .prepare( + `SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted')` + ) + .all(bs.server) as { id: string }[]; + for (const job of pending) + cancelProvisioningJob(job.id, `Superseded by rollback of beta #${betaId}`); } cancelStagedFiles('beta', String(betaId)); if (deploymentId) { - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)") + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)" + ) .run(deploymentId, nowIso(), storedPlan(betaId) ? 'beta-test-plan' : 'ui', betaId); - for (const server of new Set(restorations.map(change => change.server))) { - db().prepare('INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)') - .run(deploymentId, server, restorations.filter(change => change.server === server).length, - options.restart === 'manual' ? 'held' : 'pending'); + for (const server of new Set(restorations.map((change) => change.server))) { + db() + .prepare( + 'INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)' + ) + .run( + deploymentId, + server, + restorations.filter((change) => change.server === server).length, + options.restart === 'manual' ? 'held' : 'pending' + ); } for (const change of restorations) { - const file = db().prepare(`INSERT INTO deployment_files + const file = db() + .prepare( + `INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) - VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server, - change.content === null ? 'removed' : 'updated', change.relPath, - change.content === null ? null : sha256Buffer(change.content), putFileBlob(change.content)); - queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server, - kind: change.kind, relPath: change.relPath, content: change.content, - scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`, - deploymentFileId: Number(file.lastInsertRowid) }); + VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)` + ) + .run( + deploymentId, + change.server, + change.content === null ? 'removed' : 'updated', + change.relPath, + change.content === null ? null : sha256Buffer(change.content), + putFileBlob(change.content) + ); + queueFileChange({ + sourceKind: 'campaign', + sourceId: deploymentId, + server: change.server, + kind: change.kind, + relPath: change.relPath, + content: change.content, + scriptVersionId: change.versionId, + inventorySource: `rollback:${betaId}`, + deploymentFileId: Number(file.lastInsertRowid) + }); } } if (deploymentId) { - for (const server of new Set(restorations.map(change => change.server))) { + for (const server of new Set(restorations.map((change) => change.server))) { requestRestart(server, { kind: 'campaign', id: deploymentId }); } } // The queued restoration owns a durable copy before snapshots are released. db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); - db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId); + db() + .prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?") + .run(nowIso(), betaId); })(); - if (deploymentId) emit({ type: 'deployment-opened', deploymentId, action: 'rollback', servers: [...new Set(restorations.map(change => change.server))] }); + if (deploymentId) + emit({ + type: 'deployment-opened', + deploymentId, + action: 'rollback', + servers: [...new Set(restorations.map((change) => change.server))] + }); // Wipe archive + close out beta. const archive = betaArchiveOf(betaId); @@ -805,7 +1161,9 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual .get(versionId, betaId); const isCurrent = script.current_version_id === versionId; if (!stillOk && !isCurrent) { - db().prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`).run(versionId); + db() + .prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`) + .run(versionId); } } @@ -825,66 +1183,134 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual export interface PromoteResult { betaId: number; deploymentId: string; - landed: string[]; // unique servers that received at least one file + landed: string[]; // unique servers that received at least one file scriptIds: number[]; awaitingActivation: number; failedServers: string[]; } /** Snapshot exactly the scope expansion and composed configuration reviewed by the operator. */ -function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configContent: Map } { +function promotionPlan(betaId: number): { + preview: BetaPromotionPreview; + configContent: Map; +} { const beta = getBeta(betaId); if (!beta) throw new BetaError('This beta no longer exists.', 'no-such-beta'); - const bundle = listBetaScripts(betaId).map(row => { - const script = getScript(row.script_id), version = getVersion(row.script_version_id); - if (!script || !version) throw new BetaError('A selected file version is no longer available.', 'no-such-version'); + const bundle = listBetaScripts(betaId).map((row) => { + const script = getScript(row.script_id), + version = getVersion(row.script_version_id); + if (!script || !version) + throw new BetaError( + 'A selected file version is no longer available.', + 'no-such-version' + ); return { script, version }; }); if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts'); const participants = listBetaServers(betaId); - const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server)); - const targetsByScript = new Map(bundle.map(({ script }) => { - const participants = [...tested].filter(server => getBetaServerScript(betaId, server, script.id)); - const scopeServers = script.scope_type && script.scope_value ? serversMatchingScope(script.scope_type, script.scope_value) : []; - return [script.id, [...new Set([...participants, ...scopeServers])].sort()]; - })); + const tested = new Set( + participants + .filter((row) => isActiveBetaServer(betaId, row.server)) + .map((row) => row.server) + ); + const targetsByScript = new Map( + bundle.map(({ script }) => { + const participants = [...tested].filter((server) => + getBetaServerScript(betaId, server, script.id) + ); + const scopeServers = + script.scope_type && script.scope_value + ? serversMatchingScope(script.scope_type, script.scope_value) + : []; + return [script.id, [...new Set([...participants, ...scopeServers])].sort()]; + }) + ); const targets = (script: ScriptRow) => targetsByScript.get(script.id)!; - const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind, - path: script.target_rel_path, versionId: version.id, version: version.version, - release_version: version.release_version, sha256: version.sha256, - scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) })); - const names = [...new Set(files.flatMap(file => file.servers))].sort(); - const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server)) - .map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`); - const servers = names.map(name => { + const files = bundle.map(({ script, version }) => ({ + id: script.id, + name: script.name, + kind: script.kind, + path: script.target_rel_path, + versionId: version.id, + version: version.version, + release_version: version.release_version, + sha256: version.sha256, + scope: script.scope_type + ? `${script.scope_type}: ${script.scope_value}` + : 'Selected test servers only', + servers: targets(script) + })); + const names = [...new Set(files.flatMap((file) => file.servers))].sort(); + const blockers: string[] = participants + .filter((row) => !isActiveBetaServer(betaId, row.server)) + .map( + (row) => + `${row.server}: original test server was deleted; start a new beta before promoting.` + ); + const servers = names.map((name) => { const server = getServer(name); - if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`); + if (!server || server.destroyed_at) + blockers.push(`${name}: server is no longer available.`); const destinations = new Map(); - for (const file of files.filter(file => file.servers.includes(name) && file.kind !== 'config')) { + for (const file of files.filter( + (file) => file.servers.includes(name) && file.kind !== 'config' + )) { const target = file.kind === 'baseline' ? 'baseline' : `${file.kind}:${file.path}`; const prior = destinations.get(target); - if (prior !== undefined) blockers.push(`${name}: files ${prior} and ${file.id} share target ${file.path}.`); + if (prior !== undefined) + blockers.push(`${name}: files ${prior} and ${file.id} share target ${file.path}.`); destinations.set(target, file.id); } - return { name, tested: tested.has(name), game: server?.game ?? null, map: server?.map ?? null }; + return { + name, + tested: tested.has(name), + game: server?.game ?? null, + map: server?.map ?? null + }; }); const collapsing = bundle.filter(({ script }) => isCollapsingKind(script.kind)); const configContent = new Map(); - const configWrites = [...new Set(collapsing.flatMap(({ script }) => targets(script)))].sort().map(server => { - const row = getServer(server); - const matching = collapsing.filter(({ script }) => targets(script).includes(server)); - const composed = row && composeCfgForServer(server, row.game, row.map, { - inBetaBaseline: matching.find(({ script }) => script.kind === 'baseline')?.version, - inBetaOverlays: matching.filter(({ script }) => script.kind === 'config') + const configWrites = [...new Set(collapsing.flatMap(({ script }) => targets(script)))] + .sort() + .map((server) => { + const row = getServer(server); + const matching = collapsing.filter(({ script }) => targets(script).includes(server)); + const composed = + row && + composeCfgForServer(server, row.game, row.map, { + inBetaBaseline: matching.find(({ script }) => script.kind === 'baseline') + ?.version, + inBetaOverlays: matching.filter(({ script }) => script.kind === 'config') + }); + const error = composed + ? null + : `${server}: no baseline is available to compose its configuration.`; + if (error) blockers.push(error); + const content = composed ? Buffer.from(composed.text, 'utf8') : null; + if (content) configContent.set(server, content); + return { + server, + path: configFileOf(server), + sha256: content ? sha256Buffer(content) : null, + error + }; }); - const error = composed ? null : `${server}: no baseline is available to compose its configuration.`; - if (error) blockers.push(error); - const content = composed ? Buffer.from(composed.text, 'utf8') : null; - if (content) configContent.set(server, content); - return { server, path: configFileOf(server), sha256: content ? sha256Buffer(content) : null, error }; - }); - const impact = { betaId, files, servers, configWrites, blockers, restartPolicy: 'additional-when-empty' as const }; - return { preview: { ...impact, token: sha256Buffer(Buffer.from(JSON.stringify(impact))), createdAt: nowIso() }, configContent }; + const impact = { + betaId, + files, + servers, + configWrites, + blockers, + restartPolicy: 'additional-when-empty' as const + }; + return { + preview: { + ...impact, + token: sha256Buffer(Buffer.from(JSON.stringify(impact))), + createdAt: nowIso() + }, + configContent + }; } export function previewBetaPromotion(betaId: number): BetaPromotionPreview { @@ -892,7 +1318,10 @@ export function previewBetaPromotion(betaId: number): BetaPromotionPreview { } function promoteDeploymentId(betaId: number): string { - const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); + const stamp = new Date() + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d+Z$/, 'Z'); return `dep-${stamp}-promote-${betaId}`; } @@ -906,7 +1335,8 @@ export async function promoteBeta(betaId: number, reviewedToken?: string): Promi async function promoteBetaInternal(betaId: number, reviewedToken?: string): Promise { assertNativeActivationSettled(betaId); const test = storedPlan(betaId); - if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote'); + if (test && test.outcome !== 'passed') + throw new BetaError('Test plan has not passed', 'not-ready-promote'); const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'ready-promote') { @@ -916,9 +1346,14 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom const checkReview = () => { const current = previewBetaPromotion(betaId); if (current.token !== (reviewedToken ?? reviewed.preview.token)) { - throw new BetaError('Promotion targets or versions changed. Review the updated plan before promoting.', 'preview-changed', current); + throw new BetaError( + 'Promotion targets or versions changed. Review the updated plan before promoting.', + 'preview-changed', + current + ); } - if (current.blockers.length) throw new BetaError(current.blockers.join(' '), 'invalid-state'); + if (current.blockers.length) + throw new BetaError(current.blockers.join(' '), 'invalid-state'); }; checkReview(); const bundleRows = listBetaScripts(betaId); @@ -928,7 +1363,11 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom version: getVersion(br.script_version_id)! })); - const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server)); + const betaServerSet = new Set( + listBetaServers(betaId) + .filter((row) => isActiveBetaServer(betaId, row.server)) + .map((r) => r.server) + ); // Split bundle into non-collapsing (per-file) and collapsing (composed cfg). const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind)); @@ -953,7 +1392,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom ); for (const { script, version } of nonCollapsing) { - const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers; + const targets = reviewed.preview.files.find((file) => file.id === script.id)!.servers; for (const name of targets) { let plan: Plan; @@ -961,8 +1400,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom if (bss && betaServerSet.has(name)) { if (bss?.had_original) { const snap = getSnapshotForPromote.get(betaId, name, script.target_rel_path) as - | { prior_content: Buffer; prior_sha256: string } - | undefined; + { prior_content: Buffer; prior_sha256: string } | undefined; let priorContent: Buffer | null = null; let priorSha: string | null = null; if (snap) { @@ -970,19 +1408,46 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom priorSha = snap.prior_sha256; } else if (bss.prior_version_id) { const pv = getVersion(bss.prior_version_id); - if (pv) { priorContent = getVersionContent(pv.id); priorSha = pv.sha256; } + if (pv) { + priorContent = getVersionContent(pv.id); + priorSha = pv.sha256; + } } plan = { - script, version, server: name, - priorVersionId: bss.prior_version_id, priorSha, priorContent, hadFile: true, tested: true + script, + version, + server: name, + priorVersionId: bss.prior_version_id, + priorSha, + priorContent, + hadFile: true, + tested: true }; } else { - plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false, tested: true }; + plan = { + script, + version, + server: name, + priorVersionId: null, + priorSha: null, + priorContent: null, + hadFile: false, + tested: true + }; } } else { // Campaign activation captures the actual before image just // before replacing this file. Do not wait on remote reads here. - plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false, tested: false }; + plan = { + script, + version, + server: name, + priorVersionId: null, + priorSha: null, + priorContent: null, + hadFile: false, + tested: false + }; } plans.push(plan); } @@ -1005,7 +1470,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom } const cfgPlans: CfgPlan[] = []; if (collapsing.length > 0) { - const cfgTargets = reviewed.preview.configWrites.map(write => write.server); + const cfgTargets = reviewed.preview.configWrites.map((write) => write.server); for (const name of cfgTargets) { const srv = getServer(name); if (!srv) { @@ -1025,21 +1490,33 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom let priorContent: Buffer | null = null; let priorSha: string | null = null; let hadFile = false; - const tested = betaServerSet.has(name) && collapsing.some(({ script }) => getBetaServerScript(betaId, name, script.id)); + const tested = + betaServerSet.has(name) && + collapsing.some(({ script }) => getBetaServerScript(betaId, name, script.id)); // Beta server: preserve its original bytes captured at activation. if (tested) { const snap = getSnapshotForPromote.get(betaId, name, targetRel) as - | { prior_content: Buffer; prior_sha256: string } - | undefined; + { prior_content: Buffer; prior_sha256: string } | undefined; if (snap) { priorContent = snap.prior_content; priorSha = snap.prior_sha256; hadFile = true; } } - const requiresActivation = !tested || !await exists(targetPath) - || sha256Buffer(await readFile(targetPath)) !== composedSha; - cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation, tested }); + const requiresActivation = + !tested || + !(await exists(targetPath)) || + sha256Buffer(await readFile(targetPath)) !== composedSha; + cfgPlans.push({ + server: name, + priorContent, + priorSha, + composedContent: composedBuf, + composedSha, + hadFile, + requiresActivation, + tested + }); } } @@ -1049,14 +1526,36 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom // A scope expansion from another promotion may have been queued while this // beta was active. Reassert the tested bytes after that older campaign, even // when they already match the live file, so it cannot overwrite this release. - const queuedByServer = new Map(reviewed.preview.servers.map(server => [server.name, unresolvedCampaignWrites(server.name)])); - const plansToActivate = new Set(plans.filter(p => !p.tested - || queuedByServer.get(p.server)?.some(row => row.kind === p.script.kind && row.rel_path === p.script.target_rel_path))); + const queuedByServer = new Map( + reviewed.preview.servers.map((server) => [ + server.name, + unresolvedCampaignWrites(server.name) + ]) + ); + const plansToActivate = new Set( + plans.filter( + (p) => + !p.tested || + queuedByServer + .get(p.server) + ?.some( + (row) => + row.kind === p.script.kind && row.rel_path === p.script.target_rel_path + ) + ) + ); for (const cp of cfgPlans) { - if (queuedByServer.get(cp.server)?.some(row => row.kind === 'config' && row.rel_path === configFileOf(cp.server))) cp.requiresActivation = true; + if ( + queuedByServer + .get(cp.server) + ?.some((row) => row.kind === 'config' && row.rel_path === configFileOf(cp.server)) + ) + cp.requiresActivation = true; } - const activationServers = new Set([...Array.from(plansToActivate, plan => plan.server), - ...cfgPlans.filter(plan => plan.requiresActivation).map(plan => plan.server)]); + const activationServers = new Set([ + ...Array.from(plansToActivate, (plan) => plan.server), + ...cfgPlans.filter((plan) => plan.requiresActivation).map((plan) => plan.server) + ]); // Build the aggregate deployment. const depId = promoteDeploymentId(betaId); const startedAt = nowIso(); @@ -1081,15 +1580,20 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` ); - const uniqueServers = Array.from(new Set([ - ...plans.map((p) => p.server), - ...cfgPlans.map((c) => c.server) - ])); + const uniqueServers = Array.from( + new Set([...plans.map((p) => p.server), ...cfgPlans.map((c) => c.server)]) + ); db().transaction(() => { insertDep.run(depId, startedAt, betaId); for (const { script, version } of bundle) { - insertDepScript.run(depId, script.id, version.id, script.current_version_id, script.kind); + insertDepScript.run( + depId, + script.id, + version.id, + script.current_version_id, + script.kind + ); } // Record the reviewed change. Scope expansion gets its actual before // image when activated; tested servers keep their pre-beta history. @@ -1107,10 +1611,18 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom p.tested ? 0 : 1 ); if (plansToActivate.has(p)) { - queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server, - kind: p.script.kind, relPath: p.script.target_rel_path, content: getVersionContent(p.version.id), - scriptVersionId: p.version.id, inventorySource: `promote:${depId}`, - deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: p.tested }); + queueFileChange({ + sourceKind: 'campaign', + sourceId: depId, + server: p.server, + kind: p.script.kind, + relPath: p.script.target_rel_path, + content: getVersionContent(p.version.id), + scriptVersionId: p.version.id, + inventorySource: `promote:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), + preserveBeforeImage: p.tested + }); } } for (const cp of cfgPlans) { @@ -1127,10 +1639,18 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom cp.tested ? 0 : 1 ); if (cp.requiresActivation) { - queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server, - kind: 'config', relPath: configFileOf(cp.server), content: cp.composedContent, - scriptVersionId: null, inventorySource: `promote:${depId}`, - deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: cp.tested }); + queueFileChange({ + sourceKind: 'campaign', + sourceId: depId, + server: cp.server, + kind: 'config', + relPath: configFileOf(cp.server), + content: cp.composedContent, + scriptVersionId: null, + inventorySource: `promote:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), + preserveBeforeImage: cp.tested + }); } } // deployment_servers — one row per unique server. Beta servers are @@ -1147,30 +1667,42 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom cfgPlans.filter((c) => c.server === name).length; if (!activationServers.has(name)) { const bsRestartedAt = db() - .prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`) + .prepare( + `SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?` + ) .get(betaId, name) as { restarted_at: string | null } | undefined; - insertDepServer.run(depId, name, filesChanged, 'done', bsRestartedAt?.restarted_at ?? startedAt, null); + insertDepServer.run( + depId, + name, + filesChanged, + 'done', + bsRestartedAt?.restarted_at ?? startedAt, + null + ); } else { insertDepServer.run(depId, name, filesChanged, 'pending', null, null); } } // Update each script's current_version_id in the same transaction. for (const { script, version } of bundle) { - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(version.id, script.id); } })(); - db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId); + db() + .prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`) + .run(nowIso(), betaId); // The deployment history owns the pre-beta bytes for future rollback. db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); const archive = betaArchiveOf(betaId); if (await exists(archive)) await rm(archive, { recursive: true, force: true }); - const landed = Array.from(new Set([ - ...plans.map((p) => p.server), - ...cfgPlans.map((c) => c.server) - ])).sort(); + const landed = Array.from( + new Set([...plans.map((p) => p.server), ...cfgPlans.map((c) => c.server)]) + ).sort(); emit({ type: 'beta-state', betaId, state: 'promoted' }); emit({ type: 'deployment-opened', deploymentId: depId, action: 'promote', servers: landed }); @@ -1180,7 +1712,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom deploymentId: depId, landed, scriptIds: bundle.map((b) => b.script.id), - awaitingActivation: [...activationServers].filter(name => !serverErr.has(name)).length, + awaitingActivation: [...activationServers].filter((name) => !serverErr.has(name)).length, failedServers: [...serverErr.keys()] }; } diff --git a/src/lib/server/fleet/chat.ts b/src/lib/server/fleet/chat.ts index c3a11a9..6feb135 100644 --- a/src/lib/server/fleet/chat.ts +++ b/src/lib/server/fleet/chat.ts @@ -1,32 +1,61 @@ import { createHash } from 'node:crypto'; import { db } from '../db/index.js'; -import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type ContextualChatEntry } from '../../game-chat.js'; +import { + enrichGameChatTimeline, + type ChatMatch, + type ChatRoundReport, + type ContextualChatEntry +} from '../../game-chat.js'; import { parseGameEvent } from '../../telemetry.js'; import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js'; import { getServer, listServers, type ServerRow } from './servers.js'; -import { gameLogKey, resolveLogPath, readLogChunk, type LogCursor, type TailResult } from './logs.js'; +import { + gameLogKey, + resolveLogPath, + readLogChunk, + type LogCursor, + type TailResult +} from './logs.js'; import { listHosts } from '../hosts/index.js'; import { observationReader } from './status.js'; /** Historical context is scoped to this registration, including round-one matches. */ export function serverChatTimeline(server: ServerRow, tail: TailResult): ContextualChatEntry[] { - const unscoped = enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, matches: [], reports: [] }); - const times = unscoped.flatMap(entry => entry.context.realTime ? [entry.context.realTime] : []); + const unscoped = enrichGameChatTimeline(tail.lines, { + mtime: tail.mtime, + matches: [], + reports: [] + }); + const times = unscoped.flatMap((entry) => + entry.context.realTime ? [entry.context.realTime] : [] + ); if (!times.length) return unscoped; - const since = times.reduce((a, b) => a < b ? a : b), until = times.reduce((a, b) => a > b ? a : b); + const since = times.reduce((a, b) => (a < b ? a : b)), + until = times.reduce((a, b) => (a > b ? a : b)); const handle = db(); - const matches = handle.prepare(`SELECT id, started_at AS startedAt, ended_at AS endedAt, + const matches = handle + .prepare( + `SELECT id, started_at AS startedAt, ended_at AS endedAt, last_received_at AS lastReceivedAt, status, start_observed AS startObserved FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL AND started_at <= ? AND (COALESCE(ended_at,last_received_at) >= ? OR status='active') - ORDER BY id DESC LIMIT 2000`).all(server.name, until, since) as Array & { startObserved: number }>; + ORDER BY id DESC LIMIT 2000` + ) + .all(server.name, until, since) as Array< + Omit & { startObserved: number } + >; // Diagnostic history survives deletion. Its receipt time must belong to the // current registration so a reused server name cannot inherit old rounds. const cutoff = telemetryCutoff(); const roundSince = new Date(Date.parse(since) - 120_000).toISOString(); - const matchSince = matches.reduce((earliest, match) => match.startedAt < earliest ? match.startedAt : earliest, since); - const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round, + const matchSince = matches.reduce( + (earliest, match) => (match.startedAt < earliest ? match.startedAt : earliest), + since + ); + const reports = handle + .prepare( + `SELECT received_at AS receivedAt, event, round, player_count AS playerCount FROM telemetry_requests WHERE server = ? AND status = 200 AND received_at >= ? AND julianday(received_at) >= julianday(?) @@ -34,36 +63,69 @@ export function serverChatTimeline(server: ServerRow, tail: TailResult): Context AND received_at <= ? AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end') AND player_count >= 0 AND (event = 'game_start' OR round >= 0) - ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, roundSince, matchSince, + ORDER BY id DESC LIMIT ?` + ) + .all( + server.name, + cutoff, + server.created_at, + roundSince, + matchSince, new Date(Date.parse(until) + 120_000).toISOString(), - (telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[]; + telemetryRetention().maxPerServer || 10000 + ) as ChatRoundReport[]; // Saving gameplay does not depend on diagnostic logging succeeding. Keep the // surviving accepted snapshot useful when its request log was not recorded. - const latest = handle.prepare(`SELECT received_at AS receivedAt, payload FROM server_telemetry - WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)`) - .get(server.name, cutoff, server.created_at) as { receivedAt: string; payload: string | null } | undefined; + const latest = handle + .prepare( + `SELECT received_at AS receivedAt, payload FROM server_telemetry + WHERE server = ? AND received_at >= ? AND julianday(received_at) >= julianday(?)` + ) + .get(server.name, cutoff, server.created_at) as + { receivedAt: string; payload: string | null } | undefined; if (latest?.payload) { try { const event = parseGameEvent(JSON.parse(latest.payload)); const round = event.event === 'game_start' ? null : event.round; - if (event.server === server.name && !reports.some(report => report.receivedAt === latest.receivedAt - && report.event === event.event && report.round === round && report.playerCount === event.players.length)) { - reports.push({ receivedAt: latest.receivedAt, event: event.event, round, playerCount: event.players.length }); + if ( + event.server === server.name && + !reports.some( + (report) => + report.receivedAt === latest.receivedAt && + report.event === event.event && + report.round === round && + report.playerCount === event.players.length + ) + ) { + reports.push({ + receivedAt: latest.receivedAt, + event: event.event, + round, + playerCount: event.players.length + }); } - } catch { /* Legacy malformed snapshots cannot establish chat context. */ } + } catch { + /* Legacy malformed snapshots cannot establish chat context. */ + } } return enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, - matches: matches.map(match => ({ ...match, startObserved: Boolean(match.startObserved) })), + matches: matches.map((match) => ({ + ...match, + startObserved: Boolean(match.startObserved) + })), reports }); } export function matchChatTimeline(matchId: number): ContextualChatEntry[] { - return (db().prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid') - .all(matchId) as { entry: string }[]).map(row => JSON.parse(row.entry)); + return ( + db() + .prepare('SELECT entry FROM match_chat WHERE match_id = ? ORDER BY rowid') + .all(matchId) as { entry: string }[] + ).map((row) => JSON.parse(row.entry)); } /** Save before filtering the live view so completed matches retain their messages. */ @@ -86,15 +148,24 @@ function archiveChat(entries: ContextualChatEntry[]): void { })(); } -export function currentMatchChat(server: ServerRow, tail: TailResult | null): ContextualChatEntry[] { +export function currentMatchChat( + server: ServerRow, + tail: TailResult | null +): ContextualChatEntry[] { const entries = tail ? serverChatTimeline(server, tail) : []; archiveChat(entries); return currentChatEntries(server, entries); } -function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): ContextualChatEntry[] { - const match = db().prepare(`SELECT id, started_at AS startedAt FROM leaderboard_matches - WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`) +function currentChatEntries( + server: ServerRow, + entries: ContextualChatEntry[] +): ContextualChatEntry[] { + const match = db() + .prepare( + `SELECT id, started_at AS startedAt FROM leaderboard_matches + WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1` + ) .get(server.name) as { id: number; startedAt: string } | undefined; let startIndex = entries.length - 1; while (startIndex >= 0) { @@ -105,27 +176,44 @@ function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): const current = entries.slice(Math.max(0, startIndex)); if (!match) { // Servers without gameplay telemetry still have log-defined sessions. - return current.filter(entry => entry.context.realTime !== null - && Date.parse(entry.context.realTime) >= Date.parse(server.created_at)); + return current.filter( + (entry) => + entry.context.realTime !== null && + Date.parse(entry.context.realTime) >= Date.parse(server.created_at) + ); } const start = entries[startIndex]; // A log restart can arrive before game_start telemetry. Clear immediately // instead of falling back to messages from the previous recorded match. - if (start && !current.some(entry => entry.matchId === match.id) - && (!start.context.realTime || Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000)) return []; + if ( + start && + !current.some((entry) => entry.matchId === match.id) && + (!start.context.realTime || + Date.parse(start.context.realTime) > Date.parse(match.startedAt) + 1000) + ) + return []; return matchChatTimeline(match.id); } const pending = new Map>(); interface ChatReadState { - key: string; cursor?: LogCursor; lines: string[]; partial: Buffer; skipLine: boolean; - signature?: string; entries: ContextualChatEntry[]; contextVersion?: string; wasOccupied: boolean; + key: string; + cursor?: LogCursor; + lines: string[]; + partial: Buffer; + skipLine: boolean; + signature?: string; + entries: ContextualChatEntry[]; + contextVersion?: string; + wasOccupied: boolean; } const readers = new Map(); const failures = new Map(); /** Shared by the live API and the worker; collection does not require a viewer. */ -export function collectServerChat(server: ServerRow): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> { +export function collectServerChat( + server: ServerRow +): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> { const key = `${server.name}:${server.created_at}:${server.native_instance_id}:${server.instance_generation}`; const existing = pending.get(key); if (existing) return existing; @@ -135,12 +223,26 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context let saved = readers.get(server.name); if (saved?.key !== key) saved = undefined; // Publish the new cursor only after its messages have been processed. - const state: ChatReadState = saved ? { ...saved } : { key, lines: [], partial: Buffer.alloc(0), skipLine: false, entries: [], wasOccupied: false }; + const state: ChatReadState = saved + ? { ...saved } + : { + key, + lines: [], + partial: Buffer.alloc(0), + skipLine: false, + entries: [], + wasOccupied: false + }; const tail = await readLogChunk(path, state.cursor); const current = getServer(server.name); // A remote read may finish after destruction or replacement. - if (!current || current.destroyed_at || current.created_at !== server.created_at - || current.native_instance_id !== server.native_instance_id || current.instance_generation !== server.instance_generation) { + if ( + !current || + current.destroyed_at || + current.created_at !== server.created_at || + current.native_instance_id !== server.native_instance_id || + current.instance_generation !== server.instance_generation + ) { return { entries: [], missing: true }; } if (!tail) { @@ -149,22 +251,40 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context } let changed = false; if ('cursor' in tail) { - if (tail.reset) { state.lines = []; state.partial = Buffer.alloc(0); state.skipLine = tail.truncated; state.entries = []; } + if (tail.reset) { + state.lines = []; + state.partial = Buffer.alloc(0); + state.skipLine = tail.truncated; + state.entries = []; + } state.cursor = tail.cursor; const bytes = Buffer.concat([state.partial, Buffer.from(tail.contentBase64, 'base64')]); const end = bytes.lastIndexOf(10) + 1; if (end) { - const lines = bytes.subarray(0, end).toString('utf8').split('\n').slice(0, -1).map(line => line.replace(/\r$/, '')); + const lines = bytes + .subarray(0, end) + .toString('utf8') + .split('\n') + .slice(0, -1) + .map((line) => line.replace(/\r$/, '')); if (state.skipLine) lines.shift(); state.skipLine = false; state.lines = [...state.lines, ...lines].slice(-2000); - let length = 0, start = state.lines.length; - while (start > 0 && (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024) start--; + let length = 0, + start = state.lines.length; + while ( + start > 0 && + (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024 + ) + start--; state.lines = state.lines.slice(start); changed = true; } state.partial = Buffer.from(bytes.subarray(end)); - if (state.partial.length > 256 * 1024) { state.partial = Buffer.alloc(0); state.skipLine = true; } + if (state.partial.length > 256 * 1024) { + state.partial = Buffer.alloc(0); + state.skipLine = true; + } // File mtime anchors the final line only after the backlog has been read. if (tail.cursor.offset < tail.size || state.partial.length) { state.signature = undefined; @@ -177,9 +297,21 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context if (changed) state.lines = tail.lines; } const signature = `${tail.size}:${tail.mtime}`; - const needsContext = state.entries.some(entry => entry.type === 'message' && entry.matchId === undefined); - const contextVersion = needsContext ? (db().prepare('SELECT received_at FROM server_telemetry WHERE server=?').get(server.name) as { received_at: string } | undefined)?.received_at : undefined; - if (changed || state.signature === undefined || (needsContext && contextVersion !== state.contextVersion)) { + const needsContext = state.entries.some( + (entry) => entry.type === 'message' && entry.matchId === undefined + ); + const contextVersion = needsContext + ? ( + db() + .prepare('SELECT received_at FROM server_telemetry WHERE server=?') + .get(server.name) as { received_at: string } | undefined + )?.received_at + : undefined; + if ( + changed || + state.signature === undefined || + (needsContext && contextVersion !== state.contextVersion) + ) { state.entries = serverChatTimeline(current, { ...tail, lines: state.lines }); archiveChat(state.entries); state.contextVersion = contextVersion; @@ -193,15 +325,16 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context } export async function collectFleetChat(): Promise { - const hosts = new Map(listHosts().map(host => [host.id, host])); + const hosts = new Map(listHosts().map((host) => [host.id, host])); const observe = observationReader(); - const live = listServers().filter(server => !server.destroyed_at); - const names = new Set(live.map(server => server.name)); + const live = listServers().filter((server) => !server.destroyed_at); + const names = new Set(live.map((server) => server.name)); for (const name of readers.keys()) if (!names.has(name)) readers.delete(name); for (const name of failures.keys()) if (!names.has(name)) failures.delete(name); - const servers = live.filter(server => { + const servers = live.filter((server) => { if (server.host_id && !hosts.get(server.host_id)?.connected) return false; - if (server.container_state && !['running', 'unknown'].includes(server.container_state)) return false; + if (server.container_state && !['running', 'unknown'].includes(server.container_state)) + return false; const players = observe(server).last_players ?? server.last_players; // One final read after players leave retains messages written at match end. return players !== 0 || readers.get(server.name)?.wasOccupied; @@ -212,7 +345,9 @@ export async function collectFleetChat(): Promise { try { await collectServerChat(server); const state = readers.get(server.name); - if (state) state.wasOccupied = (observe(server).last_players ?? server.last_players ?? 0) > 0; + if (state) + state.wasOccupied = + (observe(server).last_players ?? server.last_players ?? 0) > 0; failures.delete(server.name); } catch (error) { if (Date.now() - (failures.get(server.name) ?? -Infinity) >= 5 * 60_000) { diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index a169a0a..674df07 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -1,7 +1,12 @@ import { scalingOwnershipBlocker } from '../autoscaling/ownership.js'; import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; -import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js'; +import type { + CollectionDetail, + CollectionMode, + CollectionPreview, + CollectionSummary +} from '$lib/deployment-plan.js'; import { db } from '../db/index.js'; import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js'; import { creationQueue, destroyServer } from '../provisioning/index.js'; @@ -18,97 +23,203 @@ import { canCancelCreation, cancelCreation } from '../provisioning/cancel-creati import { canCancelRemoval, cancelRemoval } from '../provisioning/history.js'; function coverage(entries: BlueprintEntry[], includeHealth = true) { - type Server = { name: string; game: string | null; map_slug: string | null; map: string | null }; - const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[]; - const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map + type Server = { + name: string; + game: string | null; + map_slug: string | null; + map: string | null; + }; + const registered = db() + .prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL') + .all() as Server[]; + const queued = db() + .prepare( + `SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[]; - const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server)); + WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')` + ) + .all() as Server[]; + const removing = new Set( + ( + db() + .prepare( + "SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')" + ) + .all() as { server: string }[] + ).map((row) => row.server) + ); // A provisioning server can appear in both tables; count its identity once. - const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()]; - return entries.map(entry => { - const existingServers = servers.filter(server => { - if (removing.has(server.name)) return false; - const parsed = parseServerName(server.name); - const game = server.game ?? parsed?.game; - if (game !== entry.recipe.game) return false; - if (server.map_slug) return server.map_slug === entry.recipe.mapSlug; - const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined; - if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug; - // Legacy canonical maps are usable only when they identify one variant. - const matches = zombieMaps(game).filter(map => map.canonical === server.map); - return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug; - }).map(server => server.name).sort(); - const coverage = includeHealth ? existingServers.map(name => { - const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined; - const server = getServer(name); - return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting' - : server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped' - : server ? serverObservation(server).health : 'offline', - href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` }; - }) : []; - return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) }; + const servers = [ + ...new Map([...queued, ...registered].map((server) => [server.name, server])).values() + ]; + return entries.map((entry) => { + const existingServers = servers + .filter((server) => { + if (removing.has(server.name)) return false; + const parsed = parseServerName(server.name); + const game = server.game ?? parsed?.game; + if (game !== entry.recipe.game) return false; + if (server.map_slug) return server.map_slug === entry.recipe.mapSlug; + const namedMap = parsed?.mapCanonical + ? zombieMapBySlug(game, parsed.mapSlug) + : undefined; + if (namedMap && (!server.map || server.map === namedMap.canonical)) + return namedMap.slug === entry.recipe.mapSlug; + // Legacy canonical maps are usable only when they identify one variant. + const matches = zombieMaps(game).filter((map) => map.canonical === server.map); + return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug; + }) + .map((server) => server.name) + .sort(); + const coverage = includeHealth + ? existingServers.map((name) => { + const operation = db() + .prepare( + "SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1" + ) + .get(name) as { id: string; state: string } | undefined; + const server = getServer(name); + return { + name, + health: operation + ? operation.state === 'queued' + ? 'queued' + : 'starting' + : server?.container_state && + ['stopped', 'exited', 'created', 'dead'].includes( + server.container_state + ) + ? 'stopped' + : server + ? serverObservation(server).health + : 'offline', + href: operation + ? `/provisioning/${encodeURIComponent(operation.id)}` + : `/servers/${encodeURIComponent(name)}` + }; + }) + : []; + return { + existingServers, + coverage, + existing: existingServers.length, + missing: Math.max(0, entry.count - existingServers.length) + }; }); } function fleetSnapshot() { - return housePartyTargets().map(name => ({ name, - server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null, - creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null, - removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null + return housePartyTargets().map((name) => ({ + name, + server: + db() + .prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?') + .get(name) ?? null, + creation: + db() + .prepare( + "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1" + ) + .get(name) ?? null, + removal: + db() + .prepare( + "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')" + ) + .get(name) ?? null })); } function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) { const counts = coverage(entries, includeHealth); // Preserve deployed lobbies before queued ones, then the lowest lobby IDs. - const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) => - Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true }) - ).slice(0, mode === 'exact' ? entries[index].count : undefined)); + const retainedServers = counts.flatMap((entry, index) => + [...entry.existingServers] + .sort( + (a, b) => + Number(!!getServer(b)) - Number(!!getServer(a)) || + a.localeCompare(b, 'en', { numeric: true }) + ) + .slice(0, mode === 'exact' ? entries[index].count : undefined) + ); const retained = new Set(retainedServers); - const matching = new Set(counts.flatMap(entry => entry.existingServers)); - const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name, - reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal' - })) : []; + const matching = new Set(counts.flatMap((entry) => entry.existingServers)); + const removals = + mode === 'exact' + ? housePartyTargets() + .filter((name) => !retained.has(name)) + .map((name) => ({ + name, + reason: matching.has(name) + ? 'Exceeds the target lobby count' + : 'Outside this collection or already awaiting removal' + })) + : []; return { counts, retainedServers, removals }; } -function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] { - const occupied = db().prepare(`SELECT name FROM servers UNION +function allocate( + entries: BlueprintEntry[], + counts = coverage(entries) +): CollectionPreview['servers'] { + const occupied = db() + .prepare( + `SELECT name FROM servers UNION SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION SELECT server_id AS name FROM server_key_leases UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health') - OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`) + OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))` + ) .all() as { name: string }[]; const lobbies = new Map>(); for (const { name } of occupied) { const match = /^(.*)_(\d+)$/.exec(name); if (!match) continue; const taken = lobbies.get(match[1]) ?? new Set(); - taken.add(Number(match[2])); lobbies.set(match[1], taken); + taken.add(Number(match[2])); + lobbies.set(match[1], taken); } const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') }; - return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => { - const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; - const taken = lobbies.get(base) ?? new Set(); - const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null; - const name = lobby === null ? null : `${base}_${lobby}`; - const native = entry.recipe.native; - const port = null; - if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); } - const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0; - if (hasSavedKey) availableKeys[entry.recipe.game]--; - return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey }; - })); + return entries.flatMap((entry, entryIndex) => + Array.from({ length: counts[entryIndex].missing }, () => { + const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; + const taken = lobbies.get(base) ?? new Set(); + const lobby = + Array.from({ length: 20 }, (_, index) => index + 1).find( + (number) => !taken.has(number) + ) ?? null; + const name = lobby === null ? null : `${base}_${lobby}`; + const native = entry.recipe.native; + const port = null; + if (lobby !== null) { + taken.add(lobby); + lobbies.set(base, taken); + } + const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0; + if (hasSavedKey) availableKeys[entry.recipe.game]--; + return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey }; + }) + ); } -function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] { +function blockersFor( + servers: CollectionPreview['servers'], + entries: BlueprintEntry[] = [] +): string[] { const blockers: string[] = []; - for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); } + for (const server of servers) + if (server.name) { + const owned = scalingOwnershipBlocker(server.name); + if (owned) blockers.push(owned); + } if (!servers.length) return blockers; - if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.'); - if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); + if (servers.some((server) => !server.native && !entries[server.entryIndex]?.recipe.native)) + blockers.push('Select and prepare a host before launching.'); + if (servers.some((server) => !server.name)) + blockers.push( + 'There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.' + ); return blockers; } @@ -116,11 +227,16 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] { const blockers: string[] = []; const native = (name: string) => { if (nativeInstance(name)) return true; - const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL - AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined; + const creation = db() + .prepare( + `SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL + AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1` + ) + .get(name) as { id: string } | undefined; return !!creation && nativeDeployment(creation.id); }; - if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.'); + if (removals.some((server) => !native(server.name))) + blockers.push('Every server to remove must have a native host instance.'); for (const server of removals) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); @@ -130,50 +246,118 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] { return blockers; } -export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record): CollectionPreview { - if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.'); - if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); +export function previewCollection( + blueprintId: unknown, + mode: unknown = 'fill', + selection?: Record +): CollectionPreview { + if (typeof blueprintId !== 'string' || !blueprintId) + throw new Error('Choose a collection to launch.'); + if (mode !== 'fill' && mode !== 'exact') + throw new Error('Choose fill or exact collection mode.'); return db().transaction((): CollectionPreview => { const { entries, blueprintName } = blueprintEntries(blueprintId); const libraries = selection?.libraryIds as Record | undefined; for (const entry of entries) { - entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); + entry.recipe.native = nativeLaunchSelection( + { ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, + entry.recipe.game + ); entry.recipe.image = entry.recipe.native.image!; } const { counts, retainedServers, removals } = collectionPlan(entries, mode); const servers = allocate(entries, counts); const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; - const selections = servers.flatMap(server => server.native ? [server.native] : []); + const selections = servers.flatMap((server) => (server.native ? [server.native] : [])); const availability = selections.length ? launchAvailability(selections) : {}; const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); - db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString()); - db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)') - .run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null); - return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0, - entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) }; + db() + .prepare( + 'DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL' + ) + .run(new Date().toISOString()); + db() + .prepare( + 'INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)' + ) + .run( + id, + expiresAt, + blueprintName, + JSON.stringify(entries), + JSON.stringify(servers), + mode, + mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null + ); + return { + id, + expiresAt, + blueprintName, + mode, + retainedServers, + removals, + launchConcurrency: fleetSettings().createConcurrency, + servers, + blockers, + ...availability, + ready: blockers.length === 0, + entries: entries.map((entry, index) => ({ + count: entry.count, + ...counts[index], + recipe: summarizeRecipe(entry.recipe) + })) + }; })(); } /** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */ -export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } { - if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.'); - if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.'); +export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { + ok: true; + collectionId: string | null; +} { + if (typeof input.previewId !== 'string' || !input.previewId) + throw new Error('Review a collection preview before launching.'); + if ( + input.keys !== undefined && + (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys)) + ) + throw new Error('Provide keys indexed by the exact server ID.'); const keys = (input.keys ?? {}) as Record; return db().transaction(() => { - const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as { - id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null; - } | undefined; - if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.'); - if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id }; - if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.'); + const preview = db() + .prepare('SELECT * FROM server_collection_previews WHERE id = ?') + .get(input.previewId) as + | { + id: string; + expires_at: string; + blueprint_name: string; + entries: string; + servers: string; + collection_id: string | null; + mode: CollectionMode; + fleet_snapshot: string | null; + } + | undefined; + if (!preview) + throw new Error('Collection preview expired or was not found. Refresh the preview.'); + if (preview.collection_id) + return { ok: true as const, collectionId: preview.collection_id }; + if (preview.expires_at <= new Date().toISOString()) + throw new Error('Collection preview expired. Refresh it before launching.'); const entries: BlueprintEntry[] = JSON.parse(preview.entries); // Older previews embedded plugin bytes. Queued jobs need only the // reviewed metadata; provisioning captures the current files at dispatch. - for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file); + for (const entry of entries) + entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file); const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers); - if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) { - throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'); + if ( + preview.mode === 'exact' && + preview.fleet_snapshot !== JSON.stringify(fleetSnapshot()) + ) { + throw new Error( + 'The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.' + ); } const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false); const servers = reviewed; @@ -181,7 +365,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): for (const server of servers) { const recipe = entries[server.entryIndex].recipe; const selection = server.native ?? recipe.native; - if (!selection) throw new Error('Select a native host before launching. Refresh the preview.'); + if (!selection) + throw new Error('Select a native host before launching. Refresh the preview.'); const identity = JSON.stringify([recipe.game, selection]); if (validatedSelections.has(identity)) continue; // Recheck saved configuration once per selection without allocating @@ -191,50 +376,102 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): } const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); - if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) { - throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.'); + if ( + entries.some( + (_entry, index) => + counts[index].missing !== + reviewed.filter((server) => server.entryIndex === index).length + ) + ) { + throw new Error( + 'Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.' + ); } - const expected = new Set(servers.map(server => server.name)); + const expected = new Set(servers.map((server) => server.name)); for (const [name, value] of Object.entries(keys)) { - if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.'); + if (!expected.has(name) || typeof value !== 'string') + throw new Error('Keys must be strings for server IDs shown in this preview.'); } if (!servers.length && !removals.length) return { ok: true as const, collectionId: null }; const collectionId = randomUUID(); - db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)') - .run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers)); + db() + .prepare( + 'INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)' + ) + .run( + collectionId, + preview.blueprint_name, + new Date().toISOString(), + preview.mode, + JSON.stringify(retainedServers) + ); removals.forEach((server, position) => { - const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined; + const queued = db() + .prepare( + "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'" + ) + .get(server.name) as { id: string } | undefined; let id: string; if (queued && !getServer(server.name)) { // No container or files exist yet: cancel the reservation instead of booting just to remove it. const result = destroyServer(server.name, 'blueprint-exact'); if (!result.ok) throw new Error(result.message); - if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); + if (!result.deploymentId) + throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; - db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id); - recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.'); + db() + .prepare( + "UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?" + ) + .run(new Date().toISOString(), queued.id); + recordDeploymentEvent( + queued.id, + 'Queued creation cancelled by exact collection; saved key retained.' + ); deploymentStage(id, 'Cancelled queued creation; no server was started'); finishDeployment(id, { ok: true }); } else { const result = destroyServer(server.name, 'blueprint-exact'); if (!result.ok) throw new Error(result.message); - if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); + if (!result.deploymentId) + throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; } - db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position); + db() + .prepare( + 'INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)' + ) + .run(collectionId, id, position); }); servers.forEach((server, index) => { - const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) }; - if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.'); - const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug, - selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name }); + const recipe = { + ...entries[server.entryIndex].recipe, + ...(server.native ? { native: server.native } : {}) + }; + if (!recipe.native) + throw new Error('Select a native host before launching. Refresh the preview.'); + const result = queueNativeCreationIntent({ + server: server.name!, + game: recipe.game, + mapSlug: recipe.mapSlug, + selection: recipe.native, + recipe, + plutoKey: keys[server.name!] as string | undefined, + createdBy: 'ui', + blueprintName: preview.blueprint_name + }); // Throw so the outer transaction also reverses all earlier keys and children. - if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`); - db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)') + if (!result.ok) + throw new Error(`${server.name}: ${result.message} Nothing was queued.`); + db() + .prepare( + 'INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)' + ) .run(collectionId, result.deploymentId, index); - }); - db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id); + db() + .prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?') + .run(collectionId, preview.id); return { ok: true as const, collectionId }; })(); } @@ -243,39 +480,60 @@ const membershipSql = `SELECT collection_id, deployment_id, position FROM server UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`; function collectionCancellationCandidates(id: string) { - const creations = (db().prepare(`SELECT d.id FROM server_collection_members m + const creations = ( + db() + .prepare( + `SELECT d.id FROM server_collection_members m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.collection_id=? - AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)`) - .all(id) as { id: string }[]).filter(job => canCancelCreation(job.id)); + AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)` + ) + .all(id) as { id: string }[] + ).filter((job) => canCancelCreation(job.id)); // Exact launches can reuse manual removals or creation cleanup jobs. Membership // alone must never give a launch permission to withdraw those operations. - const removals = (db().prepare(`SELECT d.id FROM server_collection_removals m + const removals = ( + db() + .prepare( + `SELECT d.id FROM server_collection_removals m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.collection_id=? AND d.triggered_by='blueprint-exact' AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.operation_id IS NOT NULL) - AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)`) - .all(id) as { id: string }[]).filter(job => canCancelRemoval(job.id)); + AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)` + ) + .all(id) as { id: string }[] + ).filter((job) => canCancelRemoval(job.id)); return { creations, removals }; } /** Withdraw the remaining launch atomically; partial instances keep durable cleanup jobs. */ export function cancelCollection(id: string): boolean { return db().transaction(() => { - const collection = db().prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?').get(id) as { cancelled_at: string | null } | undefined; + const collection = db() + .prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?') + .get(id) as { cancelled_at: string | null } | undefined; if (!collection) return false; if (collection.cancelled_at) return true; const { creations, removals } = collectionCancellationCandidates(id); if (!creations.length && !removals.length) return false; - db().prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?').run(new Date().toISOString(), id); + db() + .prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?') + .run(new Date().toISOString(), id); for (const creation of creations) { - if (!cancelCreation(creation.id, 'collection-cancel')) throw new Error('Could not cancel a collection creation. Refresh the launch and try again.'); + if (!cancelCreation(creation.id, 'collection-cancel')) + throw new Error( + 'Could not cancel a collection creation. Refresh the launch and try again.' + ); } for (const removal of removals) { // A shared removal remains requested until every owning launch withdraws. - const shared = db().prepare(`SELECT 1 FROM server_collection_removals m + const shared = db() + .prepare( + `SELECT 1 FROM server_collection_removals m JOIN server_launch_collections c ON c.id=m.collection_id - WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1`).get(removal.id, id); + WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1` + ) + .get(removal.id, id); if (!shared) cancelRemoval(removal.id); } return true; @@ -299,26 +557,59 @@ const summarySql = `SELECT c.id, c.name, c.created_at, c.cancelled_at, c.mode, C LEFT JOIN server_deployments cleanup ON cleanup.id = cc.cleanup_id`; export function listCollections(): CollectionSummary[] { - return db().prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC').all() as CollectionSummary[]; + return db() + .prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC') + .all() as CollectionSummary[]; } export function getCollection(id: string): CollectionDetail | null { - const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined; + const collection = db() + .prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id') + .get(id) as CollectionSummary | undefined; if (!collection) return null; - const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug, + const servers = db() + .prepare( + `SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id - LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers']; + LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position` + ) + .all(id) as CollectionDetail['servers']; const queue = creationQueue(); const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1])); - const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string }; - const cancellations = db().prepare(`SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error + const stored = db() + .prepare( + 'SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?' + ) + .get(id) as { retained_servers: string; retained_server_deletions: string }; + const cancellations = db() + .prepare( + `SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error FROM server_collection_members m JOIN native_creation_cancellations cc ON cc.creation_id=m.deployment_id - JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?`).all(id) as - (NonNullable & { creation_id: string })[]; - const cleanupByCreation = new Map(cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup])); + JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?` + ) + .all(id) as (NonNullable & { + creation_id: string; + })[]; + const cleanupByCreation = new Map( + cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup]) + ); const candidates = collection.cancelled_at ? null : collectionCancellationCandidates(id); - return { collection, canCancel: !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0), - retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length }, - servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null, cancellation: cleanupByCreation.get(server.id) ?? null })) }; + return { + collection, + canCancel: + !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0), + retainedServers: JSON.parse(stored.retained_servers), + retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), + queue: { + concurrency: fleetSettings().createConcurrency, + active: queue.active, + queued: queue.queued.length + }, + servers: servers.map((server) => ({ + ...server, + queuePosition: positions.get(server.id) ?? null, + cancellation: cleanupByCreation.get(server.id) ?? null + })) + }; } diff --git a/src/lib/server/fleet/config-inspect.ts b/src/lib/server/fleet/config-inspect.ts index bf1f39c..bab4bbd 100644 --- a/src/lib/server/fleet/config-inspect.ts +++ b/src/lib/server/fleet/config-inspect.ts @@ -1,62 +1,115 @@ import { composeServerCfg, parseCfg } from './config-compose.js'; import { selectedZombieMap, mapRotation } from './map-aliases.js'; -import { configWarnings, type ConfigAssignment, type ConfigInspection, type ConfigSource } from '../../config-workbench.js'; +import { + configWarnings, + type ConfigAssignment, + type ConfigInspection, + type ConfigSource +} from '../../config-workbench.js'; -export interface ConfigLayer { source: ConfigSource; text: string; } +export interface ConfigLayer { + source: ConfigSource; + text: string; +} /** Trace file composition, including assignments left in place by the composer. * This is an intended-file preview, not a query of live runtime dvars. */ -export function inspectConfig(baseline: ConfigLayer, overlays: ConfigLayer[], game: string | null, map: string | null, mapSlug?: string | null): ConfigInspection { +export function inspectConfig( + baseline: ConfigLayer, + overlays: ConfigLayer[], + game: string | null, + map: string | null, + mapSlug?: string | null +): ConfigInspection { const sorted = [...overlays].sort((a, b) => a.source.id! - b.source.id!); const parsed = parseCfg(baseline.text); type TracedLine = { line: string; history: ConfigAssignment[] }; function assignment(line: string, source: ConfigSource, index: number): ConfigAssignment { - const match = /^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line) - ?? /^\s*(sv_maprotation)\s+(.*)$/i.exec(line); + const match = + /^\s*(?:(?:set|seta)\s+)(\S+)\s+(.*)$/i.exec(line) ?? + /^\s*(sv_maprotation)\s+(.*)$/i.exec(line); return { value: match?.[2] ?? '', source, line: index + 1 }; } const lines: TracedLine[] = parsed.order.map((entry, index) => ({ - line: entry.line, history: [assignment(entry.line, baseline.source, index)] + line: entry.line, + history: [assignment(entry.line, baseline.source, index)] })); const replacements = new Map(); - const warnings = configWarnings(baseline.text, false).map(w => `${baseline.source.name}: ${w}`); + const warnings = configWarnings(baseline.text, false).map( + (w) => `${baseline.source.name}: ${w}` + ); for (const overlay of sorted) { - warnings.push(...configWarnings(overlay.text, true).map(w => `${overlay.source.name}: ${w}`)); + warnings.push( + ...configWarnings(overlay.text, true).map((w) => `${overlay.source.name}: ${w}`) + ); overlay.text.split(/\r?\n/).forEach((line, index) => { const match = /^\s*set\s+(\S+)\s+/.exec(line); if (!match) return; const previous = replacements.get(match[1]); - replacements.set(match[1], { line, history: [...(previous?.history ?? []), assignment(line, overlay.source, index)] }); + replacements.set(match[1], { + line, + history: [...(previous?.history ?? []), assignment(line, overlay.source, index)] + }); }); } for (const [key, replacement] of replacements) { const index = parsed.entries.get(key); if (index === undefined) lines.push(replacement); - else lines[index] = { line: replacement.line, history: [...lines[index].history, ...replacement.history] }; + else + lines[index] = { + line: replacement.line, + history: [...lines[index].history, ...replacement.history] + }; } - const sources = [baseline.source, ...sorted.map(o => o.source)]; + const sources = [baseline.source, ...sorted.map((o) => o.source)]; const selected = selectedZombieMap(game, map, mapSlug); if (selected) { - const source: ConfigSource = { id: null, name: 'Server map selection', version: null, kind: 'map' }; + const source: ConfigSource = { + id: null, + name: 'Server map selection', + version: null, + kind: 'map' + }; sources.push(source); const rotation = /^\s*(?:(?:set|seta)\s+)?sv_maprotation\s+/i; - const history = lines.filter(entry => rotation.test(entry.line)).flatMap(entry => entry.history); - for (let i = lines.length - 1; i >= 0; i--) if (rotation.test(lines[i].line)) lines.splice(i, 1); + const history = lines + .filter((entry) => rotation.test(entry.line)) + .flatMap((entry) => entry.history); + for (let i = lines.length - 1; i >= 0; i--) + if (rotation.test(lines[i].line)) lines.splice(i, 1); const line = `set sv_maprotation "${mapRotation(selected)}"`; lines.push({ line, history: [...history, assignment(line, source, 0)] }); } const byKey = new Map(); for (const entry of lines) { - const match = /^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ?? /^\s*(sv_maprotation)\s+/i.exec(entry.line); + const match = + /^\s*(?:set|seta)\s+(\S+)\s+/i.exec(entry.line) ?? + /^\s*(sv_maprotation)\s+/i.exec(entry.line); if (!match) continue; const normalized = match[1].toLowerCase(); const existing = byKey.get(normalized); - byKey.set(normalized, { key: match[1], history: [...(existing?.history ?? []), ...entry.history] }); + byKey.set(normalized, { + key: match[1], + history: [...(existing?.history ?? []), ...entry.history] + }); } return { - text: composeServerCfg(baseline.text, sorted.map(o => ({ id: o.source.id!, text: o.text })), game, map, mapSlug), - sources, warnings, - settings: [...byKey.values()].map(({ key, history }) => ({ key, history, value: history.at(-1)!.value, source: history.at(-1)!.source })) + text: composeServerCfg( + baseline.text, + sorted.map((o) => ({ id: o.source.id!, text: o.text })), + game, + map, + mapSlug + ), + sources, + warnings, + settings: [...byKey.values()] + .map(({ key, history }) => ({ + key, + history, + value: history.at(-1)!.value, + source: history.at(-1)!.source + })) .sort((a, b) => a.key.localeCompare(b.key)) }; } diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index 1e125d8..58f0895 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -1,6 +1,12 @@ import { putFileBlob } from '../db/file-blobs.js'; import { getVersionContent } from './scripts.js'; -import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js'; +import { + readdir, + readFile, + stat, + nativeFileManifest, + type FileFingerprint +} from '../backends/files.js'; import { randomUUID } from 'node:crypto'; import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js'; import { join, posix, relative, sep } from 'node:path'; @@ -27,12 +33,18 @@ async function scanBatch(items: readonly T[], work: (item: T) => Promise result.status === 'rejected'); + const workers = await Promise.allSettled( + Array.from({ length: Math.min(4, items.length) }, pump) + ); + const failure = workers.find((result) => result.status === 'rejected'); if (failure?.status === 'rejected') throw failure.reason; return results; } @@ -73,15 +85,22 @@ function toPosix(p: string): string { } function hasActiveBeta(server: string): boolean { - return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server); + return !!db() + .prepare( + `SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1` + ) + .get(server); } async function requireServerDirectory(server: string): Promise { try { - if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory'); + if (!(await stat(serverDir(server))).isDirectory()) + throw new Error('path is not a directory'); } catch (error) { - throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`); + throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`, { + cause: error + }); } } @@ -92,7 +111,10 @@ async function readExistingFile(path: string): Promise { }); } -async function fingerprint(path: string, manifest: Map | null): Promise { +async function fingerprint( + path: string, + manifest: Map | null +): Promise { if (manifest) return manifest.get(path) ?? null; const content = await readExistingFile(path); return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) }; @@ -102,10 +124,11 @@ async function walkFiles(root: string): Promise { const out: string[] = []; let directories = [root]; while (directories.length) { - const children = await scanBatch(directories, async dir => { + const children = await scanBatch(directories, async (dir) => { let entries; - try { entries = await readdir(dir, { withFileTypes: true }); } - catch (error) { + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; } @@ -131,51 +154,83 @@ async function walkFiles(root: string): Promise { * artifacts with the promoted versions. */ export async function scanServerDrift(server: string): Promise { - return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => scanServerFiles(server)); + return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => + scanServerFiles(server) + ); } async function scanServerFiles(server: string): Promise { const srv = getServer(server); if (!srv || srv.destroyed_at || hasActiveBeta(server)) return []; // A missing mount is unavailable evidence, not an empty managed tree. - const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path)); - if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server))); - if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE)); - const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths); + const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => + join(destOfKind(server, kind), script.target_rel_path) + ); + if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) + expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server))); + if ( + srv.created_by && + srv.game_mode === 'zombie' && + getServerMapConfig(srv.game, srv.map, srv.map_slug) + ) + expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE)); + const manifest = await nativeFileManifest( + serverDir(server), + DRIFT_DESTS.map((kind) => destOfKind(server, kind)), + [...DRIFT_IGNORE_FILENAMES], + expectedPaths + ); if (!manifest) await requireServerDirectory(server); const drift = await scanManagedForServer(server, manifest); // Previous inventory is an audit of installed bytes, not permission to // retain artifacts whose scope was moved or removed from the library. - const managed = new Set(promotedArtifactMetadata(srv.game, srv.map) - .map(({ kind, script }) => `${kind}:${script.target_rel_path}`)); + const managed = new Set( + promotedArtifactMetadata(srv.game, srv.map).map( + ({ kind, script }) => `${kind}:${script.target_rel_path}` + ) + ); const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined; if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`); // Without a promoted baseline, Plutainer owns the seeded config. Keep it // until a baseline is promoted; startup arguments handle map selection. - const keepDefaultConfig = srv?.game - && selectedZombieMap(srv.game, srv.map, srv.map_slug) - && !baseline?.current_version_id; + const keepDefaultConfig = + srv?.game && + selectedZombieMap(srv.game, srv.map, srv.map_slug) && + !baseline?.current_version_id; // Scan for untracked files only after resolving managed targets. for (const kind of DRIFT_DESTS) { const dest = destOfKind(server, kind); const onDisk = manifest - ? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path))) - .filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort() + ? [...manifest.keys()] + .filter((path) => path.startsWith(`${dest}${sep}`)) + .map((path) => toPosix(relative(dest, path))) + .filter( + (path) => !path.split('/').some((name) => DRIFT_IGNORE_FILENAMES.has(name)) + ) + .sort() : await walkFiles(dest); if (onDisk.length === 0) continue; const untracked = await scanBatch(onDisk, async (relPath): Promise => { if (managed.has(`${kind}:${relPath}`)) return null; - if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null; - if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by - && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null; + if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) + return null; + if ( + kind === 'config' && + relPath === MAP_CONFIG_FILE && + srv?.created_by && + srv.game_mode === 'zombie' && + getServerMapConfig(srv.game, srv.map, srv.map_slug) + ) + return null; const abs = join(dest, relPath); try { const info = await fingerprint(abs, manifest); if (!info) return null; return { - server, relPath, + server, + relPath, ...info, kind, driftKind: 'untracked' @@ -196,38 +251,63 @@ async function scanServerFiles(server: string): Promise { // Every promoted artifact whose scope matches this server's game/map should // have its target file present on disk. Anything absent is 'missing' drift. // Configs are handled as a composed layer (baseline + overlays → one cfg). -async function scanManagedForServer(server: string, manifest: Map | null): Promise { +async function scanManagedForServer( + server: string, + manifest: Map | null +): Promise { const srv = getServer(server); if (!srv) return []; const out: DriftEntry[] = []; // Provisioned servers load this generated file after their main cfg. It // remains managed even when there is no promoted baseline yet. - const mapConfig = srv.created_by && srv.game_mode === 'zombie' - ? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null; + const mapConfig = + srv.created_by && srv.game_mode === 'zombie' + ? getServerMapConfig(srv.game, srv.map, srv.map_slug) + : null; if (mapConfig) { const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE); const content = await fingerprint(abs, manifest); if (content?.sha256 !== sha256Buffer(mapConfig)) { - out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing', - size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) }); + out.push({ + server, + relPath: MAP_CONFIG_FILE, + kind: 'config', + driftKind: content ? 'replace' : 'missing', + size: mapConfig.byteLength, + sha256: sha256Buffer(mapConfig) + }); } } // Inventory records the last managed write, not the current disk bytes. // Check both so edits and outdated versions are detected on tracked files. - const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise => { - const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest); - const inventory = getServerScript(server, script.target_rel_path); - if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256 - && inventory.script_version_id === version.id) return null; - return { - server, relPath: script.target_rel_path, kind, - driftKind: content ? 'replace' : 'missing', - size: version.size, sha256: version.sha256, - scriptId: script.id, scriptVersionId: version.id - }; - }); + const managed = await scanBatch( + promotedArtifactMetadata(srv.game, srv.map), + async ({ kind, script, version }): Promise => { + const content = await fingerprint( + join(destOfKind(server, kind), script.target_rel_path), + manifest + ); + const inventory = getServerScript(server, script.target_rel_path); + if ( + content?.sha256 === version.sha256 && + inventory?.sha256 === version.sha256 && + inventory.script_version_id === version.id + ) + return null; + return { + server, + relPath: script.target_rel_path, + kind, + driftKind: content ? 'replace' : 'missing', + size: version.size, + sha256: version.sha256, + scriptId: script.id, + scriptVersionId: version.id + }; + } + ); out.push(...managed.filter((entry): entry is DriftEntry => entry !== null)); // Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted @@ -247,13 +327,24 @@ async function scanManagedForServer(server: string, manifest: Map { } export interface RunDriftInput { - servers?: string[]; // if omitted, scans the whole fleet + servers?: string[]; // if omitted, scans the whole fleet dryRun?: boolean; triggeredBy?: 'ui' | 'sweep' | 'cron'; expectedFingerprint?: string; } export class DriftPlanChanged extends Error { - constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); } + constructor(public plan: DriftPlan) { + super( + 'The files or target servers changed. Nothing was changed. Review the updated plan before continuing.' + ); + } } function driftTargets(servers?: string[]): string[] { - const registered = listServers().map(server => server.name); - if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.'); + const registered = listServers().map((server) => server.name); + if (servers && (!servers.length || servers.some((name) => !registered.includes(name)))) + throw new Error( + 'Select at least one registered server. Refresh the fleet if a server was removed.' + ); return [...new Set(servers ?? registered)].sort(); } async function describePlan(servers: string[], entries: DriftEntry[]): Promise { const files: DriftPlanFile[] = []; for (const entry of entries) { - const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => { + const before = await readFile( + join(destOfKind(entry.server, entry.kind), entry.relPath) + ).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); - const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry); - files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind, - beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null, - beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null }); + const after = + entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry); + files.push({ + server: entry.server, + relPath: entry.relPath, + kind: entry.kind, + driftKind: entry.driftKind, + beforeSha256: before ? sha256Buffer(before) : null, + afterSha256: after ? sha256Buffer(after) : null, + beforeSize: before?.byteLength ?? null, + afterSize: after?.byteLength ?? null + }); } return buildPlan(servers, files); } function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan { - files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`)); - return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) }; + files.sort((a, b) => + `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`) + ); + return { + servers, + files, + scannedAt: new Date().toISOString(), + fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) + }; } export async function previewDrift(servers?: string[]): Promise { const targets = driftTargets(servers); const entries: DriftEntry[] = []; - for (const server of targets) entries.push(...await scanServerDrift(server)); + for (const server of targets) entries.push(...(await scanServerDrift(server))); return describePlan(targets, entries); } @@ -373,7 +491,10 @@ export interface RunDriftResult { } function deploymentIdNow(): string { - const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); + const stamp = new Date() + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d+Z$/, 'Z'); return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`; } @@ -386,47 +507,103 @@ export interface PreparedDriftCampaign { export async function prepareDriftCampaign(server: string): Promise { const servers = driftTargets([server]); const entries = await scanServerDrift(server); - const files = await scanBatch(entries, async entry => { + const files = await scanBatch(entries, async (entry) => { const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath)); - const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry); - if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`); - return { before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind, - beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null, - beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } }; + const after = + entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry); + if (entry.driftKind !== 'untracked' && !after) + throw new Error(`Cannot resolve promoted content for ${entry.relPath}`); + return { + before, + after, + file: { + server, + relPath: entry.relPath, + kind: entry.kind, + driftKind: entry.driftKind, + beforeSha256: before ? sha256Buffer(before) : null, + afterSha256: after ? sha256Buffer(after) : null, + beforeSize: before?.byteLength ?? null, + afterSize: after?.byteLength ?? null + } + }; }); if (entries.length) await requireServerDirectory(server); - if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred'); - return { plan: buildPlan(servers, files.map(({ file }) => file)), files }; + if (hasActiveBeta(server)) + throw new Error('Active beta owns this server; reconciliation deferred'); + return { + plan: buildPlan( + servers, + files.map(({ file }) => file) + ), + files + }; } /** Record the audited plan atomically with a queued rebuild, never a second restart. */ export function createDriftRebuildCampaign( - server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign + server: string, + operationId: string, + incidentId: string | number, + prepared: PreparedDriftCampaign ): string | null { if (!prepared.files.length) return null; - if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server - || prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server'); - if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred'); + if ( + prepared.plan.servers.length !== 1 || + prepared.plan.servers[0] !== server || + prepared.files.some(({ file }) => file.server !== server) + ) + throw new Error('Drift evidence belongs to another server'); + if (hasActiveBeta(server)) + throw new Error('Active beta owns this server; reconciliation deferred'); return db().transaction(() => { - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) { + if ( + !db() + .prepare( + "SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'" + ) + .get(operationId, server) + ) { throw new Error('Drift campaign requires the matching rebuild operation'); } const id = deploymentIdNow(); - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')") + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')" + ) .run(id, new Date().toISOString()); const insertFile = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`); for (const { file, before, after } of prepared.files) { - insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated', - file.relPath, file.beforeSha256, file.afterSha256, putFileBlob(before), putFileBlob(after)); + insertFile.run( + id, + server, + after === null ? 'removed' : before === null ? 'added' : 'updated', + file.relPath, + file.beforeSha256, + file.afterSha256, + putFileBlob(before), + putFileBlob(after) + ); } - db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id) - VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId); - db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)") + db() + .prepare( + `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id) + VALUES (?, ?, ?, 'pending', ?)` + ) + .run(id, server, prepared.files.length, operationId); + db() + .prepare( + "INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)" + ) .run(operationId, id, server); - db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') - .run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`); + db() + .prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') + .run( + id, + `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.` + ); emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] }); return id; })(); @@ -455,7 +632,12 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise server.entries)) : null; + const plan = input.expectedFingerprint + ? await describePlan( + scanTargets, + perServer.flatMap((server) => server.entries) + ) + : null; if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan); const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0); if (totalDrift === 0 || dryRun) { @@ -464,7 +646,9 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise ({ server: s.server, removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath), - restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath), + restored: s.entries + .filter((e) => e.driftKind !== 'untracked') + .map((e) => e.relPath), skipped: [] })), totalRemoved: perServer.reduce( @@ -484,9 +668,11 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath); + const reviewed = plan?.files.find( + (file) => + file.server === s.server && + file.kind === entry.kind && + file.relPath === entry.relPath + ); if (reviewed) { - const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); + const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return null; + throw error; + }); if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) { - s.skipped.push(`${entry.relPath}: changed after review; scan and review again`); + s.skipped.push( + `${entry.relPath}: changed after review; scan and review again` + ); continue; } } if (entry.driftKind === 'untracked') { // Capture the reviewed deletion without touching the live tree. const content = await readFile(abs); - if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred'); + if (hasActiveBeta(s.server)) + throw new Error('active beta owns this server; reconciliation deferred'); db().transaction(() => { - const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, putFileBlob(content), null); - queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, - kind: entry.kind, relPath: entry.relPath, content: null, - inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), - expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) }); + const file = insertFile.run( + depId, + s.server, + 'removed', + entry.relPath, + sha256Buffer(content), + null, + putFileBlob(content), + null + ); + queueFileChange({ + sourceKind: 'campaign', + sourceId: depId, + server: s.server, + kind: entry.kind, + relPath: entry.relPath, + content: null, + inventorySource: `drift:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), + expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) + }); })(); s.removed.push(entry.relPath); } else { // Persist exact replacement bytes so waiting cannot change the plan. const written = await resolveManagedEntry(s.server, entry); if (!written) { - s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`); + s.skipped.push( + `${entry.relPath}: missing artifact could not be resolved from the DB` + ); continue; } if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) { - s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`); + s.skipped.push( + `${entry.relPath}: promoted content changed after review; scan and review again` + ); continue; } const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => { if (err.code === 'ENOENT') return null; throw err; }); - if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred'); + if (hasActiveBeta(s.server)) + throw new Error('active beta owns this server; reconciliation deferred'); db().transaction(() => { - const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath, - before ? sha256Buffer(before) : null, sha256Buffer(written), putFileBlob(before), putFileBlob(written)); - queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, - kind: entry.kind, relPath: entry.relPath, content: written, - scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null, - inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), - expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null }); + const file = insertFile.run( + depId, + s.server, + before ? 'updated' : 'added', + entry.relPath, + before ? sha256Buffer(before) : null, + sha256Buffer(written), + putFileBlob(before), + putFileBlob(written) + ); + queueFileChange({ + sourceKind: 'campaign', + sourceId: depId, + server: s.server, + kind: entry.kind, + relPath: entry.relPath, + content: written, + scriptVersionId: + entry.kind === 'config' ? null : (entry.scriptVersionId ?? null), + inventorySource: `drift:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), + expectedBeforeSha: reviewed + ? reviewed.beforeSha256 + : before + ? sha256Buffer(before) + : null + }); })(); s.restored.push(entry.relPath); } @@ -579,7 +819,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise ({ - server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped + server: s.server, + removed: s.removed, + restored: s.restored, + skipped: s.skipped })), totalRemoved: 0, totalRestored: 0, @@ -598,7 +841,10 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise ({ - server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped + server: s.server, + removed: s.removed, + restored: s.restored, + skipped: s.skipped })), totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0), totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0), diff --git a/src/lib/server/fleet/health-observations.ts b/src/lib/server/fleet/health-observations.ts index 4509006..89699ea 100644 --- a/src/lib/server/fleet/health-observations.ts +++ b/src/lib/server/fleet/health-observations.ts @@ -4,7 +4,11 @@ import { db } from '../db/index.js'; import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js'; import { nativeInstance, type NativeInstance } from '../backends/native.js'; import { nativeHealth } from '../backends/native.js'; -import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js'; +import { + rconStatusResult, + agentRconStatusResult, + type AgentRconReply +} from '../integrations/rcon.js'; import { withOperationSource } from '../hosts/operation-context.js'; import { getServer } from './servers.js'; import { latestGameSubmission } from './lifecycle-evidence.js'; @@ -14,8 +18,15 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js'; const MAX_AGE_MS = 30_000; const MAX_SERVERS = 1024; type Probe = 'inspect' | 'rcon'; -interface Observation { result: T; observedAt: string } -interface Identity { key: string; port: number | null; checkedAt: string | null } +interface Observation { + result: T; + observedAt: string; +} +interface Identity { + key: string; + port: number | null; + checkedAt: string | null; +} interface Entry { identity: Identity; startedAt: number; @@ -28,21 +39,50 @@ export class MonitoringSkippedError extends Error {} function identity(server: string, probe: Probe, snapshot?: MonitoringSnapshot): Identity { const row = snapshot ? snapshot.get(server)?.server : getServer(server); - if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.'); - const native = snapshot ? snapshot.get(server)?.native : row.native_instance_id ? nativeInstance(server) : null; - if (row.native_instance_id && (!native || (snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !== 'online')) { + if (!row || row.destroyed_at) + throw new MonitoringChangedError( + 'Server is no longer registered; monitoring evidence is unavailable.' + ); + const native = snapshot + ? snapshot.get(server)?.native + : row.native_instance_id + ? nativeInstance(server) + : null; + if ( + row.native_instance_id && + (!native || + (snapshot ? snapshot.get(server)?.host : getHost(native.host_id))?.connectionState !== + 'online') + ) { throw new Error('Host agent is offline; monitoring evidence is unavailable.'); } // Stop requests also invalidate observations, even though they do not // establish a new game-telemetry epoch in latestGameSubmission(). const savedLifecycle = snapshot?.get(server)?.lifecycle; - const lifecycle = snapshot ? savedLifecycle ? { id: savedLifecycle.id, action: savedLifecycle.action, submitted_at: savedLifecycle.submitted_at } : null : db().prepare(`SELECT id, action, submitted_at FROM server_deployments - WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null; + const lifecycle = snapshot + ? savedLifecycle + ? { + id: savedLifecycle.id, + action: savedLifecycle.action, + submitted_at: savedLifecycle.submitted_at + } + : null + : (db() + .prepare( + `SELECT id, action, submitted_at FROM server_deployments + WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1` + ) + .get(server) ?? null); return { - key: JSON.stringify([row.created_at, row.desired_state, snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server), lifecycle, + key: JSON.stringify([ + row.created_at, + row.desired_state, + snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server), + lifecycle, native ? [native.host_id, native.id, native.generation] : null, - probe === 'rcon' ? [row.rcon_host, config.rcon] : null]), - port: probe === 'rcon' ? native?.port ?? row.game_port : null, + probe === 'rcon' ? [row.rcon_host, config.rcon] : null + ]), + port: probe === 'rcon' ? (native?.port ?? row.game_port) : null, checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at }; } @@ -58,14 +98,32 @@ function reusable(startedAt: number): boolean { return age >= 0 && age < MAX_AGE_MS; } -async function observe(server: string, fresh: boolean, probe: Probe, - entries: Map>, request: (server: string) => Promise, snapshot?: MonitoringSnapshot, batch = false): Promise> { +async function observe( + server: string, + fresh: boolean, + probe: Probe, + entries: Map>, + request: (server: string) => Promise, + snapshot?: MonitoringSnapshot, + batch = false +): Promise> { let current: Identity; - try { current = identity(server, probe, snapshot); } - catch (error) { entries.delete(server); throw error; } + try { + current = identity(server, probe, snapshot); + } catch (error) { + entries.delete(server); + throw error; + } const saved = entries.get(server); - const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0; - if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) { + const savedAt = saved?.observation + ? Date.parse(saved.observation.observedAt) + : (saved?.startedAt ?? 0); + if ( + !fresh && + saved && + sameIdentity(saved.identity, current) && + !newerEvidence(current, savedAt) + ) { if (saved.pending) return saved.pending; if (saved.observation && reusable(saved.startedAt)) return saved.observation; } @@ -76,22 +134,27 @@ async function observe(server: string, fresh: boolean entries.delete(server); entries.set(server, entry); while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!); - const pending = Promise.resolve().then(() => request(server)).then(result => { - if (!batch) snapshot?.refresh(server); - const latest = identity(server, probe, snapshot); - if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) { - throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.'); - } - const observation = { result, observedAt: new Date().toISOString() }; - if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) { - entry.identity = latest; - entry.observation = observation; - } - return observation; - }).finally(() => { - entry.pending = undefined; - if (entries.get(server) === entry && !entry.observation) entries.delete(server); - }); + const pending = Promise.resolve() + .then(() => request(server)) + .then((result) => { + if (!batch) snapshot?.refresh(server); + const latest = identity(server, probe, snapshot); + if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) { + throw new MonitoringChangedError( + 'Server changed during monitoring; the previous observation was discarded.' + ); + } + const observation = { result, observedAt: new Date().toISOString() }; + if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) { + entry.identity = latest; + entry.observation = observation; + } + return observation; + }) + .finally(() => { + entry.pending = undefined; + if (entries.get(server) === entry && !entry.observation) entries.delete(server); + }); entry.pending = pending; return pending; } @@ -112,14 +175,19 @@ export function monitoringIdentity(server: string, snapshot?: MonitoringSnapshot const container = identity(server, 'inspect', snapshot); const game = identity(server, 'rcon', snapshot); // Identity includes connection configuration; persist only its fingerprint. - return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex'); + return createHash('sha256') + .update(JSON.stringify([container.key, game.key, game.port])) + .digest('hex'); } /** Capture ownership before a status pass; publish only evidence for that instance. */ export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnapshot) { let before: string; - try { before = identity(server, 'inspect', snapshot).key; } - catch { return () => {}; } + try { + before = identity(server, 'inspect', snapshot).key; + } catch { + return () => {}; + } const slot = { snapshot: statuses.get(server)?.snapshot }; statuses.delete(server); statuses.set(server, slot); @@ -133,12 +201,17 @@ export function monitoringStatusWriter(server: string, snapshot?: MonitoringSnap const current = slot.snapshot; if (current && current.container.observedAt > container.observedAt) return; slot.snapshot = { identity: monitoringIdentity(server, snapshot), container, game }; - } catch { statuses.delete(server); } + } catch { + statuses.delete(server); + } }; } /** Latest completed status-monitor pass, with stale or superseded evidence excluded. */ -export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot): MonitoringStatus | null { +export function monitoringStatus( + server: string, + snapshot?: MonitoringSnapshot +): MonitoringStatus | null { const saved = statuses.get(server)?.snapshot; if (!saved) return null; const fresh = (at: string) => { @@ -146,43 +219,94 @@ export function monitoringStatus(server: string, snapshot?: MonitoringSnapshot): return age >= 0 && age <= ONLINE_MAX_MS; }; try { - if (saved.identity !== monitoringIdentity(server, snapshot) - || !fresh(saved.container.observedAt) - || newerEvidence(identity(server, 'inspect', snapshot), Date.parse(saved.container.observedAt))) return null; - const game = saved.game && fresh(saved.game.observedAt) - && !newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt)) ? saved.game : undefined; + if ( + saved.identity !== monitoringIdentity(server, snapshot) || + !fresh(saved.container.observedAt) || + newerEvidence( + identity(server, 'inspect', snapshot), + Date.parse(saved.container.observedAt) + ) + ) + return null; + const game = + saved.game && + fresh(saved.game.observedAt) && + !newerEvidence(identity(server, 'rcon', snapshot), Date.parse(saved.game.observedAt)) + ? saved.game + : undefined; return { ...saved, game }; - } catch { return null; } + } catch { + return null; + } } /** Keep the actual sample time: a later heartbeat must not renew old evidence. */ -export function heartbeatInspection(server: string, snapshot?: MonitoringSnapshot): Observation>> | null { +export function heartbeatInspection( + server: string, + snapshot?: MonitoringSnapshot +): Observation>> | null { const row = snapshot ? snapshot.get(server)?.server : getServer(server); if (!row?.native_instance_id || !row.host_id) return null; const host = snapshot ? snapshot.get(server)?.host : getHost(row.host_id); - if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null; - const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation); + if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) + return null; + const state = host.inventory.instanceStates?.find( + (state) => + state.instanceId === row.native_instance_id && + state.generation === row.instance_generation + ); if (!state) return null; const at = Date.parse(state.observedAt); - const lifecycle = snapshot ? { at: snapshot.get(server)?.lifecycleAt } : db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at - FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null }; - if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '') - || at < Date.parse((snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null; - return { observedAt: new Date(at).toISOString(), result: state.healthy - ? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } }; + const lifecycle = snapshot + ? { at: snapshot.get(server)?.lifecycleAt } + : (db() + .prepare( + `SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at + FROM server_deployments WHERE server=? AND server_deleted_at IS NULL` + ) + .get(server) as { at: string | null }); + if ( + !reusable(at) || + at < Date.parse(row.created_at) || + at < Date.parse(lifecycle.at ?? '') || + at < + Date.parse( + (snapshot ? snapshot.get(server)?.submission : latestGameSubmission(server)) ?? '' + ) || + at < Date.parse(row.container_checked_at ?? '') + ) + return null; + return { + observedAt: new Date(at).toISOString(), + result: state.healthy + ? { ok: true, state: state.state } + : { ok: false, state: state.state, reason: `Container ${state.state}` } + }; } type RconObservation = Observation>>; /** Prime a whole host before the per-host queue admits just its first two servers. */ -export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSnapshot): Map Promise> { +export function prepareMonitoringRcon( + servers: string[], + snapshot?: MonitoringSnapshot +): Map Promise> { const probes = new Map Promise>(); - const batches = new Map }> }>(); + const batches = new Map< + string, + { + instances: NativeInstance[]; + result: Promise<{ + results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }>; + }>; + } + >(); for (const server of servers) { const instance = snapshot ? snapshot.get(server)?.native : nativeInstance(server); if (!instance) continue; const host = snapshot ? snapshot.get(server)?.host : getHost(instance.host_id); - if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue; + if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) + continue; const container = heartbeatInspection(server, snapshot); if (container && !container.result.ok && container.result.state !== 'unknown') continue; const before = identity(server, 'rcon', snapshot); @@ -193,40 +317,73 @@ export function prepareMonitoringRcon(servers: string[], snapshot?: MonitoringSn // observe() invokes each request in the same microtask turn; // enqueue afterward so cached or shared probes need no batch slot. const result = Promise.resolve().then(async () => { - const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', { - instances: instances.map(value => ({ instanceId: value.id, generation: value.generation })) - })); + const operation = withOperationSource('Status monitor', () => + queueHostOperation(instance.host_id, 'rcon.status-batch', { + instances: instances.map((value) => ({ + instanceId: value.id, + generation: value.generation + })) + }) + ); // Allow delivery and receipt overhead beyond the agent's 20-second budget. const receipt = await waitHostOperationStatus(operation.id, 25_000); snapshot?.refresh(); - if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.'); - return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }; + if (receipt.state !== 'completed') + throw new Error(receipt.error ?? 'Host status batch did not complete.'); + return receipt.result as { + results: Array<{ + instanceId: string; + generation: number; + reply: AgentRconReply; + }>; + }; }); - batch = { instances, result }; batches.set(instance.host_id, batch); + batch = { instances, result }; + batches.set(instance.host_id, batch); } batch.instances.push(instance); const endpoint = { host: `agent:${instance.host_id}`, port: instance.port }; try { const receipt = await batch.result; - const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation); - if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.'); - if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.'); + const replies = receipt.results.filter( + (value) => + value.instanceId === instance.id && value.generation === instance.generation + ); + if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') + throw new Error('Host status batch omitted a valid instance result.'); + if (replies[0].reply.reason === 'deadline') + throw new MonitoringSkippedError( + 'Host status batch budget expired before this server was checked.' + ); return agentRconStatusResult(replies[0].reply, endpoint); } catch (error) { if (error instanceof MonitoringSkippedError) throw error; - return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message }; + return { + ok: false as const, + endpoint, + reason: 'timeout' as const, + detail: (error as Error).message + }; } }; // Observe before dispatch, including servers still waiting for their // queue slot. Consume failures too, so a failed batch is never retried // once per server or left as an unhandled rejected promise. - const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then(value => ({ value }), error => ({ error })); + const pending = observe(server, false, 'rcon', rcon, request, snapshot, true).then( + (value) => ({ value }), + (error) => ({ error }) + ); probes.set(server, async () => { const result = await pending; if ('error' in result) throw result.error; const current = identity(server, 'rcon', snapshot); - if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) { - throw new MonitoringChangedError('Server changed before consuming the host status batch.'); + if ( + !sameIdentity(before, current) || + newerEvidence(current, Date.parse(result.value.observedAt)) + ) { + throw new MonitoringChangedError( + 'Server changed before consuming the host status batch.' + ); } return result.value; }); diff --git a/src/lib/server/fleet/health.ts b/src/lib/server/fleet/health.ts index 3c69f9a..78e745e 100644 --- a/src/lib/server/fleet/health.ts +++ b/src/lib/server/fleet/health.ts @@ -9,9 +9,14 @@ const STALE_MAX_MS = 120_000; * `stale` on a blocking RCON failure or if the last reply is >90-120s old. * `online` if fresh, including the allowance for polling and delivery. */ -export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health { +export function classifyHealth( + lastSeenAt: string | null, + lastError: string | null, + containerState?: string | null +): Health { if (containerState === 'starting' || containerState === 'restarting') return 'starting'; - if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline'; + if (containerState && containerState !== 'running' && containerState !== 'unknown') + return 'offline'; if (!lastSeenAt) return 'offline'; const age = Date.now() - new Date(lastSeenAt).getTime(); if (freshStatusTimestamp(lastSeenAt, Date.now())) { diff --git a/src/lib/server/fleet/house-party.ts b/src/lib/server/fleet/house-party.ts index 13c8097..bc02774 100644 --- a/src/lib/server/fleet/house-party.ts +++ b/src/lib/server/fleet/house-party.ts @@ -2,23 +2,35 @@ import { db } from '../db/index.js'; import { destroyServer } from '../provisioning/index.js'; export function housePartyTargets(): string[] { - return (db().prepare(` + return ( + db() + .prepare( + ` SELECT name FROM servers WHERE destroyed_at IS NULL UNION SELECT server AS name FROM server_deployments WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY name - `).all() as { name: string }[]).map(row => row.name); + ` + ) + .all() as { name: string }[] + ).map((row) => row.name); } export function queueHouseParty(expectedServers: string[]) { return db().transaction(() => { const servers = housePartyTargets(); const expected = [...expectedServers].sort(); - if (servers.length !== expected.length || servers.some((name, index) => name !== expected[index])) { - return { ok: false as const, error: 'The fleet changed. Refresh the preview and review the servers again.' }; + if ( + servers.length !== expected.length || + servers.some((name, index) => name !== expected[index]) + ) { + return { + ok: false as const, + error: 'The fleet changed. Refresh the preview and review the servers again.' + }; } - const deployments = servers.map(server => { + const deployments = servers.map((server) => { const result = destroyServer(server, 'house-party-protocol'); if (!result.ok) throw new Error(result.message); return { server, deploymentId: result.deploymentId! }; diff --git a/src/lib/server/fleet/inventory.ts b/src/lib/server/fleet/inventory.ts index d293639..bde0878 100644 --- a/src/lib/server/fleet/inventory.ts +++ b/src/lib/server/fleet/inventory.ts @@ -36,7 +36,9 @@ export function recordScript( } export function removeScript(server: string, relPath: string): void { - db().prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?').run(server, relPath); + db() + .prepare('DELETE FROM server_scripts WHERE server = ? AND rel_path = ?') + .run(server, relPath); } export function getServerScript(server: string, relPath: string): ServerScriptRow | undefined { diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 303d660..26c32f9 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -1,12 +1,20 @@ import { packFileJson, unpackFileJson } from '../db/file-blobs.js'; import { randomUUID } from 'node:crypto'; -import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$lib/deployment-plan.js'; +import type { + BlueprintSummary, + LaunchRecipeSummary, + LaunchFileSummary +} from '$lib/deployment-plan.js'; import { config, configFileForGame } from '../config.js'; import { db } from '../db/index.js'; import { zombieMapBySlug, type ZombieGame } from './map-aliases.js'; import { promotedArtifactMetadata, promotedArtifacts } from './promoted-artifacts.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; -import { getBaselineForScope, getVersionMetadata, listPromotedOverlaysForScope } from './scripts.js'; +import { + getBaselineForScope, + getVersionMetadata, + listPromotedOverlaysForScope +} from './scripts.js'; import { sha256Buffer } from './files.js'; export interface LaunchRecipePlan { @@ -28,165 +36,337 @@ export interface LaunchRecipe extends LaunchRecipePlan { files: (LaunchFileSummary & { content: string })[]; } -export interface BlueprintEntry { count: number; recipe: LaunchRecipePlan } -interface BlueprintSelection { count: number; game: ZombieGame; mapSlug: string } -interface BlueprintRow { id: string; name: string; description: string; created_at: string; recipe: string } +export interface BlueprintEntry { + count: number; + recipe: LaunchRecipePlan; +} +interface BlueprintSelection { + count: number; + game: ZombieGame; + mapSlug: string; +} +interface BlueprintRow { + id: string; + name: string; + description: string; + created_at: string; + recipe: string; +} /** Legacy snapshots provide the lineup only; their file versions never govern a new launch. */ function readEntries(raw: string): BlueprintSelection[] { - const saved = JSON.parse(raw) as LaunchRecipe | { format: 2; entries: BlueprintEntry[] } + const saved = JSON.parse(raw) as + | LaunchRecipe + | { format: 2; entries: BlueprintEntry[] } | { format: 3; entries: BlueprintSelection[] }; if (saved.format === 3) return saved.entries; const entries = saved.format === 2 ? saved.entries : [{ count: 1, recipe: saved }]; - return entries.map(({ count, recipe }) => ({ count, game: recipe.game, mapSlug: recipe.mapSlug })); + return entries.map(({ count, recipe }) => ({ + count, + game: recipe.game, + mapSlug: recipe.mapSlug + })); } function summarizeBlueprint(row: BlueprintRow): BlueprintSummary { - const entries = readEntries(row.recipe).map(entry => ({ ...entry, - mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug })); - return { id: row.id, name: row.name, description: row.description, createdAt: row.created_at, - entries, totalServers: entries.reduce((total, entry) => total + entry.count, 0) }; + const entries = readEntries(row.recipe).map((entry) => ({ + ...entry, + mapName: zombieMapBySlug(entry.game, entry.mapSlug)?.friendly ?? entry.mapSlug + })); + return { + id: row.id, + name: row.name, + description: row.description, + createdAt: row.created_at, + entries, + totalServers: entries.reduce((total, entry) => total + entry.count, 0) + }; } export function blueprintEntries(id: string): { entries: BlueprintEntry[]; blueprintName: string } { - const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined; - if (!row) throw new Error('This collection is no longer available. Choose another starting point.'); - const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipePlan(game, mapSlug) })); + const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as + BlueprintRow | undefined; + if (!row) + throw new Error('This collection is no longer available. Choose another starting point.'); + const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ + count, + recipe: captureRecipePlan(game, mapSlug) + })); return { entries, blueprintName: row.name }; } function buildEntries(input: unknown): BlueprintSelection[] { - if (!Array.isArray(input) || input.length < 1 || input.length > 40) throw new Error('Add between 1 and 40 map selections.'); + if (!Array.isArray(input) || input.length < 1 || input.length > 40) + throw new Error('Add between 1 and 40 map selections.'); const seen = new Set(); - const entries = input.map(entry => { - if (!entry || (entry.game !== 't5' && entry.game !== 't6') || typeof entry.mapSlug !== 'string' - || !zombieMapBySlug(entry.game, entry.mapSlug)) throw new Error('Each selection needs a supported game and map.'); - if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15) throw new Error('Choose 1–15 lobbies for each map.'); + const entries = input.map((entry) => { + if ( + !entry || + (entry.game !== 't5' && entry.game !== 't6') || + typeof entry.mapSlug !== 'string' || + !zombieMapBySlug(entry.game, entry.mapSlug) + ) + throw new Error('Each selection needs a supported game and map.'); + if (!Number.isSafeInteger(entry.count) || entry.count < 1 || entry.count > 15) + throw new Error('Choose 1–15 lobbies for each map.'); const key = `${entry.game}:${entry.mapSlug}`; - if (seen.has(key)) throw new Error('Each game/map can appear once. Increase its lobby count instead.'); + if (seen.has(key)) + throw new Error('Each game/map can appear once. Increase its lobby count instead.'); seen.add(key); - return { count: entry.count as number, game: entry.game as ZombieGame, mapSlug: entry.mapSlug as string }; + return { + count: entry.count as number, + game: entry.game as ZombieGame, + mapSlug: entry.mapSlug as string + }; }); - if (entries.reduce((total, entry) => total + entry.count, 0) > 100) throw new Error('A collection can launch at most 100 servers.'); + if (entries.reduce((total, entry) => total + entry.count, 0) > 100) + throw new Error('A collection can launch at most 100 servers.'); return entries; } -export function captureRecipe(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipe { +export function captureRecipe( + game: ZombieGame, + mapSlug: string, + rconPassword?: string +): LaunchRecipe { return buildRecipe(game, mapSlug, true, rconPassword); } /** Preview and admission keep file references; execution captures the current bytes. */ -export function captureRecipePlan(game: ZombieGame, mapSlug: string, rconPassword?: string): LaunchRecipePlan { +export function captureRecipePlan( + game: ZombieGame, + mapSlug: string, + rconPassword?: string +): LaunchRecipePlan { return buildRecipe(game, mapSlug, false, rconPassword); } -function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true, rconPassword?: string): LaunchRecipe; -function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false, rconPassword?: string): LaunchRecipePlan; -function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean, rconPassword?: string): LaunchRecipePlan { +function buildRecipe( + game: ZombieGame, + mapSlug: string, + includeContent: true, + rconPassword?: string +): LaunchRecipe; +function buildRecipe( + game: ZombieGame, + mapSlug: string, + includeContent: false, + rconPassword?: string +): LaunchRecipePlan; +function buildRecipe( + game: ZombieGame, + mapSlug: string, + includeContent: boolean, + rconPassword?: string +): LaunchRecipePlan { const map = zombieMapBySlug(game, mapSlug); if (!map) throw new Error('Choose a supported game and map.'); - const metadata = ({ kind, script, version }: ReturnType[number]): LaunchFileSummary => ({ - name: script.name, kind, path: script.target_rel_path, versionId: version.id, - version: version.version, sha256: version.sha256, size: version.size + const metadata = ({ + kind, + script, + version + }: ReturnType[number]): LaunchFileSummary => ({ + name: script.name, + kind, + path: script.target_rel_path, + versionId: version.id, + version: version.version, + sha256: version.sha256, + size: version.size }); const files: LaunchRecipePlan['files'] = includeContent - ? promotedArtifacts(game, map.canonical).map(artifact => ({ ...metadata(artifact), content: artifact.version.content.toString('base64') })) + ? promotedArtifacts(game, map.canonical).map((artifact) => ({ + ...metadata(artifact), + content: artifact.version.content.toString('base64') + })) : promotedArtifactMetadata(game, map.canonical).map(metadata); - const addConfig = (name: string, path: string, content: Buffer) => files.push({ - name, path, kind: 'config', versionId: null, version: null, - sha256: sha256Buffer(content), size: content.length, ...(includeContent ? { content: content.toString('base64') } : {}) - }); + const addConfig = (name: string, path: string, content: Buffer) => + files.push({ + name, + path, + kind: 'config', + versionId: null, + version: null, + sha256: sha256Buffer(content), + size: content.length, + ...(includeContent ? { content: content.toString('base64') } : {}) + }); const composed = getPromotedServerConfig(game, map.canonical, map.slug, rconPassword); let configSource = 'Plutainer defaults, seeded on first boot'; if (composed) { const baseline = getBaselineForScope('game', game)!; - const layers = [baseline, ...listPromotedOverlaysForScope('game', game), ...listPromotedOverlaysForScope('map', map.canonical)]; - configSource = layers.map(layer => `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`).join(' + '); + const layers = [ + baseline, + ...listPromotedOverlaysForScope('game', game), + ...listPromotedOverlaysForScope('map', map.canonical) + ]; + configSource = layers + .map( + (layer) => + `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}` + ) + .join(' + '); addConfig('Composed server configuration', configFileForGame(game), composed); } addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!); - return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical, - capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage, - configFile: configFileForGame(game), configSource, files }; + return { + format: 1, + game, + mapSlug, + mapName: map.friendly, + expectedMap: map.canonical, + capturedAt: new Date().toISOString(), + image: config.defaultRuntimeImage, + configFile: configFileForGame(game), + configSource, + files + }; } -export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary { +export function summarizeRecipe( + recipe: LaunchRecipePlan, + fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe))) +): LaunchRecipeSummary { const { format: _format, files, ...summary } = recipe; - return { ...summary, files: files.map(({ content: _content, ...file }) => file), - fingerprint }; + return { ...summary, files: files.map(({ content: _content, ...file }) => file), fingerprint }; } /** Keep the small detail response beside the immutable attempted file bytes. */ export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } { const encoded = JSON.stringify(recipe); - return { recipe: JSON.stringify(packFileJson(recipe).value), recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) }; + return { + recipe: JSON.stringify(packFileJson(recipe).value), + recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) + }; } -export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null { - const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?') +export function recipeSummaryForDeployment( + id: string +): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null { + const row = db() + .prepare( + 'SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?' + ) .get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined; if (!row) return null; - if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name }; + if (row.recipe_summary !== null) + return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name }; // Old snapshots remain readable. Populate on first access, without making // controller startup scan and decode every retained plugin in its history. const launch = recipeForDeployment(id)!; const summary = summarizeRecipe(launch.recipe); - db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id); + db() + .prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?') + .run(JSON.stringify(summary), id); return { recipe: summary, blueprintName: row.blueprint_name }; } -export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null { - const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?') +export function recipeForDeployment( + id: string +): { recipe: LaunchRecipePlan; blueprintName: string | null } | null { + const row = db() + .prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?') .get(id) as { recipe: string; blueprint_name: string | null } | undefined; - return row ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } : null; + return row + ? { recipe: unpackFileJson(JSON.parse(row.recipe)), blueprintName: row.blueprint_name } + : null; } export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } { const { entries, blueprintName } = blueprintEntries(id); - if (entries.length !== 1 || entries[0].count !== 1) throw new Error('This collection launches multiple servers. Use the collection preview.'); + if (entries.length !== 1 || entries[0].count !== 1) + throw new Error('This collection launches multiple servers. Use the collection preview.'); return { recipe: entries[0].recipe, blueprintName }; } export function listBlueprints(): BlueprintSummary[] { - return (db().prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE').all() as BlueprintRow[]).map(summarizeBlueprint); + return ( + db() + .prepare('SELECT * FROM server_blueprints ORDER BY name COLLATE NOCASE') + .all() as BlueprintRow[] + ).map(summarizeBlueprint); } function blueprintName(input: { name: unknown; description?: unknown }, exceptId = ''): string { - if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80) throw new Error('Give the collection a name of 1–80 characters.'); - if (input.description !== undefined && (typeof input.description !== 'string' || input.description.length > 500)) throw new Error('Keep the description to 500 characters.'); - if (db().prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?').get(input.name.trim(), exceptId)) throw new Error('A collection with this name already exists. Choose a new name.'); + if (typeof input.name !== 'string' || !input.name.trim() || input.name.trim().length > 80) + throw new Error('Give the collection a name of 1–80 characters.'); + if ( + input.description !== undefined && + (typeof input.description !== 'string' || input.description.length > 500) + ) + throw new Error('Keep the description to 500 characters.'); + if ( + db() + .prepare('SELECT id FROM server_blueprints WHERE name = ? COLLATE NOCASE AND id != ?') + .get(input.name.trim(), exceptId) + ) + throw new Error('A collection with this name already exists. Choose a new name.'); return input.name.trim(); } -export function saveBlueprint(input: { name: unknown; description?: unknown; previewId?: unknown; deploymentId?: unknown; entries?: unknown }): BlueprintSummary { +export function saveBlueprint(input: { + name: unknown; + description?: unknown; + previewId?: unknown; + deploymentId?: unknown; + entries?: unknown; +}): BlueprintSummary { const name = blueprintName(input); - if ([input.previewId, input.deploymentId, input.entries].filter(value => value !== undefined).length !== 1) throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.'); + if ( + [input.previewId, input.deploymentId, input.entries].filter((value) => value !== undefined) + .length !== 1 + ) + throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.'); let recipe: LaunchRecipePlan | undefined; if (typeof input.previewId === 'string') { - const row = db().prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?') + const row = db() + .prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?') .get(input.previewId, new Date().toISOString()) as { recipe: string } | undefined; if (row) recipe = JSON.parse(row.recipe); - } else if (typeof input.deploymentId === 'string') recipe = recipeForDeployment(input.deploymentId)?.recipe; - if (!recipe && input.entries === undefined) throw new Error('The launch recipe is unavailable or the preview expired. Generate a fresh preview.'); - const entries = input.entries !== undefined ? buildEntries(input.entries) - : buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]); + } else if (typeof input.deploymentId === 'string') + recipe = recipeForDeployment(input.deploymentId)?.recipe; + if (!recipe && input.entries === undefined) + throw new Error( + 'The launch recipe is unavailable or the preview expired. Generate a fresh preview.' + ); + const entries = + input.entries !== undefined + ? buildEntries(input.entries) + : buildEntries([{ count: 1, game: recipe!.game, mapSlug: recipe!.mapSlug }]); const id = randomUUID(); const createdAt = new Date().toISOString(); const saved = JSON.stringify({ format: 3, entries }); - db().prepare('INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)') + db() + .prepare( + 'INSERT INTO server_blueprints (id, name, description, created_at, recipe) VALUES (?, ?, ?, ?, ?)' + ) .run(id, name, input.description ?? '', createdAt, saved); - return summarizeBlueprint({ id, name, description: (input.description as string) ?? '', created_at: createdAt, recipe: saved }); + return summarizeBlueprint({ + id, + name, + description: (input.description as string) ?? '', + created_at: createdAt, + recipe: saved + }); } -export function updateBlueprint(id: string, input: { name: unknown; description?: unknown; entries?: unknown }): BlueprintSummary { +export function updateBlueprint( + id: string, + input: { name: unknown; description?: unknown; entries?: unknown } +): BlueprintSummary { return db().transaction(() => { - const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined; + const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as + BlueprintRow | undefined; if (!row) throw new Error('Collection not found.'); const name = blueprintName(input, id); const entries = buildEntries(input.entries); const recipe = JSON.stringify({ format: 3, entries }); const description = (input.description as string) ?? ''; - db().prepare('UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?').run(name, description, recipe, id); + db() + .prepare( + 'UPDATE server_blueprints SET name = ?, description = ?, recipe = ? WHERE id = ?' + ) + .run(name, description, recipe, id); return summarizeBlueprint({ ...row, name, description, recipe }); })(); } diff --git a/src/lib/server/fleet/leaderboard-ingest.ts b/src/lib/server/fleet/leaderboard-ingest.ts index 9e09dfd..f8f7967 100644 --- a/src/lib/server/fleet/leaderboard-ingest.ts +++ b/src/lib/server/fleet/leaderboard-ingest.ts @@ -20,7 +20,9 @@ interface Participation { } function mapIdentity(handle: Database.Database, report: GameTelemetry) { - const server = handle.prepare('SELECT game, map, map_slug FROM servers WHERE name = ?').get(report.server) as + const server = handle + .prepare('SELECT game, map, map_slug FROM servers WHERE name = ?') + .get(report.server) as { game: string | null; map: string | null; map_slug: string | null } | undefined; const parsed = parseServerName(report.server); const game = (server?.game ?? parsed?.game ?? 'unknown').toLowerCase(); @@ -28,15 +30,22 @@ function mapIdentity(handle: Database.Database, report: GameTelemetry) { // A shared engine map (notably zm_transit) is not enough to identify T6's // location/ruleset. Use the registered selection only when it matches. const parsedSlug = parsed?.mapCanonical === map ? parsed.mapSlug : undefined; - const slug = server?.map === map ? server.map_slug ?? parsedSlug : parsedSlug; - const candidates = zombieMaps(game).filter(entry => entry.canonical === map); - const selected = game === 't6' && candidates.length > 1 && !slug - ? undefined : selectedZombieMap(game, map, slug); - const variant = game === 't6' ? selected?.slug ?? slug ?? 'unspecified' : ''; + const slug = server?.map === map ? (server.map_slug ?? parsedSlug) : parsedSlug; + const candidates = zombieMaps(game).filter((entry) => entry.canonical === map); + const selected = + game === 't6' && candidates.length > 1 && !slug + ? undefined + : selectedZombieMap(game, map, slug); + const variant = game === 't6' ? (selected?.slug ?? slug ?? 'unspecified') : ''; return { - game, map, + game, + map, key: [game, map, ...(variant ? [variant] : [])].join(':'), - name: selected?.friendly ?? (game === 't6' && candidates.length > 1 ? `${map} (${slug ?? 'unspecified variant'})` : map) + name: + selected?.friendly ?? + (game === 't6' && candidates.length > 1 + ? `${map} (${slug ?? 'unspecified variant'})` + : map) }; } @@ -44,48 +53,92 @@ function fingerprint(report: GameTelemetry): string { // Receipt timestamps and roster ordering do not identify new gameplay. const { receivedAt: _receivedAt, timing: _timing, ...event } = report; const players = [...event.players].sort((a, b) => a.guid.localeCompare(b.guid)); - return createHash('sha256').update(JSON.stringify({ ...event, players })).digest('hex'); + return createHash('sha256') + .update(JSON.stringify({ ...event, players })) + .digest('hex'); } /** Called inside the same transaction as the latest scoreboard snapshot. */ export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void { const identity = mapIdentity(handle, report); const signature = fingerprint(report); - let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1') + let match = handle + .prepare( + 'SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1' + ) .get(report.server) as Match | undefined; if (match?.last_fingerprint === signature && match.map_key === identity.key) return; const isStart = report.event === 'game_start'; const round = isStart ? 0 : report.round; - const newMatch = !match || match.map_key !== identity.key - || (isStart && (match.status !== 'active' || match.last_event !== 'game_start')) - || (!isStart && round < match.highest_round) - || ((report.event === 'round_change' || report.event === 'heartbeat') && match.status !== 'active'); + const newMatch = + !match || + match.map_key !== identity.key || + (isStart && (match.status !== 'active' || match.last_event !== 'game_start')) || + (!isStart && round < match.highest_round) || + ((report.event === 'round_change' || report.event === 'heartbeat') && + match.status !== 'active'); if (newMatch) { if (match?.status === 'active') { - handle.prepare("UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?") + handle + .prepare( + "UPDATE leaderboard_matches SET status = 'interrupted', ended_at = ? WHERE id = ?" + ) .run(report.receivedAt, match.id); } - const inserted = handle.prepare(`INSERT INTO leaderboard_matches + const inserted = handle + .prepare( + `INSERT INTO leaderboard_matches (server, game, map, map_key, map_name, started_at, last_received_at, start_observed, last_event, last_fingerprint) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) - .run(report.server, identity.game, identity.map, identity.key, identity.name, - report.receivedAt, report.receivedAt, Number(isStart), report.event, signature); - match = { id: Number(inserted.lastInsertRowid), game: identity.game, map_key: identity.key, - status: 'active', highest_round: 0, last_event: report.event, last_fingerprint: signature }; + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + report.server, + identity.game, + identity.map, + identity.key, + identity.name, + report.receivedAt, + report.receivedAt, + Number(isStart), + report.event, + signature + ); + match = { + id: Number(inserted.lastInsertRowid), + game: identity.game, + map_key: identity.key, + status: 'active', + highest_round: 0, + last_event: report.event, + last_fingerprint: signature + }; } if (!match) return; if (isStart) { - handle.prepare('UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?') + handle + .prepare( + 'UPDATE leaderboard_matches SET start_players = COALESCE(start_players, ?) WHERE id = ?' + ) .run(report.players.length, match.id); } - const previous = new Map((handle.prepare('SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?') - .all(match.id) as Participation[]).map(player => [player.guid, player])); - handle.prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?').run(match.id); - const saveIdentity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + const previous = new Map( + ( + handle + .prepare( + 'SELECT guid, last_round, present FROM leaderboard_match_players WHERE match_id = ?' + ) + .all(match.id) as Participation[] + ).map((player) => [player.guid, player]) + ); + handle + .prepare('UPDATE leaderboard_match_players SET present = 0 WHERE match_id = ?') + .run(match.id); + const saveIdentity = + handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END, last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`); @@ -108,33 +161,76 @@ export function recordLeaderboardTelemetry(handle: Database.Database, report: Ga const completedRounds = Math.max(0, round - 1); for (const player of report.players) { const prior = previous.get(player.guid); - const survived = !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0; - saveIdentity.run(identity.game, player.guid, player.name, report.receivedAt, report.receivedAt); + const survived = + !isStart && prior?.present ? Math.max(0, completedRounds - prior.last_round) : 0; + saveIdentity.run( + identity.game, + player.guid, + player.name, + report.receivedAt, + report.receivedAt + ); const stats = 'kills' in player ? player : null; - savePlayer.run(match.id, player.guid, round, survived, - stats?.kills ?? 0, stats?.headshots ?? 0, stats?.downs ?? 0, stats?.revives ?? 0, stats?.score ?? 0, - completedRounds, Number(!isStart), Number(report.event === 'game_end'), report.receivedAt, report.receivedAt, round, Number(isStart)); + savePlayer.run( + match.id, + player.guid, + round, + survived, + stats?.kills ?? 0, + stats?.headshots ?? 0, + stats?.downs ?? 0, + stats?.revives ?? 0, + stats?.score ?? 0, + completedRounds, + Number(!isStart), + Number(report.event === 'game_end'), + report.receivedAt, + report.receivedAt, + round, + Number(isStart) + ); } - handle.prepare(`UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?), + handle + .prepare( + `UPDATE leaderboard_matches SET highest_round = MAX(highest_round, ?), last_received_at = ?, last_event = ?, last_fingerprint = ?, status = CASE WHEN ? = 'game_end' THEN 'completed' ELSE status END, - ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?`) - .run(round, report.receivedAt, report.event, signature, report.event, report.event, report.receivedAt, match.id); + ended_at = CASE WHEN ? = 'game_end' THEN ? ELSE ended_at END WHERE id = ?` + ) + .run( + round, + report.receivedAt, + report.event, + signature, + report.event, + report.event, + report.receivedAt, + match.id + ); } /** Seed the single surviving scoreboard per server once; older games are gone. */ export function backfillLeaderboard(handle: Database.Database): void { if (handle.prepare("SELECT 1 FROM settings WHERE key = 'leaderboard_v1'").get()) return; handle.transaction(() => { - const reports = handle.prepare(`SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server`) + const reports = handle + .prepare( + `SELECT payload FROM server_telemetry WHERE payload IS NOT NULL ORDER BY received_at, server` + ) .all() as Array<{ payload: string }>; for (const row of reports) { let report: GameTelemetry; try { const stored = JSON.parse(row.payload); - if (typeof stored.receivedAt !== 'string' || !Number.isFinite(Date.parse(stored.receivedAt))) continue; + if ( + typeof stored.receivedAt !== 'string' || + !Number.isFinite(Date.parse(stored.receivedAt)) + ) + continue; report = { ...parseGameEvent(stored), receivedAt: stored.receivedAt }; - } catch { continue; } // Legacy malformed snapshots cannot establish player records. + } catch { + continue; + } // Legacy malformed snapshots cannot establish player records. recordLeaderboardTelemetry(handle, report); } handle.prepare("INSERT INTO settings (key, value) VALUES ('leaderboard_v1', '1')").run(); diff --git a/src/lib/server/fleet/leaderboard-records.ts b/src/lib/server/fleet/leaderboard-records.ts index 8aacbd3..ee7e714 100644 --- a/src/lib/server/fleet/leaderboard-records.ts +++ b/src/lib/server/fleet/leaderboard-records.ts @@ -1,5 +1,10 @@ import type Database from 'better-sqlite3'; -import type { LeaderboardPlayerCount, PersonalBestMetric, PersonalBestRecords, RecordEvidence } from '../../leaderboard.js'; +import type { + LeaderboardPlayerCount, + PersonalBestMetric, + PersonalBestRecords, + RecordEvidence +} from '../../leaderboard.js'; import { MIN_RECORDED_ROUND } from '../../match-history.js'; interface RecordRequest extends Record { @@ -10,14 +15,27 @@ interface RecordRequest extends Record { } /** Fetch evidence only for requested players and scopes, never rank the whole fleet's history. */ -export function getPersonalBestRecords(handle: Database.Database, requests: RecordRequest[]): PersonalBestRecords[] { +export function getPersonalBestRecords( + handle: Database.Database, + requests: RecordRequest[] +): PersonalBestRecords[] { const result: PersonalBestRecords[] = []; // Page requests are at most 100 players; chunk unusually large profile map collections too. for (let offset = 0; offset < requests.length; offset += 100) { const batch = requests.slice(offset, offset + 100); - const parameters = batch.flatMap((request, index) => [index, request.guid, request.game ?? null, - request.map ?? null, request.players ?? null, request.highestRound, request.bestKills, request.bestScore]); - const rows = handle.prepare(`WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS ( + const parameters = batch.flatMap((request, index) => [ + index, + request.guid, + request.game ?? null, + request.map ?? null, + request.players ?? null, + request.highestRound, + request.bestKills, + request.bestScore + ]); + const rows = handle + .prepare( + `WITH requested (requestId, guid, game, map, players, highestRound, bestKills, bestScore) AS ( VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?, ?)').join(', ')} ), candidates AS ( SELECT r.requestId, m.id AS matchId, m.game, m.map_key AS mapKey, m.map_name AS mapName, @@ -41,14 +59,23 @@ export function getPersonalBestRecords(handle: Database.Database, requests: Reco SELECT *, ROW_NUMBER() OVER (PARTITION BY requestId, metric ORDER BY completionOrder, matchId) AS choice FROM valuesByMetric ) SELECT requestId, metric, value, matchId, game, mapKey, mapName, server, startedAt, endedAt, - startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1`) - .all(...parameters) as Array & { - requestId: number; metric: PersonalBestMetric; startObserved: number; presentAtStart: number | null; - }>; - const records = batch.map(() => ({} as PersonalBestRecords)); + startObserved, firstObservedRound, presentAtStart FROM ranked WHERE choice = 1` + ) + .all(...parameters) as Array< + Omit & { + requestId: number; + metric: PersonalBestMetric; + startObserved: number; + presentAtStart: number | null; + } + >; + const records = batch.map(() => ({}) as PersonalBestRecords); for (const { requestId, metric, startObserved, presentAtStart, ...record } of rows) { - records[requestId][metric] = { ...record, startObserved: Boolean(startObserved), - presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart) }; + records[requestId][metric] = { + ...record, + startObserved: Boolean(startObserved), + presentAtStart: presentAtStart === null ? null : Boolean(presentAtStart) + }; } result.push(...records); } diff --git a/src/lib/server/fleet/leaderboard.ts b/src/lib/server/fleet/leaderboard.ts index 9f0b6df..e2a9b36 100644 --- a/src/lib/server/fleet/leaderboard.ts +++ b/src/lib/server/fleet/leaderboard.ts @@ -2,19 +2,34 @@ import { db } from '../db/index.js'; import { MIN_RECORDED_ROUND } from '../../match-history.js'; import { getPersonalBestRecords } from './leaderboard-records.js'; import { countsAsGamePlayed } from './leaderboard-participation.js'; -import { EFFICIENCY_MIN_GAMES, EFFICIENCY_MIN_KILLS, EFFICIENCY_MIN_DOWNS, - isEfficiencyMetric } from '../../leaderboard-efficiency.js'; -import { leaderboardMetrics, type LeaderboardEntry, type LeaderboardMap, type LeaderboardMetric, - type LeaderboardOptions, type LeaderboardResponse } from '../../leaderboard.js'; +import { + EFFICIENCY_MIN_GAMES, + EFFICIENCY_MIN_KILLS, + EFFICIENCY_MIN_DOWNS, + isEfficiencyMetric +} from '../../leaderboard-efficiency.js'; +import { + leaderboardMetrics, + type LeaderboardEntry, + type LeaderboardMap, + type LeaderboardMetric, + type LeaderboardOptions, + type LeaderboardResponse +} from '../../leaderboard.js'; -const metrics = new Set(leaderboardMetrics.map(metric => metric.key)); +const metrics = new Set(leaderboardMetrics.map((metric) => metric.key)); export const MAX_LEADERBOARD_OFFSET = 1_000_000; type RankedEntry = Omit; -interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] } +interface RankedScope { + revision: number; + entries: RankedEntry[]; + maps: LeaderboardMap[]; + summary: LeaderboardResponse['summary']; +} const scopeCaches = new WeakMap>(); // Match SQLite's built-in lower(), which folds ASCII rather than Unicode. -const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase()); +const searchText = (value: string) => value.replace(/[A-Z]/g, (ch) => ch.toLowerCase()); interface LeaderboardRow extends Omit { // A left join keeps metadata available when the requested page has no players. @@ -39,35 +54,78 @@ export function getLeaderboard(options: LeaderboardOptions = {}): LeaderboardRes } /** Select the exact player after ranking everyone eligible in the requested scope. */ -export function getPlayerLeaderboardEntry(guid: string, - options: Pick = {}): LeaderboardEntry | null { - return readLeaderboard({ game: options.game, map: options.map, players: options.players, metric: options.metric, limit: 1 }, guid); +export function getPlayerLeaderboardEntry( + guid: string, + options: Pick = {} +): LeaderboardEntry | null { + return readLeaderboard( + { + game: options.game, + map: options.map, + players: options.players, + metric: options.metric, + limit: 1 + }, + guid + ); } -function addRecordSources(handle: ReturnType, entries: RankedEntry[], options: LeaderboardOptions): LeaderboardEntry[] { - const records = getPersonalBestRecords(handle, entries.map(entry => ({ - ...entry, game: options.game || undefined, map: options.map || undefined, players: options.players - }))); +function addRecordSources( + handle: ReturnType, + entries: RankedEntry[], + options: LeaderboardOptions +): LeaderboardEntry[] { + const records = getPersonalBestRecords( + handle, + entries.map((entry) => ({ + ...entry, + game: options.game || undefined, + map: options.map || undefined, + players: options.players + })) + ); return entries.map((entry, index) => ({ ...entry, records: records[index] })); } function readLeaderboard(options: LeaderboardOptions): LeaderboardResponse; function readLeaderboard(options: LeaderboardOptions, exactGuid: string): LeaderboardEntry | null; -function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): LeaderboardResponse | LeaderboardEntry | null { +function readLeaderboard( + options: LeaderboardOptions, + exactGuid?: string +): LeaderboardResponse | LeaderboardEntry | null { const handle = db(); - const metric = options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound'; - const limit = Number.isSafeInteger(options.limit) ? Math.min(100, Math.max(1, options.limit!)) : 50; - const offset = Number.isSafeInteger(options.offset) ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) : 0; - const conditions = ["m.game IN ('t5', 't6')", "m.status = 'completed'", 'mp.has_stats = 1', `m.highest_round >= ${MIN_RECORDED_ROUND}`, - `mp.highest_round >= ${MIN_RECORDED_ROUND}`]; + const metric = + options.metric && isLeaderboardMetric(options.metric) ? options.metric : 'highestRound'; + const limit = Number.isSafeInteger(options.limit) + ? Math.min(100, Math.max(1, options.limit!)) + : 50; + const offset = Number.isSafeInteger(options.offset) + ? Math.min(MAX_LEADERBOARD_OFFSET, Math.max(0, options.offset!)) + : 0; + const conditions = [ + "m.game IN ('t5', 't6')", + "m.status = 'completed'", + 'mp.has_stats = 1', + `m.highest_round >= ${MIN_RECORDED_ROUND}`, + `mp.highest_round >= ${MIN_RECORDED_ROUND}` + ]; const parameters: Array = []; - if (options.game) { conditions.push('m.game = ?'); parameters.push(options.game); } - if (options.map) { conditions.push('m.map_key = ?'); parameters.push(options.map); } - if (options.players) { conditions.push('m.start_players = ?'); parameters.push(options.players); } + if (options.game) { + conditions.push('m.game = ?'); + parameters.push(options.game); + } + if (options.map) { + conditions.push('m.map_key = ?'); + parameters.push(options.map); + } + if (options.players) { + conditions.push('m.start_players = ?'); + parameters.push(options.players); + } const qualification = isEfficiencyMetric(metric) - ? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL` - + (metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '') - + (metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '') + ? `WHERE measures.gamesPlayed >= ${EFFICIENCY_MIN_GAMES} AND measures.${metric} IS NOT NULL` + + (metric === 'headshotRate' ? ` AND measures.kills >= ${EFFICIENCY_MIN_KILLS}` : '') + + (metric === 'killsPerDown' ? ` AND measures.downs >= ${EFFICIENCY_MIN_DOWNS}` : '') : ''; // A board reuses observations for player totals and distinct match metadata. // Exact-player requests only consume the totals, so they can stream the join once. @@ -100,14 +158,31 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification} )`; let revision = -1; - try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; } - catch { /* Standalone history tools can operate without cache metadata. */ } + try { + revision = ( + handle + .prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'") + .get() as { revision: number } + ).revision; + } catch { + /* Standalone history tools can operate without cache metadata. */ + } let cache = scopeCaches.get(handle); - if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); } - const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', options.players ?? '', metric]); + if (!cache) { + cache = new Map(); + scopeCaches.set(handle, cache); + } + const cacheKey = JSON.stringify([ + options.game ?? '', + options.map ?? '', + options.players ?? '', + metric + ]); let saved = cache.get(cacheKey); if (!saved || revision < 0 || saved.revision !== revision) { - const rows = handle.prepare(`${aggregated}, playerSummary AS ( + const rows = handle + .prepare( + `${aggregated}, playerSummary AS ( SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills, COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived, COALESCE(MAX(highestRound), 0) AS highestRound FROM totals @@ -121,36 +196,73 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound, metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt FROM metadata LEFT JOIN ranked ON 1 - ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`) - .all(...parameters) as LeaderboardRow[]; - const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null) - .map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds, - summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry); - const metadata = rows[0]; - const summary: LeaderboardResponse['summary'] = { - players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills, - roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound, - firstTrackedAt: metadata.summaryFirstTrackedAt, updatedAt: metadata.summaryUpdatedAt - }; - // Keep qualifying map variants available, even after their server is removed. - const maps = handle.prepare(`SELECT map_key AS key, game, map, map_name AS name + ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid` + ) + .all(...parameters) as LeaderboardRow[]; + const rankedEntries = rows + .filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null) + .map( + ({ + matchingTotal, + summaryPlayers, + summaryMatches, + summaryKills, + summaryRounds, + summaryHighestRound, + summaryFirstTrackedAt, + summaryUpdatedAt, + ...entry + }) => entry + ); + const metadata = rows[0]; + const summary: LeaderboardResponse['summary'] = { + players: metadata.summaryPlayers, + matches: metadata.summaryMatches, + kills: metadata.summaryKills, + roundsSurvived: metadata.summaryRounds, + highestRound: metadata.summaryHighestRound, + firstTrackedAt: metadata.summaryFirstTrackedAt, + updatedAt: metadata.summaryUpdatedAt + }; + // Keep qualifying map variants available, even after their server is removed. + const maps = handle + .prepare( + `SELECT map_key AS key, game, map, map_name AS name FROM leaderboard_matches WHERE game IN ('t5', 't6') AND status = 'completed' AND highest_round >= ${MIN_RECORDED_ROUND}${options.game ? ' AND game = ?' : ''} GROUP BY game, map_key HAVING id = MAX(id) - ORDER BY game, name COLLATE NOCASE, map_key`) - .all(...(options.game ? [options.game] : [])) as LeaderboardMap[]; - saved = { revision, entries: rankedEntries, maps, summary }; - cache.delete(cacheKey); - if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved); - // Bound total retained players as well as the number of scopes. - while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!); + ORDER BY game, name COLLATE NOCASE, map_key` + ) + .all(...(options.game ? [options.game] : [])) as LeaderboardMap[]; + saved = { revision, entries: rankedEntries, maps, summary }; + cache.delete(cacheKey); + if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved); + // Bound total retained players as well as the number of scopes. + while ( + cache.size > 8 || + [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000 + ) + cache.delete(cache.keys().next().value!); } if (exactGuid !== undefined) { - const entry = saved.entries.find(entry => entry.guid === exactGuid); + const entry = saved.entries.find((entry) => entry.guid === exactGuid); return entry ? addRecordSources(handle, [entry], options)[0] : null; } const query = searchText(options.q?.trim() ?? ''); - const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries; + const matching = query + ? saved.entries.filter( + (entry) => + searchText(entry.name).includes(query) || searchText(entry.guid).includes(query) + ) + : saved.entries; const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options); - return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary }; + return { + entries, + maps: saved.maps, + total: matching.length, + limit, + offset, + metric, + summary: saved.summary + }; } diff --git a/src/lib/server/fleet/library-metadata.ts b/src/lib/server/fleet/library-metadata.ts index 047e07e..a635cbd 100644 --- a/src/lib/server/fleet/library-metadata.ts +++ b/src/lib/server/fleet/library-metadata.ts @@ -6,7 +6,9 @@ export function scriptCatalog(kind?: ArtifactKind) { const version = (alias: string) => `CASE WHEN ${alias}.id IS NOT NULL THEN json_object( 'id',${alias}.id,'version',${alias}.version,'release_version',${alias}.release_version, 'sha256',${alias}.sha256,'size',${alias}.size,'state',${alias}.state,'uploaded_at',${alias}.uploaded_at) END`; - const rows = db().prepare(`SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json, + const rows = db() + .prepare( + `SELECT s.*, ${version('c')} current_json, ${version('v')} latest_json, COALESCE(counts.n,0) versionCount, active.beta_id runningBetaId FROM scripts s LEFT JOIN (SELECT script_id, COUNT(*) n, MAX(version) latest FROM script_versions GROUP BY script_id) counts ON counts.script_id=s.id @@ -14,32 +16,77 @@ export function scriptCatalog(kind?: ArtifactKind) { LEFT JOIN script_versions c ON c.id=s.current_version_id LEFT JOIN (SELECT bs.script_id, MIN(b.id) beta_id FROM beta_scripts bs JOIN betas b ON b.id=bs.beta_id WHERE b.state IN ('running','ready-promote') GROUP BY bs.script_id) active ON active.script_id=s.id - ${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name`).all(...(kind ? [kind] : [])) as Array; - const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { name: string; game: string | null; map: string | null }[]; - return rows.map(({ current_json, latest_json, ...row }) => ({ ...row, - currentVersion: current_json ? JSON.parse(current_json) as ScriptVersionRow : null, - latestVersion: latest_json ? JSON.parse(latest_json) as ScriptVersionRow : null, - servers: row.scope_type && row.scope_value ? servers.filter(server => server[row.scope_type!] === row.scope_value).map(server => server.name) : [] + ${kind ? 'WHERE s.kind=?' : ''} ORDER BY s.name` + ) + .all(...(kind ? [kind] : [])) as Array< + ScriptRow & { + current_json: string | null; + latest_json: string | null; + versionCount: number; + runningBetaId: number | null; + } + >; + const servers = db().prepare('SELECT name, game, map FROM servers ORDER BY name').all() as { + name: string; + game: string | null; + map: string | null; + }[]; + return rows.map(({ current_json, latest_json, ...row }) => ({ + ...row, + currentVersion: current_json ? (JSON.parse(current_json) as ScriptVersionRow) : null, + latestVersion: latest_json ? (JSON.parse(latest_json) as ScriptVersionRow) : null, + servers: + row.scope_type && row.scope_value + ? servers + .filter((server) => server[row.scope_type!] === row.scope_value) + .map((server) => server.name) + : [] })); } export function betaScriptMetadata(ids: number[]) { - const rows = db().prepare(`SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name, + const rows = db() + .prepare( + `SELECT bs.beta_id, bs.script_id id, COALESCE(s.name,'(deleted)') name, COALESCE(s.target_rel_path,'') target_rel_path, COALESCE(s.kind,bs.kind,'script') kind, v.id version_id, v.version, v.release_version, v.sha256 FROM beta_scripts bs LEFT JOIN scripts s ON s.id=bs.script_id LEFT JOIN script_versions v ON v.id=bs.script_version_id - WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id`).all(JSON.stringify(ids)) as Array<{ - beta_id: number; id: number; name: string; target_rel_path: string; kind: ArtifactKind; - version_id: number | null; version: number; release_version: string | null; sha256: string; - }>; - const grouped = new Map>(); + WHERE bs.beta_id IN (SELECT value FROM json_each(?)) ORDER BY bs.script_id` + ) + .all(JSON.stringify(ids)) as Array<{ + beta_id: number; + id: number; + name: string; + target_rel_path: string; + kind: ArtifactKind; + version_id: number | null; + version: number; + release_version: string | null; + sha256: string; + }>; + const grouped = new Map< + number, + Array<{ + id: number; + name: string; + target_rel_path: string; + kind: ArtifactKind; + version: { + id: number; + version: number; + release_version: string | null; + sha256: string; + } | null; + }> + >(); for (const { beta_id, version_id, version, release_version, sha256, ...script } of rows) { const group = grouped.get(beta_id) ?? []; - group.push({ ...script, version: version_id === null ? null : { id: version_id, version, release_version, sha256 } }); + group.push({ + ...script, + version: + version_id === null ? null : { id: version_id, version, release_version, sha256 } + }); grouped.set(beta_id, group); } return grouped; diff --git a/src/lib/server/fleet/lifecycle-evidence.ts b/src/lib/server/fleet/lifecycle-evidence.ts index 2c2b6ad..9f323d2 100644 --- a/src/lib/server/fleet/lifecycle-evidence.ts +++ b/src/lib/server/fleet/lifecycle-evidence.ts @@ -2,18 +2,27 @@ import { db } from '../db/index.js'; /** Reports from before the latest lifecycle submission describe the old process. */ export function latestGameSubmission(server: string): string | null { - return (db().prepare(`SELECT MAX(at) AS at FROM ( + return ( + db() + .prepare( + `SELECT MAX(at) AS at FROM ( SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') - UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at; + UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)` + ) + .get(server, server) as { at: string | null } + ).at; } /** Look up each registered server through the covering submission index. */ export function latestGameSubmissions(): Map { - const rows = db().prepare(`SELECT server, MAX(at) AS at FROM ( + const rows = db() + .prepare( + `SELECT server, MAX(at) AS at FROM ( SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at FROM servers s - UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as - { server: string; at: string | null }[]; - return new Map(rows.map(row => [row.server, row.at])); + UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server` + ) + .all() as { server: string; at: string | null }[]; + return new Map(rows.map((row) => [row.server, row.at])); } diff --git a/src/lib/server/fleet/logs.ts b/src/lib/server/fleet/logs.ts index 4b25597..29ca5ad 100644 --- a/src/lib/server/fleet/logs.ts +++ b/src/lib/server/fleet/logs.ts @@ -14,7 +14,8 @@ export const KNOWN_LOGS = { } as const; export type LogKey = keyof typeof KNOWN_LOGS; -export const gameLogKey = (server: string): LogKey => gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp'; +export const gameLogKey = (server: string): LogKey => + gameOfServer(server) === 't6' ? 'games_zm' : 'games_sp'; export function isLogKey(v: string): v is LogKey { return Object.prototype.hasOwnProperty.call(KNOWN_LOGS, v); @@ -45,33 +46,62 @@ export function resolveLogPath(serverName: string, key: LogKey): string | null { export interface TailResult { lines: string[]; size: number; - mtime: string; // ISO + mtime: string; // ISO truncated: boolean; // true when we didn't read from the start of the file } export interface LogCursor { - fileId: string; offset: number; size: number; modified: string; - anchorLength: number; anchorHash: string; + fileId: string; + offset: number; + size: number; + modified: string; + anchorLength: number; + anchorHash: string; } export interface LogChunk { - contentBase64: string; size: number; mtime: string; truncated: boolean; - cursor: LogCursor; reset: boolean; + contentBase64: string; + size: number; + mtime: string; + truncated: boolean; + cursor: LogCursor; + reset: boolean; } /** Bounded byte reads; an anchor before the offset detects copy-truncate and regrowth. */ -export async function readLogChunk(path: string, cursor?: LogCursor): Promise { +export async function readLogChunk( + path: string, + cursor?: LogCursor +): Promise { const remote = await nativeLogChunk(path, cursor); if (remote !== undefined) return remote; let file; - try { file = await open(path, 'r'); } - catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; } + try { + file = await open(path, 'r'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } try { const st = await file.stat(); if (!st.isFile()) return null; - const fileId = `${st.dev}:${st.ino}`, modified = `${st.mtimeMs}:${st.ctimeMs}`, size = st.size; + const fileId = `${st.dev}:${st.ino}`, + modified = `${st.mtimeMs}:${st.ctimeMs}`, + size = st.size; const mtime = st.mtime.toISOString(); - if (cursor?.fileId === fileId && cursor.modified === modified && cursor.size === size && cursor.offset === size) { - return { contentBase64: '', size, mtime, truncated: cursor.offset > 0, cursor, reset: false }; + if ( + cursor?.fileId === fileId && + cursor.modified === modified && + cursor.size === size && + cursor.offset === size + ) { + return { + contentBase64: '', + size, + mtime, + truncated: cursor.offset > 0, + cursor, + reset: false + }; } const hash = (value: Buffer) => createHash('sha256').update(value).digest('hex'); let reset = !cursor || cursor.fileId !== fileId || cursor.offset > size; @@ -86,9 +116,24 @@ export async function readLogChunk(path: string, cursor?: LogCursor): Promise 0, reset, - cursor: { fileId, modified, size, offset, anchorLength: anchor.length, anchorHash: hash(anchor) } }; - } finally { await file.close(); } + return { + contentBase64: content.subarray(0, read.bytesRead).toString('base64'), + size, + mtime, + truncated: start > 0, + reset, + cursor: { + fileId, + modified, + size, + offset, + anchorLength: anchor.length, + anchorHash: hash(anchor) + } + }; + } finally { + await file.close(); + } } const DEFAULT_TAIL_LINES = 200; diff --git a/src/lib/server/fleet/match-history.ts b/src/lib/server/fleet/match-history.ts index 443ad8b..453eb2d 100644 --- a/src/lib/server/fleet/match-history.ts +++ b/src/lib/server/fleet/match-history.ts @@ -1,6 +1,12 @@ import { db } from '../db/index.js'; import { matchChatTimeline } from './chat.js'; -import { MIN_RECORDED_ROUND, type MatchDetail, type MatchPlayer, type MatchHistoryResponse, type ServerMatch } from '../../match-history.js'; +import { + MIN_RECORDED_ROUND, + type MatchDetail, + type MatchPlayer, + type MatchHistoryResponse, + type ServerMatch +} from '../../match-history.js'; import { readMatchQuery, type MatchHistoryQuery } from '../../match-history-query.js'; @@ -15,29 +21,69 @@ const matchSummary = `m.id, m.game, m.highest_round AS observedRound, m.server, MAX(0, ROUND((julianday(m.ended_at) - julianday(m.started_at)) * 86400000)) END AS durationMs, CASE WHEN m.status = 'completed' THEN m.highest_round END AS finalRound`; -export function serverMatchHistory(server?: string, limit = 25, query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()), now = Date.now()): MatchHistoryResponse { +export function serverMatchHistory( + server?: string, + limit = 25, + query: MatchHistoryQuery = readMatchQuery(new URLSearchParams()), + now = Date.now() +): MatchHistoryResponse { const handle = db(); const clauses = [`m.highest_round >= ${MIN_RECORDED_ROUND}`]; const parameters: Array = []; - if (server !== undefined) { clauses.push('m.server = ?'); parameters.push(server); } - if (query.game) { clauses.push('m.game = ?'); parameters.push(query.game); } - const maps = handle.prepare(`SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game - FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game`) + if (server !== undefined) { + clauses.push('m.server = ?'); + parameters.push(server); + } + if (query.game) { + clauses.push('m.game = ?'); + parameters.push(query.game); + } + const maps = handle + .prepare( + `SELECT m.map_key AS key, MAX(m.map_name) AS name, m.game + FROM leaderboard_matches m WHERE ${clauses.join(' AND ')} GROUP BY m.map_key, m.game ORDER BY name COLLATE NOCASE, game` + ) .all(...parameters) as MatchHistoryResponse['maps']; - if (query.map) { clauses.push('m.map_key = ?'); parameters.push(query.map); } - if (query.status) { clauses.push('m.status = ?'); parameters.push(query.status); } - if (query.days) { clauses.push('julianday(m.started_at) >= julianday(?)'); parameters.push(new Date(now - query.days * 86400000).toISOString()); } + if (query.map) { + clauses.push('m.map_key = ?'); + parameters.push(query.map); + } + if (query.status) { + clauses.push('m.status = ?'); + parameters.push(query.status); + } + if (query.days) { + clauses.push('julianday(m.started_at) >= julianday(?)'); + parameters.push(new Date(now - query.days * 86400000).toISOString()); + } // Search readable instance names as well as their stored identifiers. Tokens allow "Town Lobby 1". - for (const token of query.q.toLowerCase().replace(/\blobby\s+(?=\d+\b)/g, '').split(/[\s_]+/).filter(Boolean)) { - clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0"); parameters.push(token); + for (const token of query.q + .toLowerCase() + .replace(/\blobby\s+(?=\d+\b)/g, '') + .split(/[\s_]+/) + .filter(Boolean)) { + clauses.push("instr(lower(m.server || ' ' || m.map_name), ?) > 0"); + parameters.push(token); } const where = `WHERE ${clauses.join(' AND ')}`; - const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total; + const total = ( + handle + .prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`) + .get(...parameters) as { total: number } + ).total; const page = Math.min(query.page, Math.max(1, Math.ceil(total / limit))); - const column = { started: 'startedAt', round: 'observedRound', duration: 'durationMs', players: 'uniquePlayers' }[query.sort]; + const column = { + started: 'startedAt', + round: 'observedRound', + duration: 'durationMs', + players: 'uniquePlayers' + }[query.sort]; const direction = query.direction === 'asc' ? 'ASC' : 'DESC'; - const matches = handle.prepare(`SELECT ${matchSummary} FROM leaderboard_matches m - ${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?`) + const matches = handle + .prepare( + `SELECT ${matchSummary} FROM leaderboard_matches m + ${where} ORDER BY ${column} IS NULL ASC, ${column} ${direction}, m.id DESC LIMIT ? OFFSET ?` + ) .all(...parameters, limit, (page - 1) * limit) as ServerMatch[]; return { matches, total, page, limit, maps }; } @@ -46,13 +92,20 @@ export function serverMatchHistory(server?: string, limit = 25, query: MatchHist export function getMatchDetail(id: number): MatchDetail | null { if (!Number.isSafeInteger(id) || id < 1) return null; const handle = db(); - const match = handle.prepare(`SELECT ${matchSummary}, m.start_observed AS startObserved + const match = handle + .prepare( + `SELECT ${matchSummary}, m.start_observed AS startObserved FROM leaderboard_matches m WHERE m.id = ? AND m.status = 'completed' - AND m.highest_round >= ${MIN_RECORDED_ROUND}`) - .get(id) as (Omit & { startObserved: number }) | undefined; + AND m.highest_round >= ${MIN_RECORDED_ROUND}` + ) + .get(id) as + | (Omit & { startObserved: number }) + | undefined; if (!match) return null; - const players = handle.prepare(`SELECT p.guid, + const players = handle + .prepare( + `SELECT p.guid, COALESCE((SELECT identity.name FROM leaderboard_players identity WHERE identity.guid = p.guid AND identity.game IN ('t5', 't6') ORDER BY identity.last_seen_at DESC, identity.game LIMIT 1), @@ -76,12 +129,24 @@ export function getMatchDetail(id: number): MatchDetail | null { AND history.highest_round >= ${MIN_RECORDED_ROUND} AND recorded.highest_round >= ${MIN_RECORDED_ROUND})) AS canViewProfile FROM leaderboard_match_players p WHERE p.match_id = ? - ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid`) - .all(match.game, id) as Array & { - hasStats: number; presentAtEnd: number; canViewProfile: number; - }>; - return { ...match, chat: matchChatTimeline(id), startObserved: Boolean(match.startObserved), players: players.map(player => ({ - ...player, hasStats: Boolean(player.hasStats), presentAtEnd: Boolean(player.presentAtEnd), - canViewProfile: Boolean(player.canViewProfile) - })) }; + ORDER BY hasStats DESC, kills DESC, highestRound DESC, name COLLATE NOCASE, p.guid` + ) + .all(match.game, id) as Array< + Omit & { + hasStats: number; + presentAtEnd: number; + canViewProfile: number; + } + >; + return { + ...match, + chat: matchChatTimeline(id), + startObserved: Boolean(match.startObserved), + players: players.map((player) => ({ + ...player, + hasStats: Boolean(player.hasStats), + presentAtEnd: Boolean(player.presentAtEnd), + canViewProfile: Boolean(player.canViewProfile) + })) + }; } diff --git a/src/lib/server/fleet/metrics.ts b/src/lib/server/fleet/metrics.ts index 16d579d..77efa0d 100644 --- a/src/lib/server/fleet/metrics.ts +++ b/src/lib/server/fleet/metrics.ts @@ -1,8 +1,18 @@ import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; -import { parseMetricsQuery, type HeatmapCell, type MapMetrics, type MetricPoint, type MetricsData, - type MetricsDays, type MetricsQuery, type PreviousMetrics, type ServerMetrics, type Utilization } from '../../metrics.js'; +import { + parseMetricsQuery, + type HeatmapCell, + type MapMetrics, + type MetricPoint, + type MetricsData, + type MetricsDays, + type MetricsQuery, + type PreviousMetrics, + type ServerMetrics, + type Utilization +} from '../../metrics.js'; export type { MapMetrics, Utilization } from '../../metrics.js'; @@ -17,10 +27,13 @@ export interface MetricSample { export function recordMetrics(samples: MetricSample[], at = Date.now()): void { const handle = db(); handle.transaction(() => { - const insert = handle.prepare('INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)'); + const insert = handle.prepare( + 'INSERT OR REPLACE INTO metric_samples (at, server, game, map, players, capacity) VALUES (?, ?, ?, ?, ?, ?)' + ); for (const s of samples) insert.run(at, s.server, s.game, s.map, s.players, s.capacity); const days = fleetSettings().metricsKeepDays; - if (days) handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000); + if (days) + handle.prepare('DELETE FROM metric_samples WHERE at < ?').run(at - days * 86400_000); })(); } @@ -42,52 +55,120 @@ function addObservation(total: Utilization, players: number | null, hours: numbe * capped at the freshness limit; longer gaps contribute no observed or unknown time. * Read the indexed history once and share the intervals across every aggregation. */ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()): MetricsData { - const query = parseMetricsQuery(new URLSearchParams(typeof input === 'number' ? { days: String(input) } - : { days: String(input.days), game: input.game ?? '', map: input.map ?? '', timeZone: input.timeZone })); + const query = parseMetricsQuery( + new URLSearchParams( + typeof input === 'number' + ? { days: String(input) } + : { + days: String(input.days), + game: input.game ?? '', + map: input.map ?? '', + timeZone: input.timeZone + } + ) + ); const { days, ...scope } = query; const since = now - days * 86400_000; const bucketMs = days === 1 ? 3600_000 : 86400_000; const handle = db(); // These small grouped results keep options and retained evidence independent // of the chosen scope/window without transferring a second raw history scan. - const groups = handle.prepare(`SELECT game, map, MIN(at) AS first, MAX(at) AS latest - FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map`).all(now) as - { game: string; map: string; first: number; latest: number }[]; + const groups = handle + .prepare( + `SELECT game, map, MIN(at) AS first, MAX(at) AS latest + FROM metric_samples WHERE at <= ? GROUP BY game, map ORDER BY game, map` + ) + .all(now) as { game: string; map: string; first: number; latest: number }[]; const matchesScope = (sample: { game: string; map: string }) => - (scope.game === null || sample.game === scope.game) && (scope.map === null || sample.map === scope.map); - const evidenceBounds = (values: typeof groups) => values.reduce<{ first: number | null; latest: number | null }>((bounds, group) => ({ - first: bounds.first === null ? group.first : Math.min(bounds.first, group.first), - latest: bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest) - }), { first: null, latest: null }); + (scope.game === null || sample.game === scope.game) && + (scope.map === null || sample.map === scope.map); + const evidenceBounds = (values: typeof groups) => + values.reduce<{ first: number | null; latest: number | null }>( + (bounds, group) => ({ + first: bounds.first === null ? group.first : Math.min(bounds.first, group.first), + latest: + bounds.latest === null ? group.latest : Math.max(bounds.latest, group.latest) + }), + { first: null, latest: null } + ); const globalBounds = evidenceBounds(groups); const scopedBounds = evidenceBounds(groups.filter(matchesScope)); - const registered = new Set((handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as { name: string }[]) - .map(server => server.name)); - const metadata = { firstRetainedSampleAt: globalBounds.first, latestSampleAt: globalBounds.latest, - registeredServerCount: registered.size, scopeFirstSampleAt: scopedBounds.first, scopeLatestSampleAt: scopedBounds.latest }; - const scopeOptions = { games: [...new Set(groups.map(group => group.game))], maps: groups.map(({ game, map }) => ({ game, map })) }; + const registered = new Set( + ( + handle.prepare('SELECT name FROM servers WHERE destroyed_at IS NULL').all() as { + name: string; + }[] + ).map((server) => server.name) + ); + const metadata = { + firstRetainedSampleAt: globalBounds.first, + latestSampleAt: globalBounds.latest, + registeredServerCount: registered.size, + scopeFirstSampleAt: scopedBounds.first, + scopeLatestSampleAt: scopedBounds.latest + }; + const scopeOptions = { + games: [...new Set(groups.map((group) => group.game))], + maps: groups.map(({ game, map }) => ({ game, map })) + }; const summary = emptyUtilization(); - const previous: PreviousMetrics = { since: since - days * 86400_000, until: since, summary: emptyUtilization(), - firstSampleAt: null, lastSampleAt: null, partialHistory: false }; + const previous: PreviousMetrics = { + since: since - days * 86400_000, + until: since, + summary: emptyUtilization(), + firstSampleAt: null, + lastSampleAt: null, + partialHistory: false + }; const serverTotals = new Map(); const mapTotals = new Map>(); const points: MetricPoint[] = []; for (let at = Math.floor(since / bucketMs) * bucketMs; at < now; at += bucketMs) { const end = Math.min(at + bucketMs, now); const start = Math.max(at, since); - points.push({ at, startAt: start, endAt: end, partial: start !== at || end !== at + bucketMs, ...emptyUtilization() }); + points.push({ + at, + startAt: start, + endAt: end, + partial: start !== at || end !== at + bucketMs, + ...emptyUtilization() + }); } const cells: HeatmapCell[] = Array.from({ length: 7 * 24 }, (_, index) => ({ - weekday: Math.floor(index / 24), hour: index % 24, distinctDates: 0, ...emptyUtilization() + weekday: Math.floor(index / 24), + hour: index % 24, + distinctDates: 0, + ...emptyUtilization() })); const cellDates = cells.map(() => new Set()); - const calendar = new Intl.DateTimeFormat('en-US', { timeZone: scope.timeZone, - year: 'numeric', month: '2-digit', day: '2-digit', weekday: 'short', hour: 'numeric', hourCycle: 'h23' }); - const weekdays: Record = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 }; + const calendar = new Intl.DateTimeFormat('en-US', { + timeZone: scope.timeZone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + weekday: 'short', + hour: 'numeric', + hourCycle: 'h23' + }); + const weekdays: Record = { + Mon: 0, + Tue: 1, + Wed: 2, + Thu: 3, + Fri: 4, + Sat: 5, + Sun: 6 + }; let cachedMinute = NaN; let localCell = { index: 0, date: '' }; - interface BatchCounts { known: number; active: number; idle: number; unknown: number; players: number } + interface BatchCounts { + known: number; + active: number; + idle: number; + unknown: number; + players: number; + } function addHeatmap(counts: BatchCounts, start: number, end: number): void { // Samples are bounded by the shared freshness window. Splitting at UTC minute boundaries // handles local hour/day boundaries, DST, and fractional-hour zones. @@ -95,8 +176,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) for (let cursor = start; cursor < end;) { const minute = Math.floor(cursor / 60_000); if (minute !== cachedMinute) { - const parts = Object.fromEntries(calendar.formatToParts(cursor).map(part => [part.type, part.value])); - localCell = { index: weekdays[parts.weekday] * 24 + Number(parts.hour), date: `${parts.year}-${parts.month}-${parts.day}` }; + const parts = Object.fromEntries( + calendar.formatToParts(cursor).map((part) => [part.type, part.value]) + ); + localCell = { + index: weekdays[parts.weekday] * 24 + Number(parts.hour), + date: `${parts.year}-${parts.month}-${parts.day}` + }; cachedMinute = minute; } const stop = Math.min(end, (minute + 1) * 60_000); @@ -120,11 +206,13 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) const samples = scope.game === null ? batch : batch.filter(matchesScope); if (!samples.length) return; const end = Math.min(nextAt, at + ONLINE_MAX_MS, now); - const priorHours = Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000; + const priorHours = + Math.max(0, Math.min(end, previous.until) - Math.max(at, previous.since)) / 3600_000; if (priorHours > 0) { previous.firstSampleAt ??= at; previous.lastSampleAt = at; - for (const sample of samples) addObservation(previous.summary, sample.players, priorHours); + for (const sample of samples) + addObservation(previous.summary, sample.players, priorHours); } // Preserve current-window zero-duration rows without bringing rows from // the separate comparison window into the visible server/map tables. @@ -146,14 +234,27 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) for (const sample of samples) { let server = serverTotals.get(sample.server); if (!server) { - server = { server: sample.server, currentRegistered: registered.has(sample.server), ...emptyUtilization() }; + server = { + server: sample.server, + currentRegistered: registered.has(sample.server), + ...emptyUtilization() + }; serverTotals.set(sample.server, server); } let game = mapTotals.get(sample.game); - if (!game) { game = new Map(); mapTotals.set(sample.game, game); } + if (!game) { + game = new Map(); + mapTotals.set(sample.game, game); + } let map = game.get(sample.map); if (!map) { - map = { game: sample.game, map: sample.map, full: 0, capacityObserved: 0, ...emptyUtilization() }; + map = { + game: sample.game, + map: sample.map, + full: 0, + capacityObserved: 0, + ...emptyUtilization() + }; game.set(sample.map, map); } addObservation(summary, sample.players, hours); @@ -167,10 +268,14 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) if (sample.players > 0) counts.active++; else if (sample.players === 0) counts.idle++; } - const known = sample.players !== null && sample.capacity !== null && sample.capacity > 0; + const known = + sample.players !== null && sample.capacity !== null && sample.capacity > 0; const full = known && sample.players! >= sample.capacity!; const previous = pressure.get(map); - pressure.set(map, { known: known && (previous?.known ?? true), full: full && (previous?.full ?? true) }); + pressure.set(map, { + known: known && (previous?.known ?? true), + full: full && (previous?.full ?? true) + }); } for (const [map, state] of pressure) { if (state.known) { @@ -181,9 +286,21 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) addHeatmap(counts, start, end); } - type SampleRow = [at: number, server: string, game: string, map: string, players: number | null, capacity: number | null]; - const samples = handle.prepare(`SELECT at, server, game, map, players, capacity FROM metric_samples - WHERE at >= ? AND at <= ? ORDER BY at`).raw().iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator; + type SampleRow = [ + at: number, + server: string, + game: string, + map: string, + players: number | null, + capacity: number | null + ]; + const samples = handle + .prepare( + `SELECT at, server, game, map, players, capacity FROM metric_samples + WHERE at >= ? AND at <= ? ORDER BY at` + ) + .raw() + .iterate(previous.since - ONLINE_MAX_MS, now) as IterableIterator; for (const [at, server, game, map, players, capacity] of samples) { if (batch.length && at !== batch[0].at) { aggregateBatch(at); @@ -193,16 +310,46 @@ export function readMetrics(input: MetricsDays | MetricsQuery, now = Date.now()) } aggregateBatch(now); - const compareText = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0; - const servers = [...serverTotals.values()].sort((a, b) => b.active - a.active || compareText(a.server, b.server)); - const maps = [...mapTotals.values()].flatMap(game => [...game.values()]) - .sort((a, b) => b.playerHours - a.playerHours || compareText(a.map, b.map) || compareText(a.game, b.game)); - previous.partialHistory = scopedBounds.first === null || scopedBounds.first > previous.since || previous.firstSampleAt === null; - for (let index = 0; index < cells.length; index++) cells[index].distinctDates = cellDates[index].size; + const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + const servers = [...serverTotals.values()].sort( + (a, b) => b.active - a.active || compareText(a.server, b.server) + ); + const maps = [...mapTotals.values()] + .flatMap((game) => [...game.values()]) + .sort( + (a, b) => + b.playerHours - a.playerHours || + compareText(a.map, b.map) || + compareText(a.game, b.game) + ); + previous.partialHistory = + scopedBounds.first === null || + scopedBounds.first > previous.since || + previous.firstSampleAt === null; + for (let index = 0; index < cells.length; index++) + cells[index].distinctDates = cellDates[index].size; const fullMapHours = maps.reduce((total, map) => total + map.full, 0); const knownMapHours = maps.reduce((total, map) => total + map.capacityObserved, 0); - const pooledPressure = { value: knownMapHours > 0 ? 100 * fullMapHours / knownMapHours : null, - fullMapHours, knownMapHours, coveredMaps: maps.filter(map => map.capacityObserved > 0).length, totalMaps: maps.length }; - return { days, since, now, ...metadata, scope, scopeOptions, summary, servers, maps, points, previous, - heatmap: { timeZone: scope.timeZone, cells }, pooledPressure }; + const pooledPressure = { + value: knownMapHours > 0 ? (100 * fullMapHours) / knownMapHours : null, + fullMapHours, + knownMapHours, + coveredMaps: maps.filter((map) => map.capacityObserved > 0).length, + totalMaps: maps.length + }; + return { + days, + since, + now, + ...metadata, + scope, + scopeOptions, + summary, + servers, + maps, + points, + previous, + heatmap: { timeZone: scope.timeZone, cells }, + pooledPressure + }; } diff --git a/src/lib/server/fleet/monitoring-snapshot.ts b/src/lib/server/fleet/monitoring-snapshot.ts index 6340839..f30e3c3 100644 --- a/src/lib/server/fleet/monitoring-snapshot.ts +++ b/src/lib/server/fleet/monitoring-snapshot.ts @@ -10,9 +10,22 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { serverObservation } from './status.js'; import { recoverySettling } from '../provisioning/restarts.js'; -const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at']; +const nativeFields = [ + 'id', + 'server', + 'host_id', + 'generation', + 'library_id', + 'image', + 'bind_address', + 'port', + 'public_address', + 'public_port', + 'created_at', + 'retired_at' +]; const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id, - CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map(key => `'${key}',n.${key}`).join(',')}) END AS instance_json, + CASE WHEN n.id IS NOT NULL THEN json_object(${nativeFields.map((key) => `'${key}',n.${key}`).join(',')}) END AS instance_json, h.inventory AS host_inventory,h.last_heartbeat AS host_heartbeat, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id=h.id AND c.revoked_at IS NULL) AS credential_active, (SELECT json_object('id',d.id,'action',d.action,'state',d.state,'submitted_at',d.submitted_at) @@ -27,65 +40,147 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`; interface Row extends ServerRow { - instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number; - lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null; - restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null; + instance_json: string | null; + host_inventory: string | null; + host_heartbeat: string | null; + credential_active: number; + lifecycle_json: string | null; + lifecycle_at: string | null; + submission: string | null; + restart_at: string | null; + restart_healthy: string | null; + restart_operation_id: string | null; + telemetry_at: string | null; + telemetry_json: string | null; } interface Record { - server: ServerRow; native: NativeInstance | null; + server: ServerRow; + native: NativeInstance | null; host: Pick | null; lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null; - lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null; - restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null; + lifecycleAt: string | null; + submission: string | null; + telemetry: GameTelemetry | null; + restartAt: string | null; + restartHealthy: string | null; + restartOperationId: string | null; } /** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */ export class MonitoringSnapshot { private records = new Map(); - constructor(serverName?: string) { ensureHostSchema(db()); this.refresh(serverName); } + constructor(serverName?: string) { + ensureHostSchema(db()); + this.refresh(serverName); + } refresh(server?: string): void { - const rows = server === undefined ? db().prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name').all() as Row[] - : db().prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL').all(server) as Row[]; - if (server === undefined) this.records.clear(); else this.records.delete(server); + const rows = + server === undefined + ? (db() + .prepare(sql + ' WHERE s.destroyed_at IS NULL ORDER BY s.name') + .all() as Row[]) + : (db() + .prepare(sql + ' WHERE s.name=? AND s.destroyed_at IS NULL') + .all(server) as Row[]); + if (server === undefined) this.records.clear(); + else this.records.delete(server); for (const row of rows) { - const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission, - restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row; - const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS; + const { + instance_json, + host_inventory, + host_heartbeat, + credential_active, + lifecycle_json, + lifecycle_at, + submission, + restart_at, + restart_healthy, + restart_operation_id, + telemetry_at, + telemetry_json, + ...fields + } = row; + const connected = + !!credential_active && + !!host_heartbeat && + Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS; let telemetry: GameTelemetry | null = null; - if (telemetry_json) try { - const saved = JSON.parse(telemetry_json) as GameTelemetry; - const event = parseGameEvent(saved); - if (event.server === row.name) telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing }; - } catch { /* Invalid telemetry cannot establish health. */ } - this.records.set(row.name, { server: fields, native: instance_json ? JSON.parse(instance_json) : null, - host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline', - inventory: parseHostInventory(row.host_id!, host_inventory) } : null, - lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry, - restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id }); + if (telemetry_json) + try { + const saved = JSON.parse(telemetry_json) as GameTelemetry; + const event = parseGameEvent(saved); + if (event.server === row.name) + telemetry = { ...event, receivedAt: telemetry_at!, timing: saved.timing }; + } catch { + /* Invalid telemetry cannot establish health. */ + } + this.records.set(row.name, { + server: fields, + native: instance_json ? JSON.parse(instance_json) : null, + host: host_inventory + ? { + connected, + connectionState: connected ? 'online' : 'offline', + inventory: parseHostInventory(row.host_id!, host_inventory) + } + : null, + lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, + lifecycleAt: lifecycle_at, + submission, + telemetry, + restartAt: restart_at, + restartHealthy: restart_healthy, + restartOperationId: restart_operation_id + }); } } - get(server: string) { return this.records.get(server); } - servers() { return [...this.records.values()].map(record => record.server); } + get(server: string) { + return this.records.get(server); + } + servers() { + return [...this.records.values()].map((record) => record.server); + } provisioning(server: string) { const operation = this.get(server)?.lifecycle; - return !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state); + return ( + !!operation && + ['create', 'rebuild', 'start'].includes(operation.action) && + ['queued', 'running', 'awaiting-health'].includes(operation.state) + ); } settling(server: string) { const record = this.get(server); - return recoverySettling(record?.restartAt ? { requested_at: record.restartAt, - healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle); + return recoverySettling( + record?.restartAt + ? { + requested_at: record.restartAt, + healthy_at: record.restartHealthy, + operation_id: record.restartOperationId + } + : undefined, + record?.lifecycle + ); } telemetryEvidence(server: string) { - const record = this.get(server), report = record?.telemetry; + const record = this.get(server), + report = record?.telemetry; if (!report) return null; - const received = Date.parse(report.receivedAt), age = Date.now() - received; - return Number.isFinite(age) && age >= 0 && age <= ONLINE_MAX_MS - && (!record?.submission || received > Date.parse(record.submission)) ? report : null; + const received = Date.parse(report.receivedAt), + age = Date.now() - received; + return Number.isFinite(age) && + age >= 0 && + age <= ONLINE_MAX_MS && + (!record?.submission || received > Date.parse(record.submission)) + ? report + : null; } - observe = (server: ServerRow) => serverObservation(server, { - provisioning: this.provisioning(server.name), settling: this.settling(server.name), - submission: this.get(server.name)?.submission ?? null, telemetry: this.get(server.name)?.telemetry ?? null - }); + observe = (server: ServerRow) => + serverObservation(server, { + provisioning: this.provisioning(server.name), + settling: this.settling(server.name), + submission: this.get(server.name)?.submission ?? null, + telemetry: this.get(server.name)?.telemetry ?? null + }); patch(server: string, fields: Partial) { const record = this.get(server); if (record) record.server = { ...record.server, ...fields }; diff --git a/src/lib/server/fleet/overview.ts b/src/lib/server/fleet/overview.ts index 014e559..b48dc0c 100644 --- a/src/lib/server/fleet/overview.ts +++ b/src/lib/server/fleet/overview.ts @@ -15,90 +15,212 @@ export function serverOperations(server: string, now = Date.now()): FleetOperati function readOperations(now: number, server?: string): FleetOperation[] { const since = new Date(now - 86_400_000).toISOString(); const bindings = { server: server ?? null, since }; - const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`; + const scope = (column: string) => (server === undefined ? '' : `AND ${column} = @server`); const handle = db(); const operations: FleetOperation[] = []; - const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`) + const slots = handle + .prepare( + `SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}` + ) .all(bindings) as { server: string; requested_at: string; error: string | null }[]; - const recovering = new Map(slots.map(slot => [slot.server, slot])); + const recovering = new Map(slots.map((slot) => [slot.server, slot])); const representedRecovery = new Set(); - const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server, + const campaigns = handle + .prepare( + `SELECT d.id, d.action, d.started_at, ds.server, ds.restart_state, ds.error, ds.restarted_at, ds.players FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done' - ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as { - id: string; action: string; started_at: string; server: string; restart_state: string; - error: string | null; restarted_at: string | null; players: number | null; + ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})` + ) + .all(bindings) as { + id: string; + action: string; + started_at: string; + server: string; + restart_state: string; + error: string | null; + restarted_at: string | null; + players: number | null; }[]; for (const row of campaigns) { - const state = row.restart_state === 'done' ? 'done' - : ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active'; + const state = + row.restart_state === 'done' + ? 'done' + : ['failed', 'manual'].includes(row.restart_state) + ? 'attention' + : 'active'; const slot = recovering.get(row.server); if (row.restarted_at && state !== 'done') representedRecovery.add(row.server); - const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention' - : row.restart_state === 'held' ? 'Held by a beta' - : row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart'; - operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`, - kind: 'campaign', server: row.server, label, state, startedAt: row.started_at, - detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health' - : (row.players ?? 0) > 0 ? `${row.players} players at the last check` - : 'Waiting for restart eligibility and available capacity'), - href: `/campaigns/${encodeURIComponent(row.id)}` }); + const label = + state === 'done' + ? 'Update complete' + : state === 'attention' + ? 'Update needs attention' + : row.restart_state === 'held' + ? 'Held by a beta' + : row.restarted_at + ? 'Recovering' + : (row.players ?? 0) > 0 + ? 'Waiting for an empty lobby' + : 'Queued for restart'; + operations.push({ + key: `campaign:${row.id}:${row.server}`, + group: `campaign:${row.id}`, + kind: 'campaign', + server: row.server, + label, + state, + startedAt: row.started_at, + detail: + state === 'done' + ? 'Update applied and restart checks completed' + : (row.error ?? + (row.restarted_at + ? (slot?.error ?? 'Verifying server health') + : (row.players ?? 0) > 0 + ? `${row.players} players at the last check` + : 'Waiting for restart eligibility and available capacity')), + href: `/campaigns/${encodeURIComponent(row.id)}` + }); } // Only the newest lifecycle operation can still require action on a server. // Separate branches use state/rowid indexes for fleet reads and the server // index for detail history. Completed work is never loaded for the full fleet. - const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments + const lifecycle = handle + .prepare( + `SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health') - ${server === undefined ? '' : `UNION + ${ + server === undefined + ? '' + : `UNION SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL AND server = @server - AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`} + AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since` + } UNION SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted') - AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`) - .all(bindings) as { id: string; server: string; action: string; state: string; stage: string; - error: string | null; started_at: string }[]; + AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)` + ) + .all(bindings) as { + id: string; + server: string; + action: string; + state: string; + stage: string; + error: string | null; + started_at: string; + }[]; for (const row of lifecycle) { if (row.state === 'awaiting-health') representedRecovery.add(row.server); - const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done' - : ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active'; - operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment', - server: row.server, state, startedAt: row.started_at, - activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, resultState: row.state, - label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete' - : state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage, - detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` }); + const state = ['done', 'warning', 'cancelled'].includes(row.state) + ? 'done' + : ['failed', 'interrupted'].includes(row.state) + ? 'attention' + : 'active'; + operations.push({ + key: `deployment:${row.id}`, + group: `deployment:${row.id}`, + kind: 'deployment', + server: row.server, + state, + startedAt: row.started_at, + activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, + resultState: row.state, + label: + state === 'done' + ? row.state === 'cancelled' + ? 'Removal cancelled' + : row.state === 'warning' + ? 'Completed with warnings' + : 'Complete' + : state === 'attention' + ? `${actionLabels[row.action as ProvisioningAction]} needs attention` + : row.stage, + detail: row.error ?? row.stage, + href: `/provisioning/${encodeURIComponent(row.id)}` + }); } - const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error - FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`) - .all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[]; - const stages: Record = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta', - 'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering', - 'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' }; + const betas = handle + .prepare( + `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error + FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')` + ) + .all(bindings) as { + id: number; + opened_at: string; + server: string; + stage: string; + error: string | null; + }[]; + const stages: Record = { + queued: 'Waiting for an earlier test wave', + deploying: 'Deploying beta', + 'awaiting-empty': 'Waiting for an empty lobby', + restarting: 'Restarting', + 'awaiting-health': 'Recovering', + 'awaiting-match': 'Observing a match', + ready: 'Ready for review', + failed: 'Beta needs attention' + }; for (const row of betas) { - if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server); - operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`, - kind: 'beta', server: row.server, startedAt: row.opened_at, - state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage, - detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` }); + if (['restarting', 'awaiting-health'].includes(row.stage)) + representedRecovery.add(row.server); + operations.push({ + key: `beta:${row.id}:${row.server}`, + group: `beta:${row.id}`, + kind: 'beta', + server: row.server, + startedAt: row.opened_at, + state: row.stage === 'failed' ? 'attention' : 'active', + label: stages[row.stage] ?? row.stage, + detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, + href: `/betas/${row.id}` + }); } - const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed' - AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`) - .all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[]; + const maintenance = handle + .prepare( + `SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed' + AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)` + ) + .all(bindings) as { + id: number; + server: string; + kind: string; + detail: string | null; + requested_at: string; + }[]; for (const job of maintenance) { - operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance', - server: job.server, state: 'attention', startedAt: job.requested_at, - label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention', - detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` }); + operations.push({ + key: `maintenance:${job.id}`, + group: `maintenance:${job.id}`, + kind: 'maintenance', + server: job.server, + state: 'attention', + startedAt: job.requested_at, + label: + job.kind === 'update' + ? 'Update verification needs attention' + : 'Maintenance needs attention', + detail: job.detail ?? 'Inspect the maintenance result', + href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` + }); } for (const slot of slots) { if (representedRecovery.has(slot.server)) continue; - operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`, - kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active', - label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health', - href: `/servers/${encodeURIComponent(slot.server)}` }); + operations.push({ + key: `recovery:${slot.server}:${slot.requested_at}`, + group: `recovery:${slot.server}`, + kind: 'recovery', + server: slot.server, + startedAt: slot.requested_at, + state: slot.error ? 'attention' : 'active', + label: 'Recovering from restart', + detail: slot.error ?? 'Verifying server health', + href: `/servers/${encodeURIComponent(slot.server)}` + }); } return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt)); } diff --git a/src/lib/server/fleet/player-capacity.ts b/src/lib/server/fleet/player-capacity.ts index 53828b8..ac3423a 100644 --- a/src/lib/server/fleet/player-capacity.ts +++ b/src/lib/server/fleet/player-capacity.ts @@ -3,7 +3,10 @@ import { monitoringStatus } from './health-observations.js'; import { MonitoringSnapshot } from './monitoring-snapshot.js'; /** Read only the accepted, current-instance sample; never dispatch a probe. */ -export function observedPlayerCapacity(server: string, snapshot: MonitoringSnapshot): PlayerCapacityObservation { +export function observedPlayerCapacity( + server: string, + snapshot: MonitoringSnapshot +): PlayerCapacityObservation { const game = monitoringStatus(server, snapshot)?.game; const capacity = game?.result.ok ? game.result.status.capacity : null; return typeof capacity === 'number' && Number.isSafeInteger(capacity) && capacity > 0 diff --git a/src/lib/server/fleet/player-profile.ts b/src/lib/server/fleet/player-profile.ts index be5c8e4..6f8442e 100644 --- a/src/lib/server/fleet/player-profile.ts +++ b/src/lib/server/fleet/player-profile.ts @@ -2,7 +2,13 @@ import { db } from '../db/index.js'; import { MIN_RECORDED_ROUND } from '../../match-history.js'; import { getPersonalBestRecords } from './leaderboard-records.js'; import { countsAsGamePlayed } from './leaderboard-participation.js'; -import type { LeaderboardOptions, PlayerProfile, PlayerStats, PlayerMapRecord, PlayerMatch } from '../../leaderboard.js'; +import type { + LeaderboardOptions, + PlayerProfile, + PlayerStats, + PlayerMapRecord, + PlayerMatch +} from '../../leaderboard.js'; const totals = `MAX(mp.highest_round) AS highestRound, SUM(mp.kills) AS kills, SUM(mp.rounds_survived) AS roundsSurvived, SUM(mp.headshots) AS headshots, @@ -12,38 +18,68 @@ const scope = `FROM leaderboard_match_players mp JOIN leaderboard_matches m ON m WHERE m.game IN ('t5', 't6') AND m.status = 'completed' AND mp.guid = ? AND mp.has_stats = 1 AND m.highest_round >= ${MIN_RECORDED_ROUND} AND mp.highest_round >= ${MIN_RECORDED_ROUND}`; -export function getPlayerProfile(guid: string, requestedOffset = 0, - options: Pick = {}): PlayerProfile | null { +export function getPlayerProfile( + guid: string, + requestedOffset = 0, + options: Pick = {} +): PlayerProfile | null { const handle = db(); // Keep game-specific ingestion identities, but show one identity with the latest reported name. - const player = handle.prepare(`SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen, + const player = handle + .prepare( + `SELECT guid, name, MIN(first_seen_at) OVER () AS firstSeen, MAX(last_seen_at) OVER () AS lastSeen FROM leaderboard_players - WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1`) + WHERE game IN ('t5', 't6') AND guid = ? ORDER BY last_seen_at DESC, game LIMIT 1` + ) .get(guid) as Pick | undefined; if (!player) return null; - const { matchCount, ...stats } = handle.prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`) + const { matchCount, ...stats } = handle + .prepare(`SELECT COUNT(*) AS matchCount, ${totals} ${scope}`) .get(guid) as PlayerStats & { matchCount: number }; if (!matchCount) return null; const historyParameters: Array = [guid]; let historyScope = scope; - if (options.game) { historyScope += ' AND m.game = ?'; historyParameters.push(options.game); } - if (options.map) { historyScope += ' AND m.map_key = ?'; historyParameters.push(options.map); } - if (options.players) { historyScope += ' AND m.start_players = ?'; historyParameters.push(options.players); } - const matchesTotal = options.game || options.map || options.players - ? (handle.prepare(`SELECT COUNT(*) AS total ${historyScope}`).get(...historyParameters) as { total: number }).total - : matchCount; + if (options.game) { + historyScope += ' AND m.game = ?'; + historyParameters.push(options.game); + } + if (options.map) { + historyScope += ' AND m.map_key = ?'; + historyParameters.push(options.map); + } + if (options.players) { + historyScope += ' AND m.start_players = ?'; + historyParameters.push(options.players); + } + const matchesTotal = + options.game || options.map || options.players + ? ( + handle + .prepare(`SELECT COUNT(*) AS total ${historyScope}`) + .get(...historyParameters) as { total: number } + ).total + : matchCount; const limit = 25; const lastOffset = Math.max(0, Math.floor((matchesTotal - 1) / limit) * limit); - const offset = Number.isSafeInteger(requestedOffset) ? Math.min(lastOffset, Math.max(0, requestedOffset)) : 0; - const mapStats = handle.prepare(`SELECT m.game, m.map_key AS key, m.map_name AS name, + const offset = Number.isSafeInteger(requestedOffset) + ? Math.min(lastOffset, Math.max(0, requestedOffset)) + : 0; + const mapStats = handle + .prepare( + `SELECT m.game, m.map_key AS key, m.map_name AS name, MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope} - GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key`) + GROUP BY m.game, m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, m.game, key` + ) .all(guid) as Array>; - const [records, ...mapRecords] = getPersonalBestRecords(handle, [{ guid, ...stats }, - ...mapStats.map(map => ({ guid, ...map, map: map.key }))]); + const [records, ...mapRecords] = getPersonalBestRecords(handle, [ + { guid, ...stats }, + ...mapStats.map((map) => ({ guid, ...map, map: map.key })) + ]); // Rank all three bests in one pass over this player's maps. Filter the GUID // only after ranking competitors so ties and skipped ranks match the leaderboard. - const mapRanks = handle.prepare(`WITH totals AS ( + const mapRanks = handle + .prepare( + `WITH totals AS ( SELECT requested.key AS requestId, mp.guid, MAX(mp.highest_round) AS highestRound, MAX(mp.kills) AS bestKills, MAX(mp.best_score) AS bestScore FROM json_each(?) requested @@ -60,14 +96,25 @@ export function getPlayerProfile(guid: string, requestedOffset = 0, RANK() OVER (PARTITION BY requestId ORDER BY bestKills DESC) AS bestKills, RANK() OVER (PARTITION BY requestId ORDER BY bestScore DESC) AS bestScore FROM totals - ) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId`) - .all(JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))), guid) as PlayerMapRecord['ranks'][]; - const maps = mapStats.map((map, index) => ({ ...map, records: mapRecords[index], ranks: mapRanks[index] })); - const matches = handle.prepare(`SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName, + ) SELECT highestRound, bestKills, bestScore FROM ranked WHERE guid = ? ORDER BY requestId` + ) + .all( + JSON.stringify(mapStats.map(({ game, key }) => ({ game, key }))), + guid + ) as PlayerMapRecord['ranks'][]; + const maps = mapStats.map((map, index) => ({ + ...map, + records: mapRecords[index], + ranks: mapRanks[index] + })); + const matches = handle + .prepare( + `SELECT m.id, m.game, m.server, m.server_deleted_at, m.map_name AS mapName, m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed, mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived, mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore - ${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?`) + ${historyScope} ORDER BY mp.last_seen_at DESC, m.id DESC LIMIT ? OFFSET ?` + ) .all(...historyParameters, limit, offset) as PlayerMatch[]; return { ...player, stats, records, maps, matches, matchesTotal, offset, limit }; } diff --git a/src/lib/server/fleet/plugin-releases.ts b/src/lib/server/fleet/plugin-releases.ts index 9f30ff0..b4e1f78 100644 --- a/src/lib/server/fleet/plugin-releases.ts +++ b/src/lib/server/fleet/plugin-releases.ts @@ -1,13 +1,29 @@ import { createHash } from 'node:crypto'; import { db } from '../db/index.js'; -import { createScript, getScript, getVersion, uploadNewVersion, type ScopeType } from './scripts.js'; +import { + createScript, + getScript, + getVersion, + uploadNewVersion, + type ScopeType +} from './scripts.js'; import { findRunningBetaForScript } from './beta.js'; -import { downloadGithubAsset, getGithubRelease, GithubReleaseError } from '../integrations/github-releases.js'; +import { + downloadGithubAsset, + getGithubRelease, + GithubReleaseError +} from '../integrations/github-releases.js'; import type { PluginSource, PluginUpdate } from '$lib/plugin-github.js'; export function pluginSource(pluginId: number): PluginSource | null { - return db().prepare(`SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id - WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1`).get(pluginId) as PluginSource | undefined ?? null; + return ( + (db() + .prepare( + `SELECT p.* FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id + WHERE v.script_id = ? ORDER BY v.version DESC LIMIT 1` + ) + .get(pluginId) as PluginSource | undefined) ?? null + ); } export async function checkPluginUpdate(pluginId: number): Promise { @@ -19,13 +35,27 @@ export async function checkPluginUpdate(pluginId: number): Promise try { const release = await getGithubRelease(`https://github.com/${source.repository}`); // A latest-stable release may already be in history when a newer prerelease is imported. - if (release.id !== source.release_id && Date.parse(release.publishedAt) <= Date.parse(source.published_at)) { + if ( + release.id !== source.release_id && + Date.parse(release.publishedAt) <= Date.parse(source.published_at) + ) { return { ...base, release, status: 'ahead' }; } - const imported = db().prepare(`SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id + const imported = db() + .prepare( + `SELECT p.version_id FROM plugin_imports p JOIN script_versions v ON v.id = p.version_id WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_name = ? - ORDER BY v.version DESC LIMIT 1`).get(pluginId, source.repository, release.id, source.asset_name) as { version_id: number } | undefined; - if (imported) return { ...base, release, status: imported.version_id === plugin.current_version_id ? 'current' : 'imported', versionId: imported.version_id }; + ORDER BY v.version DESC LIMIT 1` + ) + .get(pluginId, source.repository, release.id, source.asset_name) as + { version_id: number } | undefined; + if (imported) + return { + ...base, + release, + status: imported.version_id === plugin.current_version_id ? 'current' : 'imported', + versionId: imported.version_id + }; return { ...base, release, status: 'available' }; } catch (error) { return { ...base, release: null, status: 'error', error: (error as Error).message }; @@ -42,44 +72,99 @@ export interface ImportPluginReleaseInput { } export async function importPluginRelease(input: ImportPluginReleaseInput) { - if (typeof input.url !== 'string' || !Number.isSafeInteger(input.assetId) || input.assetId < 1) { + if ( + typeof input.url !== 'string' || + !Number.isSafeInteger(input.assetId) || + input.assetId < 1 + ) { throw new GithubReleaseError('A release URL and DLL asset selection are required.'); } - if (input.pluginId !== undefined && (!Number.isSafeInteger(input.pluginId) || input.pluginId < 1)) { + if ( + input.pluginId !== undefined && + (!Number.isSafeInteger(input.pluginId) || input.pluginId < 1) + ) { throw new GithubReleaseError('Invalid plugin id.'); } const requirePlugin = () => { const plugin = input.pluginId === undefined ? undefined : getScript(input.pluginId); - if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin')) throw new GithubReleaseError('Plugin not found.', 404); - if (plugin && findRunningBetaForScript(plugin.id)) throw new GithubReleaseError('Finish the running beta before importing another release.', 409); + if (input.pluginId !== undefined && (!plugin || plugin.kind !== 'plugin')) + throw new GithubReleaseError('Plugin not found.', 404); + if (plugin && findRunningBetaForScript(plugin.id)) + throw new GithubReleaseError( + 'Finish the running beta before importing another release.', + 409 + ); return plugin; }; requirePlugin(); - if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim())) throw new GithubReleaseError('A plugin name is required.'); + if (input.pluginId === undefined && (typeof input.name !== 'string' || !input.name.trim())) + throw new GithubReleaseError('A plugin name is required.'); // Refetch the exact reviewed release: asset names, notes, tags and digests are never trusted from the client. const release = await getGithubRelease(input.url, true); - const asset = release.assets.find(a => a.id === input.assetId); - if (!asset) throw new GithubReleaseError('That DLL is no longer available in this release. Preview it again.'); + const asset = release.assets.find((a) => a.id === input.assetId); + if (!asset) + throw new GithubReleaseError( + 'That DLL is no longer available in this release. Preview it again.' + ); const content = await downloadGithubAsset(release, asset); const sha = createHash('sha256').update(content).digest('hex'); return db().transaction(() => { const plugin = requirePlugin(); if (plugin) { - const prior = db().prepare(`SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id + const prior = db() + .prepare( + `SELECT v.id FROM script_versions v JOIN plugin_imports p ON p.version_id = v.id WHERE v.script_id = ? AND lower(p.repository) = lower(?) AND p.release_id = ? AND p.asset_id = ? AND v.sha256 = ? - ORDER BY v.version DESC LIMIT 1`).get(plugin.id, release.repository, release.id, asset.id, sha) as { id: number } | undefined; + ORDER BY v.version DESC LIMIT 1` + ) + .get(plugin.id, release.repository, release.id, asset.id, sha) as + { id: number } | undefined; if (prior) return result(plugin.id, getVersion(prior.id)!, true); } - const created = plugin ? { script: plugin, version: uploadNewVersion(plugin.id, content, release.notes, release.tag) } - : createScript({ name: input.name!.trim(), kind: 'plugin', targetRelPath: asset.name, content, - releaseVersion: release.tag, notes: release.notes, scopeType: input.scopeType, scopeValue: input.scopeValue }); - db().prepare(`INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`).run(created.version.id, release.repository, release.id, release.url, asset.id, asset.name, release.publishedAt); + const created = plugin + ? { + script: plugin, + version: uploadNewVersion(plugin.id, content, release.notes, release.tag) + } + : createScript({ + name: input.name!.trim(), + kind: 'plugin', + targetRelPath: asset.name, + content, + releaseVersion: release.tag, + notes: release.notes, + scopeType: input.scopeType, + scopeValue: input.scopeValue + }); + db() + .prepare( + `INSERT INTO plugin_imports (version_id, repository, release_id, release_url, asset_id, asset_name, published_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .run( + created.version.id, + release.repository, + release.id, + release.url, + asset.id, + asset.name, + release.publishedAt + ); return result(created.script.id, created.version, false); })(); } -function result(pluginId: number, version: NonNullable>, alreadyImported: boolean) { - return { pluginId, versionId: version.id, releaseVersion: version.release_version, sha256: version.sha256, - alreadyImported, betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}` }; +function result( + pluginId: number, + version: NonNullable>, + alreadyImported: boolean +) { + return { + pluginId, + versionId: version.id, + releaseVersion: version.release_version, + sha256: version.sha256, + alreadyImported, + betaUrl: `/betas?pluginId=${pluginId}&versionId=${version.id}` + }; } diff --git a/src/lib/server/fleet/plutonium.ts b/src/lib/server/fleet/plutonium.ts index 67b9bd6..5de8316 100644 --- a/src/lib/server/fleet/plutonium.ts +++ b/src/lib/server/fleet/plutonium.ts @@ -1,2 +1,2 @@ // Compatibility for existing jobs and API clients. -export * from "../maintenance/runtime.js"; +export * from '../maintenance/runtime.js'; diff --git a/src/lib/server/fleet/promoted-artifacts.ts b/src/lib/server/fleet/promoted-artifacts.ts index c53e1d7..da32a68 100644 --- a/src/lib/server/fleet/promoted-artifacts.ts +++ b/src/lib/server/fleet/promoted-artifacts.ts @@ -1,10 +1,18 @@ import { getVersion, getVersionContent, getVersionMetadata, listScripts } from './scripts.js'; -function selectedArtifacts(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) { +function selectedArtifacts( + game: string | null, + map: string | null, + readVersion: (id: number) => Version | undefined +) { return (['script', 'plugin'] as const).flatMap((kind) => listScripts(kind).flatMap((script) => { - const scopeMatches = script.scope_type === 'game' ? script.scope_value === game - : script.scope_type === 'map' ? script.scope_value === map : false; + const scopeMatches = + script.scope_type === 'game' + ? script.scope_value === game + : script.scope_type === 'map' + ? script.scope_value === map + : false; if (!scopeMatches || !script.current_version_id) return []; const version = readVersion(script.current_version_id); return version ? [{ kind, script, version }] : []; @@ -14,7 +22,7 @@ function selectedArtifacts(game: string | null, map: string | null, rea /** Published file artifacts shared by provisioning and drift reconciliation. */ export function promotedArtifacts(game: string | null, map: string | null) { - return selectedArtifacts(game, map, id => { + return selectedArtifacts(game, map, (id) => { const version = getVersion(id); return version ? { ...version, content: getVersionContent(id) } : undefined; }); diff --git a/src/lib/server/fleet/provisioning.ts b/src/lib/server/fleet/provisioning.ts index d76bb49..66b4d12 100644 --- a/src/lib/server/fleet/provisioning.ts +++ b/src/lib/server/fleet/provisioning.ts @@ -2,17 +2,34 @@ import { db } from '../db/index.js'; /** Maintenance must leave pending lifecycle work and retained recovery to Provisioning. */ export function hasActiveProvisioning(server: string, exceptOperationId = ''): boolean { - return !!db().prepare(`SELECT 1 FROM server_deployments + return ( + !!db() + .prepare( + `SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND id != ? - AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1`).get(server, exceptOperationId) - || !!db().prepare(`SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL - AND (operation_id IS NULL OR operation_id != ?)`).get(server, exceptOperationId); + AND state IN ('queued', 'waiting', 'running', 'awaiting-health') LIMIT 1` + ) + .get(server, exceptOperationId) || + !!db() + .prepare( + `SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL + AND (operation_id IS NULL OR operation_id != ?)` + ) + .get(server, exceptOperationId) + ); } /** Only the latest lifecycle operation can describe the current server. */ export function isProvisioning(server: string): boolean { - const operation = db().prepare(`SELECT action, state FROM server_deployments - WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as - { action: string; state: string } | undefined; - return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state); + const operation = db() + .prepare( + `SELECT action, state FROM server_deployments + WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1` + ) + .get(server) as { action: string; state: string } | undefined; + return ( + !!operation && + ['create', 'rebuild', 'start'].includes(operation.action) && + ['queued', 'running', 'awaiting-health'].includes(operation.state) + ); } diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index b53bf46..8a36b36 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -3,7 +3,12 @@ import type { ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { emit } from '../pipeline/bus.js'; import { getVersion } from './scripts.js'; -import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js'; +import { + cancelStagedFiles, + isFileApplied, + queueFileChange, + stagedFileChanges +} from './staged-files.js'; // Roll a promoted deployment back to whatever versions were current before // the promote landed. Reads deployment_scripts as the source of truth for @@ -12,7 +17,15 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } // pre-multi-script legacy rows. export class RollbackError extends Error { - constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') { + constructor( + message: string, + public readonly kind: + | 'history-expired' + | 'no-such-deployment' + | 'not-promote' + | 'already-rolled-back' + | 'snapshot-pending' + ) { super(message); } } @@ -58,31 +71,53 @@ function nowIso(): string { } function rollbackDeploymentId(promoteId: string): string { - const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); + const stamp = new Date() + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d+Z$/, 'Z'); return `dep-${stamp}-rollback-${promoteId}`; } export interface RollbackDeploymentResult { - deploymentId: string; // the new rollback deployment - reverses: string; // the promote deployment id - scriptIds: number[]; // scripts whose current_version_id got reverted + deploymentId: string; // the new rollback deployment + reverses: string; // the promote deployment id + scriptIds: number[]; // scripts whose current_version_id got reverted affectedServers: string[]; } -export async function rollbackPromoteDeployment(promoteId: string): Promise { +export async function rollbackPromoteDeployment( + promoteId: string +): Promise { const dep = db() - .prepare(`SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id - FROM deployments WHERE id = ?`) + .prepare( + `SELECT id, action, script_id, script_version_id, prior_version_id, rolled_back_by, beta_id + FROM deployments WHERE id = ?` + ) .get(promoteId) as DeploymentRow | undefined; if (!dep) throw new RollbackError(`no such deployment ${promoteId}`, 'no-such-deployment'); - if (db().prepare("SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?").get(promoteId)) { - throw new RollbackError('Rollback file contents have expired under the retention policy.', 'history-expired'); + if ( + db() + .prepare( + "SELECT 1 FROM retention_expirations WHERE category='file-content' AND record_id=?" + ) + .get(promoteId) + ) { + throw new RollbackError( + 'Rollback file contents have expired under the retention policy.', + 'history-expired' + ); } if (dep.action !== 'promote') { - throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote'); + throw new RollbackError( + `deployment ${promoteId} is action='${dep.action}', not 'promote'`, + 'not-promote' + ); } if (dep.rolled_back_by) { - throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back'); + throw new RollbackError( + `deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, + 'already-rolled-back' + ); } // Source of truth for which scripts (and prior versions) this promote @@ -92,13 +127,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise { const candidates = scriptsByTarget.get(file.rel_path) ?? []; - const versionId = staged.find(row => row.deployment_file_id === file.id)?.script_version_id; - if (versionId) return candidates.find(script => script.script_version_id === versionId); + const versionId = staged.find( + (row) => row.deployment_file_id === file.id + )?.script_version_id; + if (versionId) return candidates.find((script) => script.script_version_id === versionId); if (dep.beta_id) { - const participant = candidates.find(script => db().prepare(`SELECT 1 FROM beta_server_scripts - WHERE beta_id = ? AND server = ? AND script_id = ?`).get(dep.beta_id, file.server, script.script_id)); + const participant = candidates.find((script) => + db() + .prepare( + `SELECT 1 FROM beta_server_scripts + WHERE beta_id = ? AND server = ? AND script_id = ?` + ) + .get(dep.beta_id, file.server, script.script_id) + ); if (participant) return participant; } - return candidates.find(script => getVersion(script.script_version_id)?.sha256 === file.sha256_after) ?? candidates[0]; + return ( + candidates.find( + (script) => getVersion(script.script_version_id)?.sha256 === file.sha256_after + ) ?? candidates[0] + ); }; const kindForFile = (file: FileRow): ArtifactKind => { - const stagedKind = staged.find(row => row.deployment_file_id === file.id)?.kind; + const stagedKind = staged.find((row) => row.deployment_file_id === file.id)?.kind; if (stagedKind) return stagedKind; const owner = scriptForFile(file); if (owner) return owner.kind ?? kindByScriptId.get(owner.script_id) ?? 'script'; @@ -164,11 +213,23 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise row.server === f.server && row.kind === kind - && row.rel_path === f.rel_path && row.preserve_before_image); - if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue; + const preservesBetaSnapshot = staged.some( + (row) => + row.server === f.server && + row.kind === kind && + row.rel_path === f.rel_path && + row.preserve_before_image + ); + if ( + !preservesBetaSnapshot && + isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false + ) + continue; if (f.snapshot_pending) { - throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending'); + throw new RollbackError( + `${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, + 'snapshot-pending' + ); } const arr = perServer.get(f.server); if (arr) arr.push(f); @@ -176,8 +237,13 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server - WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server); + const active = (server: string) => + !!db() + .prepare( + `SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server + WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL` + ) + .get(promoteId, server); // Write the new rollback deployment + mark the original. const insertDep = db().prepare( @@ -204,8 +270,12 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise file.verb !== 'added' && file.content_before === null)) { - insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely'); + if (files.some((file) => file.verb !== 'added' && file.content_before === null)) { + insertDepServer.run( + newDepId, + server, + 0, + 'failed', + 'Original file snapshot is missing; cannot restore safely' + ); continue; } insertDepServer.run(newDepId, server, files.length, 'pending', null); for (const f of files) { const invertedVerb = - f.verb === 'added' ? 'removed' - : f.verb === 'removed' ? 'added' - : 'updated'; + f.verb === 'added' ? 'removed' : f.verb === 'removed' ? 'added' : 'updated'; const inserted = insertDepFile.run( newDepId, server, @@ -242,27 +316,51 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise = ['script', 'config', 'plugin', 'baseline']; +export const ARTIFACT_KINDS: ReadonlyArray = [ + 'script', + 'config', + 'plugin', + 'baseline' +]; export function isArtifactKind(v: unknown): v is ArtifactKind { return typeof v === 'string' && (ARTIFACT_KINDS as readonly string[]).includes(v); @@ -78,8 +83,12 @@ function sha256(buf: Buffer): string { function normalizeReleaseVersion(value: string | null | undefined): string | null { if (value == null) return null; + // eslint-disable-next-line no-control-regex -- Reject control characters in release versions. if (typeof value !== 'string' || value.length > 80 || /[\u0000-\u001f\u007f]/.test(value)) { - throw new ScriptError('release version must be a single line of at most 80 characters', 'invalid-release-version'); + throw new ScriptError( + 'release version must be a single line of at most 80 characters', + 'invalid-release-version' + ); } return value.trim() || null; } @@ -94,7 +103,9 @@ export function getScriptByName(name: string): ScriptRow | undefined { export function listScripts(kind?: ArtifactKind): ScriptRow[] { if (kind) { - return db().prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name').all(kind) as ScriptRow[]; + return db() + .prepare('SELECT * FROM scripts WHERE kind = ? ORDER BY name') + .all(kind) as ScriptRow[]; } return db().prepare('SELECT * FROM scripts ORDER BY name').all() as ScriptRow[]; } @@ -104,7 +115,10 @@ export function listScripts(kind?: ArtifactKind): ScriptRow[] { // gated through the beta pipeline. `includeUntested` returns the baseline row // even without a promoted version (used at beta-open time when the baseline is // being introduced in the same beta as the overlays). -export function getBaselineForScope(scopeType: ScopeType, scopeValue: string): ScriptRow | undefined { +export function getBaselineForScope( + scopeType: ScopeType, + scopeValue: string +): ScriptRow | undefined { return db() .prepare( `SELECT * FROM scripts @@ -137,40 +151,49 @@ export function listPromotedOverlaysForScope( return rows.filter((r) => !skip.has(r.id)); } -export const VERSION_COLUMNS = 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at'; +export const VERSION_COLUMNS = + 'id, script_id, version, release_version, sha256, size, notes, state, kind, uploaded_at'; export function getVersionContent(id: number): Buffer { - const row = db().prepare(`SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?`).get(id) as { content: Buffer } | undefined; - if (!row?.content) throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version'); + const row = db() + .prepare( + `SELECT ${blobContent('content_hash')} AS content FROM script_versions WHERE id = ?` + ) + .get(id) as { content: Buffer } | undefined; + if (!row?.content) + throw new ScriptError(`version content is unavailable (id=${id})`, 'no-such-version'); return row.content; } export function listVersions(scriptId: number): ScriptVersionRow[] { return db() - .prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC`) + .prepare( + `SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC` + ) .all(scriptId) as ScriptVersionRow[]; } export function getVersion(id: number): ScriptVersionRow | undefined { return db().prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE id = ?`).get(id) as - | ScriptVersionRow - | undefined; + ScriptVersionRow | undefined; } export type ScriptVersionMetadata = Pick; /** Planning a launch never needs to load potentially large script or plugin bytes. */ export function getVersionMetadata(id: number): ScriptVersionMetadata | undefined { - return db().prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?').get(id) as - | ScriptVersionMetadata - | undefined; + return db() + .prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?') + .get(id) as ScriptVersionMetadata | undefined; } // Latest version is 'max(version)' — used everywhere we want the freshest // upload, whether or not it has been promoted (current_version_id). export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined { return db() - .prepare(`SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1`) + .prepare( + `SELECT ${VERSION_COLUMNS} FROM script_versions WHERE script_id = ? ORDER BY version DESC LIMIT 1` + ) .get(scriptId) as ScriptVersionRow | undefined; } @@ -191,7 +214,10 @@ export interface CreateScriptInput { // Insert script + version 1 in one transaction. Scope may be set at creation // because the script has no current_version_id yet (nothing to gate). -export function createScript(input: CreateScriptInput): { script: ScriptRow; version: ScriptVersionRow } { +export function createScript(input: CreateScriptInput): { + script: ScriptRow; + version: ScriptVersionRow; +} { if (!input.name.trim()) throw new ScriptError('script name is required', 'name-empty'); const kind: ArtifactKind = input.kind ?? 'script'; if (!isArtifactKind(kind)) throw new ScriptError(`invalid kind: ${input.kind}`, 'invalid-kind'); @@ -200,15 +226,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver // the same on-disk cfg file, so the operator never picks it. let targetRelPath = input.targetRelPath?.trim() ?? ''; if (!targetRelPath && (kind === 'config' || kind === 'baseline')) { - targetRelPath = configFileForGame(input.scopeType === 'game' ? input.scopeValue ?? null - : input.scopeValue?.startsWith('zm_') ? 't6' : 't5'); + targetRelPath = configFileForGame( + input.scopeType === 'game' + ? (input.scopeValue ?? null) + : input.scopeValue?.startsWith('zm_') + ? 't6' + : 't5' + ); } if (!targetRelPath) throw new ScriptError('target rel path is required', 'path-empty'); // Kind-specific validation. if (kind === 'baseline') { if (!input.scopeType || !input.scopeValue) { - throw new ScriptError('baselines require a scope (game or map)', 'baseline-requires-scope'); + throw new ScriptError( + 'baselines require a scope (game or map)', + 'baseline-requires-scope' + ); } if (getBaselineForScope(input.scopeType, input.scopeValue)) { throw new ScriptError( @@ -231,7 +265,8 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver } } validateScope(input.scopeType ?? null, input.scopeValue ?? null); - if (getScriptByName(input.name)) throw new ScriptError(`script "${input.name}" already exists`, 'name-taken'); + if (getScriptByName(input.name)) + throw new ScriptError(`script "${input.name}" already exists`, 'name-taken'); const sha = sha256(input.content); const releaseVersion = normalizeReleaseVersion(input.releaseVersion); @@ -249,9 +284,23 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver let scriptId!: number; let versionId!: number; db().transaction(() => { - const s = insertScript.run(input.name, targetRelPath, input.scopeType ?? null, input.scopeValue ?? null, kind); + const s = insertScript.run( + input.name, + targetRelPath, + input.scopeType ?? null, + input.scopeValue ?? null, + kind + ); scriptId = Number(s.lastInsertRowid); - const v = insertVersion.run(scriptId, sha, size, putFileBlob(input.content), input.notes ?? null, kind, releaseVersion); + const v = insertVersion.run( + scriptId, + sha, + size, + putFileBlob(input.content), + input.notes ?? null, + kind, + releaseVersion + ); versionId = Number(v.lastInsertRowid); })(); @@ -261,7 +310,12 @@ export function createScript(input: CreateScriptInput): { script: ScriptRow; ver }; } -export function uploadNewVersion(scriptId: number, content: Buffer, notes?: string, releaseVersion?: string): ScriptVersionRow { +export function uploadNewVersion( + scriptId: number, + content: Buffer, + notes?: string, + releaseVersion?: string +): ScriptVersionRow { const script = getScript(scriptId); if (!script) throw new ScriptError(`no such script (id=${scriptId})`, 'no-such-script'); @@ -285,7 +339,16 @@ export function uploadNewVersion(scriptId: number, content: Buffer, notes?: stri `INSERT INTO script_versions (script_id, version, sha256, size, content_hash, notes, kind, release_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ) - .run(scriptId, nextVersion, sha, content.byteLength, putFileBlob(content), notes ?? null, script.kind, release); + .run( + scriptId, + nextVersion, + sha, + content.byteLength, + putFileBlob(content), + notes ?? null, + script.kind, + release + ); return getVersion(Number(res.lastInsertRowid))!; } @@ -293,7 +356,10 @@ function validateScope(scopeType: ScopeType | null, scopeValue: string | null): if (scopeType === null && scopeValue === null) return; if (scopeType === 'game' || scopeType === 'map') { if (!scopeValue || !scopeValue.trim()) { - throw new ScriptError(`scope_value required for scope_type=${scopeType}`, 'invalid-scope'); + throw new ScriptError( + `scope_value required for scope_type=${scopeType}`, + 'invalid-scope' + ); } return; } @@ -352,7 +418,11 @@ export function deleteVersion(versionId: number): void { WHERE bs.script_version_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(versionId) as { id: number } | undefined; - if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta'); + if (beta) + throw new ScriptError( + `beta ${beta.id} is still running against this version`, + 'in-use-beta' + ); db().prepare('DELETE FROM script_versions WHERE id = ?').run(versionId); } @@ -370,7 +440,11 @@ export function deleteScript(scriptId: number): void { WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(scriptId) as { id: number } | undefined; - if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta'); + if (beta) + throw new ScriptError( + `beta ${beta.id} is still running against this script`, + 'has-active-beta' + ); // Every FK that references scripts(id) or script_versions(id) with NO // ACTION (the default) will abort the delete unless we null it first. @@ -386,14 +460,37 @@ export function deleteScript(scriptId: number): void { const handle = db(); const tx = handle.transaction((id: number) => { const inMyVersions = 'IN (SELECT id FROM script_versions WHERE script_id = ?)'; - const inMyBetas = 'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)'; + const inMyBetas = + 'IN (SELECT id FROM betas WHERE script_id = ? UNION SELECT beta_id FROM beta_scripts WHERE script_id = ?)'; - handle.prepare(`UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id); - handle.prepare(`UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id); - handle.prepare(`UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id); - handle.prepare(`UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}`).run(id); - handle.prepare(`UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}`).run(id); - handle.prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`).run(id, id); + handle + .prepare( + `UPDATE beta_servers SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}` + ) + .run(id); + handle + .prepare( + `UPDATE beta_server_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}` + ) + .run(id); + handle + .prepare( + `UPDATE deployment_scripts SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}` + ) + .run(id); + handle + .prepare( + `UPDATE deployments SET prior_version_id = NULL WHERE prior_version_id ${inMyVersions}` + ) + .run(id); + handle + .prepare( + `UPDATE deployments SET script_version_id = NULL WHERE script_version_id ${inMyVersions}` + ) + .run(id); + handle + .prepare(`UPDATE deployments SET beta_id = NULL WHERE beta_id ${inMyBetas}`) + .run(id, id); handle.prepare('UPDATE deployments SET script_id = NULL WHERE script_id = ?').run(id); handle.prepare('DELETE FROM scripts WHERE id = ?').run(id); }); @@ -402,7 +499,10 @@ export function deleteScript(scriptId: number): void { // Servers that match a script's scope. Empty for unscoped scripts — those are // only ever pushed to explicitly chosen beta targets. -export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: string | null): string[] { +export function serversMatchingScope( + scopeType: ScopeType | null, + scopeValue: string | null +): string[] { if (scopeType === null || scopeValue === null) return []; const rows = db() .prepare(`SELECT name FROM servers WHERE ${scopeType} = ? ORDER BY name`) @@ -415,7 +515,9 @@ export function serversMatchingScope(scopeType: ScopeType | null, scopeValue: st // server. export function listDistinctGames(): string[] { const rows = db() - .prepare(`SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game`) + .prepare( + `SELECT DISTINCT game FROM servers WHERE game IS NOT NULL AND game <> '' ORDER BY game` + ) .all() as Array<{ game: string }>; return rows.map((r) => r.game); } diff --git a/src/lib/server/fleet/server-config.ts b/src/lib/server/fleet/server-config.ts index 5334ec1..140bb23 100644 --- a/src/lib/server/fleet/server-config.ts +++ b/src/lib/server/fleet/server-config.ts @@ -7,13 +7,26 @@ import { applyRuntimePassword } from './runtime-config.js'; export const MAP_CONFIG_FILE = 'electric_cherry_map.cfg'; /** Loaded after Plutainer's default/main config and before map_rotate. */ -export function getServerMapConfig(game: string | null, map: string | null, mapSlug?: string | null): Buffer | null { +export function getServerMapConfig( + game: string | null, + map: string | null, + mapSlug?: string | null +): Buffer | null { const entry = selectedZombieMap(game, map, mapSlug); - return entry ? Buffer.from(`// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n`) : null; + return entry + ? Buffer.from( + `// Generated by Electric Cherry from the selected map.\nset sv_maprotation "${mapRotation(entry)}"\n` + ) + : null; } /** The promoted config used for first boot and drift reconciliation. */ -export function getPromotedServerConfig(game: string | null, map: string | null, mapSlug?: string | null, rconPassword?: string | null): Buffer | null { +export function getPromotedServerConfig( + game: string | null, + map: string | null, + mapSlug?: string | null, + rconPassword?: string | null +): Buffer | null { if (!game) return null; const baseline = getBaselineForScope('game', game); if (!baseline?.current_version_id) return null; @@ -24,8 +37,24 @@ export function getPromotedServerConfig(game: string | null, map: string | null, ...listPromotedOverlaysForScope('game', game), ...(map ? listPromotedOverlaysForScope('map', map) : []) ].flatMap((overlay) => { - const current = overlay.current_version_id ? getVersion(overlay.current_version_id) : undefined; - return current ? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }] : []; + const current = overlay.current_version_id + ? getVersion(overlay.current_version_id) + : undefined; + return current + ? [{ id: overlay.id, text: getVersionContent(current.id).toString('utf8') }] + : []; }); - return Buffer.from(applyRuntimePassword(composeServerCfg(getVersionContent(version.id).toString('utf8'), overlays, game, map, mapSlug), rconPassword), 'utf8'); + return Buffer.from( + applyRuntimePassword( + composeServerCfg( + getVersionContent(version.id).toString('utf8'), + overlays, + game, + map, + mapSlug + ), + rconPassword + ), + 'utf8' + ); } diff --git a/src/lib/server/fleet/server-keys.ts b/src/lib/server/fleet/server-keys.ts index a50e037..19b9cd8 100644 --- a/src/lib/server/fleet/server-keys.ts +++ b/src/lib/server/fleet/server-keys.ts @@ -15,7 +15,12 @@ function validGame(game: unknown): asserts game is KeyGame { } function normalizedKey(key: unknown): string { - if (typeof key !== 'string' || !key.trim() || key.trim().length > 512 || /\s/.test(key.trim())) { + if ( + typeof key !== 'string' || + !key.trim() || + key.trim().length > 512 || + /\s/.test(key.trim()) + ) { throw new Error('Enter a server key without whitespace (maximum 512 characters)'); } return key.trim(); @@ -29,22 +34,34 @@ function normalizedLabel(label: unknown): string { } export function listServerKeys(): ServerKey[] { - const rows = db().prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id').all() as Omit[]; - const assignments = db().prepare(`SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k + const rows = db() + .prepare('SELECT id,game,label,updated_at FROM server_keys ORDER BY game,label,id') + .all() as Omit[]; + const assignments = db() + .prepare( + `SELECT DISTINCT k.id AS key_id,l.server_id FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key JOIN server_key_leases l ON l.key_id=occupied.id - ORDER BY l.server_id`).all() as { key_id: string; server_id: string }[]; + ORDER BY l.server_id` + ) + .all() as { key_id: string; server_id: string }[]; const byKey = new Map(); for (const lease of assignments) { const servers = byKey.get(lease.key_id) ?? []; servers.push(lease.server_id); byKey.set(lease.key_id, servers); } - return rows.map(row => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] })); + return rows.map((row) => ({ ...row, assigned_servers: byKey.get(row.id) ?? [] })); } export function getServerKey(serverId: string): string | undefined { - return (db().prepare(`SELECT k.pluto_key FROM server_key_leases l - JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { pluto_key: string } | undefined)?.pluto_key; + return ( + db() + .prepare( + `SELECT k.pluto_key FROM server_key_leases l + JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1` + ) + .get(serverId) as { pluto_key: string } | undefined + )?.pluto_key; } /** Saving a credential twice cannot increase available capacity. */ @@ -52,16 +69,27 @@ export function saveServerKey(game: unknown, key: unknown, label?: unknown): str validGame(game); const credential = normalizedKey(key); const keyLabel = label === undefined ? undefined : normalizedLabel(label); - return db().transaction(() => { - const existing = db().prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?').get(game, credential) as { id: string } | undefined; - if (existing) { - if (keyLabel !== undefined) db().prepare("UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?").run(keyLabel, existing.id); - return existing.id; - } - const id = randomUUID(); - db().prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)').run(id, game, keyLabel ?? '', credential); - return id; - }).immediate(); + return db() + .transaction(() => { + const existing = db() + .prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?') + .get(game, credential) as { id: string } | undefined; + if (existing) { + if (keyLabel !== undefined) + db() + .prepare( + "UPDATE server_keys SET label=?,updated_at=datetime('now') WHERE id=?" + ) + .run(keyLabel, existing.id); + return existing.id; + } + const id = randomUUID(); + db() + .prepare('INSERT INTO server_keys(id,game,label,pluto_key) VALUES(?,?,?,?)') + .run(id, game, keyLabel ?? '', credential); + return id; + }) + .immediate(); } // Compare the credential as well as its row: a mistakenly imported copy in @@ -70,11 +98,18 @@ const availableKey = `NOT EXISTS (SELECT 1 FROM server_key_leases l JOIN server_keys occupied ON occupied.id=l.key_id WHERE occupied.pluto_key=k.pluto_key AND (l.server_id!=@server OR l.is_current=0))`; -export function serverKeyPool(game: KeyGame, serverId?: string): { total: number; available: number } { +export function serverKeyPool( + game: KeyGame, + serverId?: string +): { total: number; available: number } { validGame(game); - return db().prepare(`SELECT COUNT(*) AS total, + return db() + .prepare( + `SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN ${availableKey} THEN 1 ELSE 0 END),0) AS available - FROM server_keys k WHERE k.game=@game`).get({ game, server: serverId ?? '' }) as { total: number; available: number }; + FROM server_keys k WHERE k.game=@game` + ) + .get({ game, server: serverId ?? '' }) as { total: number; available: number }; } export function availableServerKeyCount(game: KeyGame): number { @@ -85,38 +120,75 @@ export function availableServerKeyCount(game: KeyGame): number { export function reserveServerKey(serverId: string, game: KeyGame, suppliedKey?: unknown): string { validGame(game); if (!serverId) throw new Error('Choose a server for the key reservation'); - const credential = suppliedKey === undefined || suppliedKey === null || (typeof suppliedKey === 'string' && !suppliedKey.trim()) - ? undefined : normalizedKey(suppliedKey); - return db().transaction(() => { - const existing = db().prepare(`SELECT k.game,k.pluto_key FROM server_key_leases l - JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1`).get(serverId) as { game: KeyGame; pluto_key: string } | undefined; - if (existing) { - if (existing.game !== game) throw new Error('This server already has a key reserved for another game'); - if (credential && credential !== existing.pluto_key) throw new Error('This server already has a reserved key; destroy it before changing its key'); - return existing.pluto_key; - } - const requestedId = credential ? saveServerKey(game, credential) : null; - const key = db().prepare(`SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game + const credential = + suppliedKey === undefined || + suppliedKey === null || + (typeof suppliedKey === 'string' && !suppliedKey.trim()) + ? undefined + : normalizedKey(suppliedKey); + return db() + .transaction(() => { + const existing = db() + .prepare( + `SELECT k.game,k.pluto_key FROM server_key_leases l + JOIN server_keys k ON k.id=l.key_id WHERE l.server_id=? AND l.is_current=1` + ) + .get(serverId) as { game: KeyGame; pluto_key: string } | undefined; + if (existing) { + if (existing.game !== game) + throw new Error('This server already has a key reserved for another game'); + if (credential && credential !== existing.pluto_key) + throw new Error( + 'This server already has a reserved key; destroy it before changing its key' + ); + return existing.pluto_key; + } + const requestedId = credential ? saveServerKey(game, credential) : null; + const key = db() + .prepare( + `SELECT k.id,k.pluto_key FROM server_keys k WHERE k.game=@game AND (@requestedId IS NULL OR k.id=@requestedId) AND ${availableKey} - ORDER BY k.updated_at,k.id LIMIT 1`).get({ game, server: serverId, requestedId }) as { id: string; pluto_key: string } | undefined; - if (!key) throw new Error(credential ? 'This server key is already reserved by another server' - : `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.`); - db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(serverId, key.id); - return key.pluto_key; - }).immediate(); + ORDER BY k.updated_at,k.id LIMIT 1` + ) + .get({ game, server: serverId, requestedId }) as + { id: string; pluto_key: string } | undefined; + if (!key) + throw new Error( + credential + ? 'This server key is already reserved by another server' + : `No available ${game.toUpperCase()} server keys. Add a key or destroy a server to release one.` + ); + db() + .prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)') + .run(serverId, key.id); + return key.pluto_key; + }) + .immediate(); } /** Only after confirmed destruction or a creation cancelled before any host mutation. */ export function releaseServerKey(serverId: string): boolean { - return db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0; + return ( + db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(serverId).changes > 0 + ); } export function removeServerKey(id: string): boolean { - return db().transaction(() => { - if (db().prepare(`SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key - JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?`).get(id)) { - throw new Error('This key is reserved by a server. Destroy that server before removing its key.'); - } - return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0; - }).immediate(); + return db() + .transaction(() => { + if ( + db() + .prepare( + `SELECT 1 FROM server_keys k JOIN server_keys occupied ON occupied.pluto_key=k.pluto_key + JOIN server_key_leases l ON l.key_id=occupied.id WHERE k.id=?` + ) + .get(id) + ) { + throw new Error( + 'This key is reserved by a server. Destroy that server before removing its key.' + ); + } + return db().prepare('DELETE FROM server_keys WHERE id=?').run(id).changes > 0; + }) + .immediate(); } diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index 43e176a..cb16b8f 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -35,8 +35,23 @@ export function matchServers(all: string[], patterns: string[]): string[] { return [...all]; } const regexes = patterns - .flatMap((p) => p.split(',').map((s) => s.trim()).filter(Boolean)) - .map((p) => new RegExp('^' + p.replace(/[.+^$()|{}\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$')); + .flatMap((p) => + p + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + ) + .map( + (p) => + new RegExp( + '^' + + p + .replace(/[.+^$()|{}\\]/g, '\\$&') + .replace(/\*/g, '.*') + .replace(/\?/g, '.') + + '$' + ) + ); return all.filter((name) => regexes.some((r) => r.test(name))); } @@ -49,8 +64,13 @@ export async function ensureDest( ): Promise<{ ok: true; path: string } | { ok: false; reason: string }> { const dest = destOf(server); if (nativeInstance(server)) { - try { return (await managedStat(dest)).isDirectory() ? { ok: true, path: dest } : { ok: false, reason: 'Agent script directory is unavailable' }; } - catch (error) { return { ok: false, reason: (error as Error).message }; } + try { + return (await managedStat(dest)).isDirectory() + ? { ok: true, path: dest } + : { ok: false, reason: 'Agent script directory is unavailable' }; + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } } if (existsSync(dest)) { try { @@ -152,22 +172,34 @@ export async function seedServerRegistry(): Promise { } export function getServer(name: string): ServerRow | undefined { - return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id - FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?`).get(name) as ServerRow | undefined; + return db() + .prepare( + `SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id + FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.name=?` + ) + .get(name) as ServerRow | undefined; } export function listServers(): ServerRow[] { - return db().prepare(`SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id - FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name`).all() as ServerRow[]; + return db() + .prepare( + `SELECT s.*, n.id AS native_instance_id, n.host_id, n.generation AS instance_generation, n.library_id AS game_library_id + FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL WHERE s.destroyed_at IS NULL ORDER BY s.name` + ) + .all() as ServerRow[]; } /** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */ export function activeBetaRemovalBlocker(server: string): string | null { - const betas = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + const betas = db() + .prepare( + `SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') - ORDER BY b.id`).all(server) as { id: number }[]; + ORDER BY b.id` + ) + .all(server) as { id: number }[]; if (!betas.length) return null; - return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map(beta => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`; + return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map((beta) => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`; } export function setServerField( @@ -178,7 +210,10 @@ export function setServerField( setServerFields(name, { [field]: value }); } -export function setServerFields(name: string, fields: Partial>): void { +export function setServerFields( + name: string, + fields: Partial> +): void { const allowed: Array = [ 'container_state', 'container_checked_at', @@ -211,10 +246,15 @@ export function setServerFields(name: string, fields: Partial `${field} = ?`).join(', ')} WHERE name = ? - AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`) - .run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value)); + for (const [field] of entries) + if (!allowed.includes(field as keyof ServerRow)) + throw new Error(`refusing to set ${field}`); + db() + .prepare( + `UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ? + AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})` + ) + .run(...entries.map(([, value]) => value), name, ...entries.map(([, value]) => value)); } /** diff --git a/src/lib/server/fleet/snapshot.ts b/src/lib/server/fleet/snapshot.ts index 4cc6362..19a834f 100644 --- a/src/lib/server/fleet/snapshot.ts +++ b/src/lib/server/fleet/snapshot.ts @@ -15,32 +15,62 @@ export function fleetSnapshot(serverName?: string) { const hosts = listHosts(); const serverNow = Date.now(); const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host])); - const versions = new Map(runningVersions(serverName).map(version => [version.server, version])); - const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`) - .all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance])); + const versions = new Map( + runningVersions(serverName).map((version) => [version.server, version]) + ); + const instances = new Map( + ( + db() + .prepare( + `SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}` + ) + .all(...bindings) as NativeInstance[] + ).map((instance) => [instance.server, instance]) + ); const holds = new Map(); - for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId + for (const row of db() + .prepare( + `SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_server_scripts bs ON bs.beta_id=b.id AND bs.server=bsv.server JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') - ${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as - { server: string; target: string; betaId: number }[]) { - const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries); + ${serverName === undefined ? '' : 'AND bsv.server = ?'}` + ) + .all(...bindings) as { server: string; target: string; betaId: number }[]) { + const entries = holds.get(row.server) ?? []; + entries.push({ target: row.target, betaId: row.betaId }); + holds.set(row.server, entries); } const observe = serverName === undefined ? observationReader() : serverObservation; const capacity = playerCapacityReader(serverName); const selected = serverName === undefined ? undefined : getServer(serverName); - const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : []; + const rows = + serverName === undefined + ? listServers() + : selected && !selected.destroyed_at + ? [selected] + : []; const servers = rows.map((s) => { const instance = s.native_instance_id ? instances.get(s.name) : null; return { ...observe(s), ...capacity(s.name), - host: s.host_id ? hostsById.get(s.host_id) ?? null : null, - network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null, + host: s.host_id ? (hostsById.get(s.host_id) ?? null) : null, + network: instance + ? { + bindAddress: instance.bind_address, + localPort: instance.port, + publicAddress: instance.public_address, + publicPort: instance.public_port + } + : null, plutonium: versions.get(s.name), // The host agent owns the game connection. - endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, + endpoint: instance + ? { host: `agent:${instance.host_id}`, port: instance.port } + : s.game_port + ? { host: s.rcon_host ?? rconHost, port: s.game_port } + : null, held: holds.get(s.name) ?? [] }; }); @@ -49,11 +79,22 @@ export function fleetSnapshot(serverName?: string) { serverNow, // Include hosts without assigned servers. Re-evaluate heartbeat age on // every existing snapshot refresh rather than adding a navigation poll. - hostAttention: hosts.filter(host => ['attention', 'offline'].includes(hostListStatus(host, serverNow).key)).length, - reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [], + hostAttention: hosts.filter((host) => + ['attention', 'offline'].includes(hostListStatus(host, serverNow).key) + ).length, + reservedNames: + serverName === undefined + ? ( + db() + .prepare( + "SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')" + ) + .all() as { server: string }[] + ).map((row) => row.server) + : [], integrations: { native: true, - hostsReady: hosts.filter(host => host.readiness === 'ready').length, + hostsReady: hosts.filter((host) => host.readiness === 'ready').length, hostsTotal: hosts.length, rcon: rconConfigured(), rconTimeoutMs: config.rcon.timeoutMs, diff --git a/src/lib/server/fleet/staged-files.ts b/src/lib/server/fleet/staged-files.ts index e1086e7..63778f6 100644 --- a/src/lib/server/fleet/staged-files.ts +++ b/src/lib/server/fleet/staged-files.ts @@ -21,26 +21,38 @@ interface FileChange { expectedBeforeSha?: string | null; } export interface StagedFileChange { - id: number; source_kind: SourceKind; source_id: string; server: string; - kind: ArtifactKind; rel_path: string; content: Buffer | null; - script_version_id: number | null; inventory_source: string; - deployment_file_id: number | null; preserve_before_image: number; - check_before: number; expected_before_sha: string | null; - state: 'pending' | 'applying' | 'applied' | 'cancelled'; applied_at: string | null; + id: number; + source_kind: SourceKind; + source_id: string; + server: string; + kind: ArtifactKind; + rel_path: string; + content: Buffer | null; + script_version_id: number | null; + inventory_source: string; + deployment_file_id: number | null; + preserve_before_image: number; + check_before: number; + expected_before_sha: string | null; + state: 'pending' | 'applying' | 'applied' | 'cancelled'; + applied_at: string | null; } function targetPath(server: string, kind: ArtifactKind, relPath: string): string { const root = resolve(destOfKind(server, kind)); const target = resolve(root, relPath); const rel = relative(root, target); - if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel)) throw new Error('File target must be inside its artifact directory'); + if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel)) + throw new Error('File target must be inside its artifact directory'); return target; } /** Capture immutable desired bytes without changing the server or its inventory. */ export function queueFileChange(change: FileChange): void { targetPath(change.server, change.kind, change.relPath); - db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash, + db() + .prepare( + `INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content_hash, script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET @@ -48,51 +60,122 @@ export function queueFileChange(change: FileChange): void { inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id, preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before, expected_before_sha = excluded.expected_before_sha - WHERE staged_file_changes.state = 'pending'`) - .run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, putFileBlob(change.content), - change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null, - change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null); + WHERE staged_file_changes.state = 'pending'` + ) + .run( + change.sourceKind, + change.sourceId, + change.server, + change.kind, + change.relPath, + putFileBlob(change.content), + change.scriptVersionId ?? null, + change.inventorySource, + change.deploymentFileId ?? null, + change.preserveBeforeImage ? 1 : 0, + change.expectedBeforeSha !== undefined ? 1 : 0, + change.expectedBeforeSha ?? null + ); } -export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] { - return db().prepare(`SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ? - ${server === undefined ? '' : 'AND server = ?'} ORDER BY id`) +export function stagedFileChanges( + sourceKind: SourceKind, + sourceId: string, + server?: string +): StagedFileChange[] { + return db() + .prepare( + `SELECT *, ${blobContent('content_hash')} AS content FROM staged_file_changes WHERE source_kind = ? AND source_id = ? + ${server === undefined ? '' : 'AND server = ?'} ORDER BY id` + ) .all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[]; } export function cancelStagedFiles(sourceKind: SourceKind, sourceId: string, server?: string): void { - db().prepare(`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ? - AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`) + db() + .prepare( + `UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ? + AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}` + ) .run(sourceKind, sourceId, ...(server === undefined ? [] : [server])); } -export function hasAppliedFiles(sourceKind: SourceKind, sourceId: string, server?: string): boolean { - return stagedFileChanges(sourceKind, sourceId, server).some(row => row.state === 'applied' || row.state === 'applying'); +export function hasAppliedFiles( + sourceKind: SourceKind, + sourceId: string, + server?: string +): boolean { + return stagedFileChanges(sourceKind, sourceId, server).some( + (row) => row.state === 'applied' || row.state === 'applying' + ); } /** null identifies legacy work that wrote files before staging was introduced. */ -export function isFileApplied(sourceKind: SourceKind, sourceId: string, server: string, kind: ArtifactKind, relPath: string): boolean | null { - const row = stagedFileChanges(sourceKind, sourceId, server).find(row => row.kind === kind && row.rel_path === relPath); +export function isFileApplied( + sourceKind: SourceKind, + sourceId: string, + server: string, + kind: ArtifactKind, + relPath: string +): boolean | null { + const row = stagedFileChanges(sourceKind, sourceId, server).find( + (row) => row.kind === kind && row.rel_path === relPath + ); return row ? row.state === 'applied' || row.state === 'applying' : null; } function readExisting(path: string): Buffer | null { - try { return readFileSync(path); } - catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; } + try { + return readFileSync(path); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw err; + } } /** The first activation owns the rollback image, including an absent file. */ -export function captureBetaBeforeImage(row: Pick, before: Buffer | null): void { - if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?') - .get(row.source_id, row.server, row.rel_path)) return; +export function captureBetaBeforeImage( + row: Pick, + before: Buffer | null +): void { + if ( + db() + .prepare( + 'SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?' + ) + .get(row.source_id, row.server, row.rel_path) + ) + return; const prior = getServerScript(row.server, row.rel_path); - if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots - (beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`) - .run(row.source_id, row.server, row.rel_path, putFileBlob(before), sha256Buffer(before), before.length); - db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ? - AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`) - .run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind); - db().prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)') + if (before !== null) + db() + .prepare( + `INSERT OR IGNORE INTO beta_file_snapshots + (beta_id, server, rel_path, prior_content_hash, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)` + ) + .run( + row.source_id, + row.server, + row.rel_path, + putFileBlob(before), + sha256Buffer(before), + before.length + ); + db() + .prepare( + `UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ? + AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)` + ) + .run( + before === null ? 0 : 1, + prior?.script_version_id ?? null, + row.source_id, + row.server, + row.rel_path, + row.kind + ); + db() + .prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)') .run(row.source_id, row.server, row.rel_path); } @@ -101,25 +184,44 @@ export function captureBetaBeforeImage(row: Pick { - if (row.state === 'applying') throw new Error('A previous file write was interrupted; inspect or roll back the campaign'); - const source = row.source_kind === 'campaign' - ? db().prepare(`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ? - AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`).get(row.source_id, server, operationId) - : db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ? - AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`).get(row.source_id, server, operationId); + const plans = rows.map((row) => { + if (row.state === 'applying') + throw new Error( + 'A previous file write was interrupted; inspect or roll back the campaign' + ); + const source = + row.source_kind === 'campaign' + ? db() + .prepare( + `SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ? + AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')` + ) + .get(row.source_id, server, operationId) + : db() + .prepare( + `SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ? + AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')` + ) + .get(row.source_id, server, operationId); if (!source) throw new Error('File change requester is no longer eligible'); const path = targetPath(server, row.kind, row.rel_path); const before = readExisting(path); - if (row.check_before && (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha) { + if ( + row.check_before && + (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha + ) { throw new Error(`${row.rel_path}: changed while queued; scan and review again`); } return { row, path, before }; @@ -132,14 +234,30 @@ export function applyStagedFilesForOperation(operationId: string, server: string captureBetaBeforeImage(row, before); } if (row.deployment_file_id && !row.preserve_before_image) { - db().prepare(`UPDATE deployment_files SET - verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`) - .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', putFileBlob(before), - before === null ? null : sha256Buffer(before), row.deployment_file_id); - if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') - .run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`); + db() + .prepare( + `UPDATE deployment_files SET + verb = ?, content_before_hash = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?` + ) + .run( + row.content === null ? 'removed' : before === null ? 'added' : 'updated', + putFileBlob(before), + before === null ? null : sha256Buffer(before), + row.deployment_file_id + ); + if (row.source_kind === 'campaign') + db() + .prepare( + 'INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)' + ) + .run( + row.source_id, + `${server}: captured rollback snapshot for ${row.rel_path}` + ); } - db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id); + db() + .prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?") + .run(row.id); })(); if (row.content === null) { rmSync(path, { force: true }); @@ -151,17 +269,39 @@ export function applyStagedFilesForOperation(operationId: string, server: string writeFileSync(path, row.content); // A queued version can be deleted independently; the staged bytes // still apply, but its optional inventory link must remain valid. - const version = row.script_version_id !== null && db().prepare('SELECT 1 FROM script_versions WHERE id = ?').get(row.script_version_id) - ? row.script_version_id : null; - recordScript(server, row.rel_path, version, sha256Buffer(row.content), row.content.length, row.inventory_source); + const version = + row.script_version_id !== null && + db() + .prepare('SELECT 1 FROM script_versions WHERE id = ?') + .get(row.script_version_id) + ? row.script_version_id + : null; + recordScript( + server, + row.rel_path, + version, + sha256Buffer(row.content), + row.content.length, + row.inventory_source + ); } - db().prepare("UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?") + db() + .prepare( + "UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?" + ) .run(new Date().toISOString(), row.id); - if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') - .run(row.source_id, `${server}: applied ${row.rel_path}`); + if (row.source_kind === 'campaign') + db() + .prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') + .run(row.source_id, `${server}: applied ${row.rel_path}`); } - for (const betaId of new Set(rows.filter(row => row.source_kind === 'beta').map(row => row.source_id))) { - db().prepare('UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL') + for (const betaId of new Set( + rows.filter((row) => row.source_kind === 'beta').map((row) => row.source_id) + )) { + db() + .prepare( + 'UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL' + ) .run(new Date().toISOString(), new Date().toISOString(), betaId, server); } } diff --git a/src/lib/server/fleet/status-evidence.ts b/src/lib/server/fleet/status-evidence.ts index ce7c48c..8564073 100644 --- a/src/lib/server/fleet/status-evidence.ts +++ b/src/lib/server/fleet/status-evidence.ts @@ -1,32 +1,49 @@ import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; -import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js'; +import { + rconGetPlayerCount, + rconStatusResult, + type RconStatusResult +} from '../integrations/rcon.js'; import { latestGameSubmission } from './lifecycle-evidence.js'; export { latestGameSubmission } from './lifecycle-evidence.js'; -export interface TelemetryEvidence { players: number; map: string; receivedAt: string } +export interface TelemetryEvidence { + players: number; + map: string; + receivedAt: string; +} export function telemetryReport(server: string): GameTelemetry | null { - const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as - { received_at: string; payload: string | null } | undefined; + const row = db() + .prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?') + .get(server) as { received_at: string; payload: string | null } | undefined; if (!row?.payload) return null; try { const saved = JSON.parse(row.payload) as GameTelemetry; const event = parseGameEvent(saved); - return event.server === server ? { ...event, receivedAt: row.received_at, timing: saved.timing } : null; - } catch { return null; } + return event.server === server + ? { ...event, receivedAt: row.received_at, timing: saved.timing } + : null; + } catch { + return null; + } } /** Only validated, recently received reports can establish current game state. */ -export function telemetryEvidence(server: string, after: string | null = latestGameSubmission(server)): TelemetryEvidence | null { +export function telemetryEvidence( + server: string, + after: string | null = latestGameSubmission(server) +): TelemetryEvidence | null { const report = telemetryReport(server); if (!report) return null; const received = Date.parse(report.receivedAt); const age = Date.now() - received; if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null; - if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null; + if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) + return null; return { players: report.players.length, map: report.map, receivedAt: report.receivedAt }; } @@ -45,39 +62,68 @@ export interface PlayerEvidence { /** The same eligibility decision, with provenance from the probes already performed. */ export async function probePlayerEvidence(server: string): Promise { const fromTelemetry = (value: TelemetryEvidence): PlayerEvidence => ({ - players: value.players, source: 'telemetry', observedAt: new Date().toISOString(), receivedAt: value.receivedAt + players: value.players, + source: 'telemetry', + observedAt: new Date().toISOString(), + receivedAt: value.receivedAt }); const occupied = telemetryEvidence(server); if (occupied && occupied.players > 0) return fromTelemetry(occupied); const players = await rconGetPlayerCount(server).catch(() => null); // Read after the network request: a player may join while RCON is pending. const telemetry = telemetryEvidence(server); - if (players !== null && players > 0) return { players, source: 'rcon', observedAt: new Date().toISOString() }; + if (players !== null && players > 0) + return { players, source: 'rcon', observedAt: new Date().toISOString() }; if (telemetry && telemetry.players > 0) return fromTelemetry(telemetry); if (players === 0) return { players, source: 'rcon', observedAt: new Date().toISOString() }; if (telemetry?.players === 0) return fromTelemetry(telemetry); return { players: null, source: null, observedAt: new Date().toISOString() }; } -export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & ( - { source: 'rcon'; rcon: Extract } - | { source: 'telemetry'; receivedAt: string }); +export type VerifiedGameStatus = { + ok: true; + players: number; + map: string | null; + observedAt: string; +} & ( + | { source: 'rcon'; rcon: Extract } + | { source: 'telemetry'; receivedAt: string } +); /** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */ -export async function probeServerStatus(server: string, after?: string | null): Promise< - VerifiedGameStatus | { ok: false; reason: string; observedAt: string } -> { +export async function probeServerStatus( + server: string, + after?: string | null +): Promise { const current = telemetryEvidence(server, after); - if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() }; - let reason = 'RCON unavailable'; + if (current) + return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() }; + let reason: string; try { const rcon = await rconStatusResult(server); const latest = telemetryEvidence(server, after); - if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() }; - if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() }; + if (latest) + return { + ok: true, + ...latest, + source: 'telemetry', + observedAt: new Date().toISOString() + }; + if (rcon.ok) + return { + ok: true, + players: rcon.status.players, + map: rcon.status.map, + source: 'rcon', + rcon, + observedAt: new Date().toISOString() + }; reason = rcon.reason; - } catch (err) { reason = (err as Error).message; } + } catch (err) { + reason = (err as Error).message; + } const telemetry = telemetryEvidence(server, after); - return telemetry ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() } + return telemetry + ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() } : { ok: false, reason, observedAt: new Date().toISOString() }; } diff --git a/src/lib/server/fleet/status-updates.ts b/src/lib/server/fleet/status-updates.ts index 225aa99..653670c 100644 --- a/src/lib/server/fleet/status-updates.ts +++ b/src/lib/server/fleet/status-updates.ts @@ -3,12 +3,20 @@ import type { ServerObservation } from '../../server-observation.js'; import type { ServerRow } from './servers.js'; /** Monitoring and provisioning publish the same complete RCON observation. */ -export function successfulStatusFields(result: Extract, observedAt: string): Partial { +export function successfulStatusFields( + result: Extract, + observedAt: string +): Partial { return { - last_checked_at: observedAt, last_seen_at: observedAt, - last_players: result.status.players, last_map: result.status.map, - last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster), - last_reply_snippet: null, last_error: null, rcon_failures: 0 + last_checked_at: observedAt, + last_seen_at: observedAt, + last_players: result.status.players, + last_map: result.status.map, + last_gametype: result.status.gametype, + last_roster: JSON.stringify(result.status.roster), + last_reply_snippet: null, + last_error: null, + rcon_failures: 0 }; } @@ -17,6 +25,10 @@ export function activityFields(current: ServerRow, status: ServerObservation): P if (status.health !== 'online' || status.last_players === null) return {}; return status.last_players > 0 ? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null } - : { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at, - game_started_at: null, game_ended_at: null }; + : { + active_since: null, + inactive_since: current.inactive_since ?? status.last_seen_at, + game_started_at: null, + game_ended_at: null + }; } diff --git a/src/lib/server/fleet/status.ts b/src/lib/server/fleet/status.ts index a0752fc..ff5a782 100644 --- a/src/lib/server/fleet/status.ts +++ b/src/lib/server/fleet/status.ts @@ -1,4 +1,8 @@ -import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js'; +import { + rconHealthError, + selectServerObservation, + type GameObservation +} from '../../server-observation.js'; import { classifyHealth } from './health.js'; import { isProvisioning } from './provisioning.js'; import { restartSettling, recoverySettling } from '../provisioning/restarts.js'; @@ -8,32 +12,71 @@ import { latestGameSubmissions } from './lifecycle-evidence.js'; import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; -interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null } +interface Evidence { + provisioning: boolean; + settling: boolean; + telemetry: GameTelemetry | null; + submission: string | null; +} /** One set of evidence reads per snapshot, rather than four queries per server. */ export function observationReader() { const handle = db(); - const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments - WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as - { id: string; server: string; action: string; state: string }[]).map(row => [row.server, row])); - const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as - { server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row])); + const operations = new Map( + ( + handle + .prepare( + `SELECT id, server, action, state FROM server_deployments + WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)` + ) + .all() as { id: string; server: string; action: string; state: string }[] + ).map((row) => [row.server, row]) + ); + const restarts = new Map( + ( + handle + .prepare( + 'SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout' + ) + .all() as { + server: string; + requested_at: string; + healthy_at: string | null; + operation_id: string | null; + }[] + ).map((row) => [row.server, row]) + ); const submissions = latestGameSubmissions(); const reports = new Map(); - for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as - { server: string; received_at: string; payload: string }[]) { + for (const row of handle + .prepare( + 'SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL' + ) + .all() as { server: string; received_at: string; payload: string }[]) { try { const saved = JSON.parse(row.payload) as GameTelemetry; const event = parseGameEvent(saved); - if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing }); - } catch { /* Invalid retained telemetry is unavailable. */ } + if (event.server === row.server) + reports.set(row.server, { + ...event, + receivedAt: row.received_at, + timing: saved.timing + }); + } catch { + /* Invalid retained telemetry is unavailable. */ + } } return (server: ServerRow) => { - const operation = operations.get(server.name), restart = restarts.get(server.name); + const operation = operations.get(server.name), + restart = restarts.get(server.name); return serverObservation(server, { - provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state), + provisioning: + !!operation && + ['create', 'rebuild', 'start'].includes(operation.action) && + ['queued', 'running', 'awaiting-health'].includes(operation.state), settling: recoverySettling(restart, operation), - telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null + telemetry: reports.get(server.name) ?? null, + submission: submissions.get(server.name) ?? null }); }; } @@ -41,22 +84,47 @@ export function observationReader() { /** Shared snapshot for REST, status events, metrics, and other server-side readers. */ export function serverObservation(server: ServerRow, evidence?: Evidence) { let roster = null; - try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ } - const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map, - last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, - rcon_failures: server.rcon_failures, roster }; + try { + roster = server.last_roster ? JSON.parse(server.last_roster) : null; + } catch { + /* unavailable */ + } + const rcon: GameObservation = { + last_players: server.last_players, + last_map: server.last_map, + last_gametype: server.last_gametype, + last_seen_at: server.last_seen_at, + last_error: server.last_error, + rcon_failures: server.rcon_failures, + roster + }; const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name); - const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state; + const state = (evidence ? evidence.settling : restartSettling(server.name)) + ? 'restarting' + : server.container_state; const ready = !provisioning && (!state || state === 'running' || state === 'unknown'); - const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, rconHealthError(rcon), state); + const health = provisioning + ? 'provisioning' + : classifyHealth(server.last_seen_at, rconHealthError(rcon), state); // This projection is shared by viewer-visible snapshots. Never spread saved diagnostics. const { last_reply_snippet: _snippet, ...publicServer } = server; - return selectServerObservation({ ...publicServer, ...rcon, rcon_status: rcon, container_state: state, health, - status_source: null as 'telemetry' | 'rcon' | null, - telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined, - telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name), - last_error: ready ? server.last_error : null, - active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null, - game_started_at: ready ? server.game_started_at : null, game_ended_at: ready ? server.game_ended_at : null - }, undefined, Date.now()); + return selectServerObservation( + { + ...publicServer, + ...rcon, + rcon_status: rcon, + container_state: state, + health, + status_source: null as 'telemetry' | 'rcon' | null, + telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined, + telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name), + last_error: ready ? server.last_error : null, + active_since: ready ? server.active_since : null, + inactive_since: ready ? server.inactive_since : null, + game_started_at: ready ? server.game_started_at : null, + game_ended_at: ready ? server.game_ended_at : null + }, + undefined, + Date.now() + ); } diff --git a/src/lib/server/fleet/telemetry-requests.ts b/src/lib/server/fleet/telemetry-requests.ts index 1a2902f..bf76e26 100644 --- a/src/lib/server/fleet/telemetry-requests.ts +++ b/src/lib/server/fleet/telemetry-requests.ts @@ -6,11 +6,18 @@ const MAX_PAYLOAD_LENGTH = 16 * 1024; // Bound storage even when requests claim many different, unregistered names. export function telemetryRetention() { const settings = fleetSettings(); - return { maxAgeDays: settings.telemetryKeepDays, maxPerServer: settings.telemetryMaxPerServer, maxTotal: settings.telemetryMaxTotal }; + return { + maxAgeDays: settings.telemetryKeepDays, + maxPerServer: settings.telemetryMaxPerServer, + maxTotal: settings.telemetryMaxTotal + }; } -export const telemetryCutoff = () => telemetryRetention().maxAgeDays ? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString() : ''; +export const telemetryCutoff = () => + telemetryRetention().maxAgeDays + ? new Date(Date.now() - telemetryRetention().maxAgeDays * 86_400_000).toISOString() + : ''; const cutoff = telemetryCutoff; -const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null; +const textField = (value: unknown) => (typeof value === 'string' ? value.slice(0, 255) : null); export function recordTelemetryRequest(input: { value: unknown; @@ -27,29 +34,55 @@ export function recordTelemetryRequest(input: { const payload = JSON.stringify(body); const handle = db(); handle.transaction(() => { - handle.prepare(`INSERT INTO telemetry_requests + handle + .prepare( + `INSERT INTO telemetry_requests (server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run( - body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null, - textField(body.event), textField(body.map), - Number.isSafeInteger(body.round) ? body.round as number : null, - Array.isArray(body.players) ? body.players.length : null, - textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH) - ); - if (telemetryRetention().maxAgeDays) handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff()); - if (telemetryRetention().maxPerServer) handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= ( + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + body.server, + input.receivedAt, + input.status, + input.error?.slice(0, 2000) ?? null, + textField(body.event), + textField(body.map), + Number.isSafeInteger(body.round) ? (body.round as number) : null, + Array.isArray(body.players) ? body.players.length : null, + textField(input.contentType), + payload.slice(0, MAX_PAYLOAD_LENGTH), + Number(payload.length > MAX_PAYLOAD_LENGTH) + ); + if (telemetryRetention().maxAgeDays) + handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff()); + if (telemetryRetention().maxPerServer) + handle + .prepare( + `DELETE FROM telemetry_requests WHERE server = ? AND id <= ( SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ? - )`).run(body.server, body.server, telemetryRetention().maxPerServer); - if (telemetryRetention().maxTotal) handle.prepare(`DELETE FROM telemetry_requests WHERE id <= ( + )` + ) + .run(body.server, body.server, telemetryRetention().maxPerServer); + if (telemetryRetention().maxTotal) + handle + .prepare( + `DELETE FROM telemetry_requests WHERE id <= ( SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ? - )`).run(telemetryRetention().maxTotal); + )` + ) + .run(telemetryRetention().maxTotal); })(); } export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] { - const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round, + const rows = db() + .prepare( + `SELECT id, server, received_at AS receivedAt, status, error, event, map, round, player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated - FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`) - .all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array & { payloadTruncated: number }>; - return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated })); + FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?` + ) + .all(server, cutoff(), telemetryRetention().maxPerServer || 10000) as Array< + Omit & { payloadTruncated: number } + >; + return rows.map((row) => ({ ...row, payloadTruncated: !!row.payloadTruncated })); } diff --git a/src/lib/server/fleet/telemetry.ts b/src/lib/server/fleet/telemetry.ts index a295233..0701df8 100644 --- a/src/lib/server/fleet/telemetry.ts +++ b/src/lib/server/fleet/telemetry.ts @@ -5,58 +5,119 @@ import { recordLeaderboardTelemetry } from './leaderboard-ingest.js'; import { latestGameSubmission } from './status-evidence.js'; export function listTelemetry(): GameTelemetry[] { - const rows = db().prepare(`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server - WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`).all() as Array<{ payload: string }>; - return rows.map(row => JSON.parse(row.payload) as GameTelemetry); + const rows = db() + .prepare( + `SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server + WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL` + ) + .all() as Array<{ payload: string }>; + return rows.map((row) => JSON.parse(row.payload) as GameTelemetry); } export function saveTelemetry(report: GameTelemetry): GameTelemetry { // Keep callers outside the HTTP route atomic too. const handle = db(); return handle.transaction(() => { - const row = handle.prepare('SELECT payload FROM server_telemetry WHERE server = ?').get(report.server) as { payload: string | null } | undefined; - const previous: GameTelemetry | undefined = row?.payload ? JSON.parse(row.payload) : undefined; + const row = handle + .prepare('SELECT payload FROM server_telemetry WHERE server = ?') + .get(report.server) as { payload: string | null } | undefined; + const previous: GameTelemetry | undefined = row?.payload + ? JSON.parse(row.payload) + : undefined; recordLeaderboardTelemetry(handle, report); // Record each occupancy transition at receipt, before a heartbeat can // replace game_start or an empty report between status polls. const after = latestGameSubmission(report.server); if (!after || Date.parse(report.receivedAt) > Date.parse(after)) { if (report.players.length > 0) { - handle.prepare('UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?') + handle + .prepare( + 'UPDATE servers SET active_since = COALESCE(active_since, ?), inactive_since = NULL WHERE name = ?' + ) .run(report.receivedAt, report.server); if (report.event === 'game_start') { - handle.prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server); + handle + .prepare( + 'UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?' + ) + .run(report.receivedAt, report.server); } else if (report.event === 'game_end') { - handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server); + handle + .prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?') + .run(report.receivedAt, report.server); } } else { - handle.prepare(`UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?), - game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(report.receivedAt, report.server); + handle + .prepare( + `UPDATE servers SET active_since = NULL, inactive_since = COALESCE(inactive_since, ?), + game_started_at = NULL, game_ended_at = NULL WHERE name = ?` + ) + .run(report.receivedAt, report.server); } } - const serverTiming = handle.prepare(`SELECT active_since AS activeSince, inactive_since AS inactiveSince, - game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?`).get(report.server) as ServerTiming; - const match = handle.prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1').get(report.server) as { id: number }; + const serverTiming = handle + .prepare( + `SELECT active_since AS activeSince, inactive_since AS inactiveSince, + game_started_at AS gameStartedAt, game_ended_at AS gameEndedAt FROM servers WHERE name = ?` + ) + .get(report.server) as ServerTiming; + const match = handle + .prepare('SELECT id FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1') + .get(report.server) as { id: number }; const sameMatch = previous?.timing?.matchId === match.id; // A gap beyond the normal heartbeat window cannot establish continuity. - const continuous = previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000; - report = { ...report, timing: { matchId: match.id, server: serverTiming, players: Object.fromEntries(report.players.map(player => { - const prior = continuous ? previous?.timing?.players[player.guid] : undefined; - return [player.guid, { - serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt, - matchJoinedAt: sameMatch && prior ? prior.matchJoinedAt : report.receivedAt, - roundStarted: sameMatch && prior ? prior.roundStarted : report.event === 'game_start' ? 1 : report.round - }]; - })) } }; - handle.prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload) + const continuous = + previous && Date.parse(report.receivedAt) - Date.parse(previous.receivedAt) <= 120_000; + report = { + ...report, + timing: { + matchId: match.id, + server: serverTiming, + players: Object.fromEntries( + report.players.map((player) => { + const prior = continuous + ? previous?.timing?.players[player.guid] + : undefined; + return [ + player.guid, + { + serverJoinedAt: prior?.serverJoinedAt ?? report.receivedAt, + matchJoinedAt: + sameMatch && prior ? prior.matchJoinedAt : report.receivedAt, + roundStarted: + sameMatch && prior + ? prior.roundStarted + : report.event === 'game_start' + ? 1 + : report.round + } + ]; + }) + ) + } + }; + handle + .prepare( + `INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET - round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`) - .run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report)); + round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload` + ) + .run( + report.server, + report.event === 'game_start' ? 0 : report.round, + report.players.length, + report.receivedAt, + JSON.stringify(report) + ); if (report.event === 'game_end') { // Legacy betas also take match completion directly from the script. - if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match' - AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`) - .run(report.receivedAt, report.server, report.receivedAt); + if (report.round >= MIN_RECORDED_ROUND) + handle + .prepare( + `UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match' + AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')` + ) + .run(report.receivedAt, report.server, report.receivedAt); } return report; })(); diff --git a/src/lib/server/game-files/http.ts b/src/lib/server/game-files/http.ts index 1878189..43871e3 100644 --- a/src/lib/server/game-files/http.ts +++ b/src/lib/server/game-files/http.ts @@ -6,41 +6,98 @@ import { getLibraryFile, libraryFilesRetired } from './index.js'; import { assertRegularPath, parseByteRange } from './storage.js'; export function gameFileError(cause: unknown): Response { - const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400; - const receivedBytes = typeof cause === 'object' && cause !== null && 'receivedBytes' in cause && Number.isSafeInteger(cause.receivedBytes) ? cause.receivedBytes : undefined; - return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.', ...(receivedBytes === undefined ? {} : { receivedBytes }) }, { status }); + const status = + typeof cause === 'object' && + cause !== null && + 'status' in cause && + typeof cause.status === 'number' + ? cause.status + : 400; + const receivedBytes = + typeof cause === 'object' && + cause !== null && + 'receivedBytes' in cause && + Number.isSafeInteger(cause.receivedBytes) + ? cause.receivedBytes + : undefined; + return json( + { + error: cause instanceof Error ? cause.message : 'Game file request failed.', + ...(receivedBytes === undefined ? {} : { receivedBytes }) + }, + { status } + ); } -export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise> { +export async function readGameJson( + request: Request, + maxBytes = 64 * 1024 +): Promise> { if (!request.body) throw new Error('A JSON request body is required.'); const reader = request.body.getReader(); - const chunks: Uint8Array[] = []; let bytes = 0; + const chunks: Uint8Array[] = []; + let bytes = 0; try { while (true) { - const next = await reader.read(); if (next.done) break; + const next = await reader.read(); + if (next.done) break; bytes += next.value.byteLength; - if (bytes > maxBytes) { await reader.cancel(); throw new Error('Request body exceeds its size limit.'); } + if (bytes > maxBytes) { + await reader.cancel(); + throw new Error('Request body exceeds its size limit.'); + } chunks.push(next.value); } - } finally { reader.releaseLock(); } + } finally { + reader.releaseLock(); + } const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); - if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Expected a JSON object.'); + if (!body || typeof body !== 'object' || Array.isArray(body)) + throw new Error('Expected a JSON object.'); return body; } -export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise { - if (libraryFilesRetired(versionId)) return json({ error: 'These outdated game files have been removed.' }, { status: 410 }); +export async function serveGameFile( + request: Request, + versionId: string, + requestedPath: string +): Promise { + if (libraryFilesRetired(versionId)) + return json({ error: 'These outdated game files have been removed.' }, { status: 410 }); const file = getLibraryFile(versionId, requestedPath); - if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 }); + if (!file || file.state !== 'ready') + return json({ error: 'File is not in a published manifest.' }, { status: 404 }); const path = await assertRegularPath(file.storageLocation, file.path); - if ((await stat(path)).size !== file.size) return json({ error: 'Stored file failed size verification.' }, { status: 503 }); + if ((await stat(path)).size !== file.size) + return json({ error: 'Stored file failed size verification.' }, { status: 503 }); const etag = `"${file.sha256}"`; - const headers = new Headers({ 'Content-Type': 'application/octet-stream', 'Accept-Ranges': 'bytes', ETag: etag, 'Cache-Control': 'private, no-cache', 'X-Content-SHA256': file.sha256 }); - if (request.headers.get('if-none-match') === etag) return new Response(null, { status: 304, headers }); + const headers = new Headers({ + 'Content-Type': 'application/octet-stream', + 'Accept-Ranges': 'bytes', + ETag: etag, + 'Cache-Control': 'private, no-cache', + 'X-Content-SHA256': file.sha256 + }); + if (request.headers.get('if-none-match') === etag) + return new Response(null, { status: 304, headers }); let range; - try { range = parseByteRange(request.headers.has('if-range') && request.headers.get('if-range') !== etag ? null : request.headers.get('range'), file.size); } - catch { headers.set('Content-Range', `bytes */${file.size}`); return new Response(null, { status: 416, headers }); } + try { + range = parseByteRange( + request.headers.has('if-range') && request.headers.get('if-range') !== etag + ? null + : request.headers.get('range'), + file.size + ); + } catch { + headers.set('Content-Range', `bytes */${file.size}`); + return new Response(null, { status: 416, headers }); + } headers.set('Content-Length', String(range ? range.end - range.start + 1 : file.size)); if (range) headers.set('Content-Range', `bytes ${range.start}-${range.end}/${file.size}`); - const body = request.method === 'HEAD' ? null : Readable.toWeb(createReadStream(path, range ?? undefined)) as ReadableStream; + const body = + request.method === 'HEAD' + ? null + : (Readable.toWeb( + createReadStream(path, range ?? undefined) + ) as ReadableStream); return new Response(body, { status: range ? 206 : 200, headers }); } diff --git a/src/lib/server/game-files/index.ts b/src/lib/server/game-files/index.ts index 58ee6b6..bb8bf2b 100644 --- a/src/lib/server/game-files/index.ts +++ b/src/lib/server/game-files/index.ts @@ -7,46 +7,175 @@ import { config } from '../config.js'; import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js'; import { ensureGameFileSchema } from './schema.js'; import { ensureHostSchema } from '../hosts/schema.js'; -import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js'; -import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock, uploadIsActive } from './uploads.js'; +import { + manifestChecksum, + receiveGameFile, + validateGamePath, + validateManifest, + verifyGameFile +} from './storage.js'; +import { + MAX_UPLOAD_CHUNK_BYTES, + validateUploadPlan, + getUploadFileProgress, + receiveUploadChunk, + hashUploadedFiles, + withUploadLock, + uploadIsActive +} from './uploads.js'; import { GAME_FILE_GAMES } from '../../game-files.js'; -import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js'; +import type { + GameFileEntry, + GameLibrary, + GameLibraryGame, + GameLibraryVersion, + HostLibraryCache, + LibraryCacheState, + LibraryTransferProgress +} from '../../game-files.js'; -function database() { const handle = db(); ensureGameFileSchema(handle); ensureHostSchema(handle); return handle; } -export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files')); +function database() { + const handle = db(); + ensureGameFileSchema(handle); + ensureHostSchema(handle); + return handle; +} +export const gameFilesRoot = () => + resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files')); const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id); -type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null }; -type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null; bytes_verified: number | null; transfer_progress: string | null }; -const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id, +type VersionRow = { + id: string; + library_id: string; + game: GameLibraryGame; + version: string; + description: string; + dlc: string; + state: GameLibraryVersion['state']; + size_bytes: number; + manifest: string; + manifest_sha256: string | null; + created_at: string; + published_at: string | null; + storage_location: string; + source_host_id: string; + source_volume: string; + error: string | null; + operation_id: string | null; +}; +type CacheRow = { + host_id: string; + version_id: string; + status: LibraryCacheState; + bytes_downloaded: number; + bytes_total: number; + manifest_sha256: string | null; + error: string | null; + updated_at: string; + operation_id: string | null; + bytes_verified: number | null; + transfer_progress: string | null; +}; +const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ + hostId: row.host_id, + libraryVersionId: row.version_id, + status: row.status, + bytesDownloaded: row.bytes_downloaded, + bytesTotal: row.bytes_total, + manifestSha256: row.manifest_sha256, + error: row.error, + updatedAt: row.updated_at, + operationId: row.operation_id, ...(row.bytes_verified == null ? {} : { bytesVerified: row.bytes_verified }), - ...(row.transfer_progress ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } : {}) }); -function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches?: Map): GameLibraryVersion { - const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[]; - return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version, - description: row.description, state: row.state, sizeBytes: row.size_bytes, - filesRemoved: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NOT NULL').get(row.id), - filesRemoving: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NULL').get(row.id), - files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at, - publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null, - sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id, - caches: summaryCaches ? summaryCaches.get(row.id) ?? [] : (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) }; + ...(row.transfer_progress + ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } + : {}) +}); +function versionFromRow( + row: VersionRow & { file_count?: number }, + summaryCaches?: Map +): GameLibraryVersion { + const files = summaryCaches ? [] : (JSON.parse(row.manifest) as GameFileEntry[]); + return { + id: row.id, + libraryId: row.library_id, + game: row.game, + version: row.version, + description: row.description, + state: row.state, + sizeBytes: row.size_bytes, + filesRemoved: !!database() + .prepare( + 'SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NOT NULL' + ) + .get(row.id), + filesRemoving: !!database() + .prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NULL') + .get(row.id), + files, + fileCount: + row.file_count ?? + (files.length || + ( + database() + .prepare( + 'SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?' + ) + .get(row.id) as { count: number } + ).count), + manifestSha256: row.manifest_sha256, + createdAt: row.created_at, + publishedAt: row.published_at, + storageLocation: row.storage_location, + sourceHostId: row.source_host_id || null, + sourceVolume: row.source_volume || null, + error: row.error, + operationId: row.operation_id, + caches: summaryCaches + ? (summaryCaches.get(row.id) ?? []) + : ( + database() + .prepare('SELECT * FROM host_library_caches WHERE version_id = ?') + .all(row.id) as CacheRow[] + ).map(cacheFromRow) + }; } export function getLibraryVersion(id: string): GameLibraryVersion | null { - const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined; + const row = database() + .prepare( + 'SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?' + ) + .get(id) as VersionRow | undefined; return row ? versionFromRow(row) : null; } export function libraryFilesRetired(versionId: string): boolean { - return !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(versionId); + return !!database() + .prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?') + .get(versionId); } export function getLibraryFile(versionId: string, path: string) { validateGamePath(path); - return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation - FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`) - .get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null; + return ( + (database() + .prepare( + `SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation + FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?` + ) + .get(versionId, path) as + | (GameFileEntry & { + state: GameLibraryVersion['state']; + sourceHostId: string; + storageLocation: string; + }) + | undefined) ?? null + ); } /** Launch selection needs only the ID; manifests can contain tens of thousands of files. */ export function newestReadyLibraryId(game: string): string | null { - const row = database().prepare("SELECT v.id FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as { id: string } | undefined; + const row = database() + .prepare( + "SELECT v.id FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1" + ) + .get(game) as { id: string } | undefined; return row?.id ?? null; } export function newestReadyLibrary(game: string): GameLibraryVersion | null { @@ -55,12 +184,22 @@ export function newestReadyLibrary(game: string): GameLibraryVersion | null { } /** Hosts follow the latest successful publication for each game, including legacy libraries. */ export function latestPublishedLibraries() { - const latest = database().prepare(`SELECT v.id, l.game, v.version, v.size_bytes AS sizeBytes, v.manifest_sha256 AS manifestSha256, v.published_at AS publishedAt + const latest = database() + .prepare(`SELECT v.id, l.game, v.version, v.size_bytes AS sizeBytes, v.manifest_sha256 AS manifestSha256, v.published_at AS publishedAt FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.state='ready' AND l.game=? ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC LIMIT 1`); - return GAME_FILE_GAMES.flatMap(game => { - const version = latest.get(game.id) as { id: string; game: GameLibraryGame; version: string; sizeBytes: number; manifestSha256: string; publishedAt: string | null } | undefined; + return GAME_FILE_GAMES.flatMap((game) => { + const version = latest.get(game.id) as + | { + id: string; + game: GameLibraryGame; + version: string; + sizeBytes: number; + manifestSha256: string; + publishedAt: string | null; + } + | undefined; return version ? [{ ...version, name: game.name }] : []; }); } @@ -69,32 +208,75 @@ export function listLibraries(summary = false): GameLibrary[] { let summaryCaches: Map | undefined; if (summary) { summaryCaches = new Map(); - for (const row of database().prepare('SELECT * FROM host_library_caches').all() as CacheRow[]) { - const items = summaryCaches.get(row.version_id) ?? []; items.push(cacheFromRow(row)); summaryCaches.set(row.version_id, items); + for (const row of database() + .prepare('SELECT * FROM host_library_caches') + .all() as CacheRow[]) { + const items = summaryCaches.get(row.version_id) ?? []; + items.push(cacheFromRow(row)); + summaryCaches.set(row.version_id, items); } } - const latest = new Map(latestPublishedLibraries().map(version => [version.game, version.id])); - return GAME_FILE_GAMES.map(game => { - const versions = (database().prepare(`SELECT ${summary ? `v.id, v.library_id, v.version, v.description, v.dlc, v.state, v.size_bytes, v.manifest_sha256, v.created_at, v.published_at, v.storage_location, v.source_host_id, v.source_volume, v.error, v.operation_id, CASE WHEN json_array_length(v.manifest)>0 THEN json_array_length(v.manifest) ELSE (SELECT COUNT(*) FROM game_file_upload_entries e WHERE e.version_id=v.id) END AS file_count` : 'v.*'}, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id - WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(row => versionFromRow(row, summaryCaches)); - const firstLibrary = database().prepare('SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game.id) as { created_at: string } | undefined; - return { id: game.id, name: game.name, game: game.id, description: '', createdAt: firstLibrary?.created_at ?? '', latestVersionId: latest.get(game.id) ?? null, versions }; + const latest = new Map(latestPublishedLibraries().map((version) => [version.game, version.id])); + return GAME_FILE_GAMES.map((game) => { + const versions = ( + database() + .prepare( + `SELECT ${summary ? `v.id, v.library_id, v.version, v.description, v.dlc, v.state, v.size_bytes, v.manifest_sha256, v.created_at, v.published_at, v.storage_location, v.source_host_id, v.source_volume, v.error, v.operation_id, CASE WHEN json_array_length(v.manifest)>0 THEN json_array_length(v.manifest) ELSE (SELECT COUNT(*) FROM game_file_upload_entries e WHERE e.version_id=v.id) END AS file_count` : 'v.*'}, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id + WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC` + ) + .all(game.id) as VersionRow[] + ).map((row) => versionFromRow(row, summaryCaches)); + const firstLibrary = database() + .prepare( + 'SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1' + ) + .get(game.id) as { created_at: string } | undefined; + return { + id: game.id, + name: game.name, + game: game.id, + description: '', + createdAt: firstLibrary?.created_at ?? '', + latestVersionId: latest.get(game.id) ?? null, + versions + }; }); } export function getHostLibraryCache(hostId: string, versionId: string): HostLibraryCache | null { - const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined; + const row = database() + .prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?') + .get(hostId, versionId) as CacheRow | undefined; return row ? cacheFromRow(row) : null; } /** Admission and dispatch recheck readiness without loading or parsing the file manifest. */ export function assertReadyLibrary(hostId: string, versionId: string): void { const handle = database(); - const version = handle.prepare('SELECT state, manifest_sha256 FROM game_file_versions WHERE id=?') + const version = handle + .prepare('SELECT state, manifest_sha256 FROM game_file_versions WHERE id=?') .get(versionId) as Pick | undefined; - if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.'); - if (handle.prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.'); - const cache = handle.prepare('SELECT status, manifest_sha256 FROM host_library_caches WHERE host_id=? AND version_id=?') + if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) + throw new Error('Select an available published game-file version.'); + if ( + handle + .prepare( + "SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')" + ) + .get(hostId, versionId) + ) + throw new Error('This host copy is being removed.'); + const cache = handle + .prepare( + 'SELECT status, manifest_sha256 FROM host_library_caches WHERE host_id=? AND version_id=?' + ) .get(hostId, versionId) as Pick | undefined; - if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifest_sha256 !== version.manifest_sha256) throw new Error('The selected host must finish downloading and verifying this game-file version.'); + if ( + !cache || + !['ready', 'outdated'].includes(cache.status) || + cache.manifest_sha256 !== version.manifest_sha256 + ) + throw new Error( + 'The selected host must finish downloading and verifying this game-file version.' + ); } export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion { assertReadyLibrary(hostId, versionId); @@ -102,34 +284,59 @@ export function requireReadyLibrary(hostId: string, versionId: string): GameLibr } function textField(input: unknown, label: string, max: number, required = true): string { - if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`); + if (typeof input !== 'string' || input.length > max || (required && !input.trim())) + throw new Error(`Provide a valid ${label}.`); return input.trim(); } /** Reuse a game's original storage parent without rewriting immutable historical versions. */ function libraryForGame(game: GameLibraryGame, now: string): string { const handle = database(); - const existing = handle.prepare('SELECT id FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game) as { id: string } | undefined; + const existing = handle + .prepare( + 'SELECT id FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1' + ) + .get(game) as { id: string } | undefined; if (existing) return existing.id; const id = randomUUID(); // The legacy name column is globally unique, so its private value must not collide with an old user label. - handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(id, `game-files:${game}:${id}`, game, '', now); + handle + .prepare( + 'INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)' + ) + .run(id, `game-files:${game}:${id}`, game, '', now); return id; } /** Register a folder on the controller; no host or Docker volume is needed. */ export function createLibraryUpload(input: Record): GameLibraryVersion { const game = input.game; - if (game !== 't5' && game !== 't6') throw new Error('Choose Black Ops I or Black Ops II game files.'); + if (game !== 't5' && game !== 't6') + throw new Error('Choose Black Ops I or Black Ops II game files.'); const files = validateUploadPlan(input.files); const handle = database(); - const id = randomUUID(); const now = new Date().toISOString(); + const id = randomUUID(); + const now = new Date().toISOString(); handle.transaction(() => { const libraryId = libraryForGame(game, now); // Keep legacy non-null columns for existing databases and published versions. - handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume) - VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, libraryId, id, '', files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id)); - const insert = handle.prepare('INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)'); + handle + .prepare( + `INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume) + VALUES(?,?,?,?,'[]','importing',?,?,?,'','')` + ) + .run( + id, + libraryId, + id, + '', + files.reduce((sum, file) => sum + file.size, 0), + now, + join(gameFilesRoot(), game, id) + ); + const insert = handle.prepare( + 'INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)' + ); for (const file of files) insert.run(id, file.path, file.size, file.lastModified); })(); return getLibraryVersion(id)!; @@ -138,45 +345,94 @@ export function createLibraryUpload(input: Record): GameLibrary type UploadEntry = { path: string; size: number; lastModified: number }; function requireBrowserUpload(id: string): GameLibraryVersion { const version = getLibraryVersion(id); - if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 }); - if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) { + if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) + throw Object.assign(new Error('Upload files expired under the retention policy.'), { + status: 410 + }); + if ( + !version || + version.sourceHostId !== null || + !database() + .prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1') + .get(id) + ) { throw Object.assign(new Error('Folder upload not found.'), { status: 404 }); } return version; } function uploadEntries(id: string): UploadEntry[] { - return database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? ORDER BY path').all(id) as UploadEntry[]; + return database() + .prepare( + 'SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? ORDER BY path' + ) + .all(id) as UploadEntry[]; } async function uploadStorageRoot(version: GameLibraryVersion): Promise { // Recover publication interrupted after the atomic rename but before its DB commit. - try { if ((await stat(version.storageLocation)).isDirectory()) return version.storageLocation; } - catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; } + try { + if ((await stat(version.storageLocation)).isDirectory()) return version.storageLocation; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; + } return stagingRoot(version.id); } export async function browserUploadStatus(id: string) { const version = requireBrowserUpload(id); touchUpload(id); - const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const; + const status = + version.state === 'ready' + ? ('published' as const) + : publishing.has(id) + ? ('verifying' as const) + : version.state === 'failed' + ? ('failed' as const) + : ('pending' as const); const root = await uploadStorageRoot(version); const entries = uploadEntries(id); const files = []; // Avoid opening thousands of files concurrently for a large folder. - for (const file of entries) files.push({ ...file, ...(status === 'published' ? { receivedBytes: file.size, complete: true } : await getUploadFileProgress(root, file)) }); + for (const file of entries) + files.push({ + ...file, + ...(status === 'published' + ? { receivedBytes: file.size, complete: true } + : await getUploadFileProgress(root, file)) + }); const { files: _manifest, ...summary } = version; - return { status, chunkSize: MAX_UPLOAD_CHUNK_BYTES, files, version: summary, error: version.error }; + return { + status, + chunkSize: MAX_UPLOAD_CHUNK_BYTES, + files, + version: summary, + error: version.error + }; } -export async function uploadBrowserFile(id: string, path: string, chunk: { offset: number; size: number; sha256: string }, body: ReadableStream | null) { +export async function uploadBrowserFile( + id: string, + path: string, + chunk: { offset: number; size: number; sha256: string }, + body: ReadableStream | null +) { validateGamePath(path); const checkWritable = () => { const version = requireBrowserUpload(id); - if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 }); + if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) + throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 }); }; checkWritable(); touchUpload(id); - const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined; + const file = database() + .prepare( + 'SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?' + ) + .get(id, path) as UploadEntry | undefined; if (!file) throw new Error('File is not listed in this upload.'); const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable); - database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id); + database() + .prepare( + "UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')" + ) + .run(id); touchUpload(id); return progress; } @@ -184,35 +440,66 @@ export async function publishBrowserUpload(id: string): Promise { const root = await uploadStorageRoot(version); const files = await hashUploadedFiles(root, uploadEntries(id)); const checksum = manifestChecksum(files); - if (version.manifestSha256 && checksum !== version.manifestSha256) throw new Error('Stored files no longer match the verified manifest.'); + if (version.manifestSha256 && checksum !== version.manifestSha256) + throw new Error('Stored files no longer match the verified manifest.'); database().transaction(() => { - database().prepare("UPDATE game_file_versions SET state='importing',manifest=?,manifest_sha256=?,error=NULL WHERE id=? AND state IN ('importing','failed')").run(JSON.stringify(files), checksum, id); + database() + .prepare( + "UPDATE game_file_versions SET state='importing',manifest=?,manifest_sha256=?,error=NULL WHERE id=? AND state IN ('importing','failed')" + ) + .run(JSON.stringify(files), checksum, id); database().prepare('DELETE FROM game_file_entries WHERE version_id=?').run(id); - const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)'); + const insert = database().prepare( + 'INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)' + ); for (const file of files) insert.run(id, file.path, file.size, file.sha256); })(); if (root !== version.storageLocation) { await mkdir(dirname(version.storageLocation), { recursive: true }); await rename(root, version.storageLocation); } - database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id); + database() + .prepare( + "UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'" + ) + .run(new Date().toISOString(), id); return getLibraryVersion(id)!; }); publishing.set(id, task); - try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); } + try { + const result = await task; + await cleanupControllerLibraries(); + return result; + } finally { + publishing.delete(id); + } } export async function requestBrowserPublication(id: string) { const version = requireBrowserUpload(id); if (version.state !== 'ready' && !publishing.has(id)) { - database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id); - void publishBrowserUpload(id).catch(cause => { - database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'") - .run((cause instanceof Error ? cause.message : 'Folder publication failed.').slice(0, 2000), id); + database() + .prepare( + "UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')" + ) + .run(id); + void publishBrowserUpload(id).catch((cause) => { + database() + .prepare( + "UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'" + ) + .run( + (cause instanceof Error ? cause.message : 'Folder publication failed.').slice( + 0, + 2000 + ), + id + ); }); } return browserUploadStatus(id); @@ -220,26 +507,56 @@ export async function requestBrowserPublication(id: string) { export function createLibraryImport(input: Record): GameLibraryVersion { const game = input.game; - if (game !== 't5' && game !== 't6') throw new Error('Choose Black Ops I or Black Ops II game files.'); + if (game !== 't5' && game !== 't6') + throw new Error('Choose Black Ops I or Black Ops II game files.'); const hostId = textField(input.hostId, 'source host', 128); const host = getHost(hostId); - if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled source host.'); + if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') + throw new Error('Choose an online, enrolled source host.'); const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files'; const handle = database(); - const id = randomUUID(); const now = new Date().toISOString(); + const id = randomUUID(); + const now = new Date().toISOString(); handle.transaction(() => { const libraryId = libraryForGame(game, now); - handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) - VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, libraryId, id, '', '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume); + handle + .prepare( + `INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) + VALUES(?,?,?,?,?,'importing',?,?,?,?)` + ) + .run( + id, + libraryId, + id, + '', + '[]', + now, + join(gameFilesRoot(), game, id), + hostId, + sourceVolume + ); })(); try { - const operation = queueHostOperation(hostId, 'library.import', { - importId: id, game, sourceVolume, manifestUrl: `/api/agent/imports/${id}`, - uploadBaseUrl: `/api/agent/imports/${id}/files`, completeUrl: `/api/agent/imports/${id}/complete` - }, { idempotencyKey: `library-import:${id}` }); - handle.prepare('UPDATE game_file_versions SET operation_id=? WHERE id=?').run(operation.id, id); + const operation = queueHostOperation( + hostId, + 'library.import', + { + importId: id, + game, + sourceVolume, + manifestUrl: `/api/agent/imports/${id}`, + uploadBaseUrl: `/api/agent/imports/${id}/files`, + completeUrl: `/api/agent/imports/${id}/complete` + }, + { idempotencyKey: `library-import:${id}` } + ); + handle + .prepare('UPDATE game_file_versions SET operation_id=? WHERE id=?') + .run(operation.id, id); } catch (cause) { - handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=?").run((cause as Error).message, id); + handle + .prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=?") + .run((cause as Error).message, id); throw cause; } return getLibraryVersion(id)!; @@ -247,29 +564,53 @@ export function createLibraryImport(input: Record): GameLibrary export function requireImportOwner(hostId: string, id: string): GameLibraryVersion { const version = getLibraryVersion(id); - if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 }); - if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.'); + if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) + throw Object.assign(new Error('Upload files expired under the retention policy.'), { + status: 410 + }); + if (!version || version.sourceHostId !== hostId) + throw new Error('Import not found for this host.'); return version; } export function setImportManifest(hostId: string, id: string, input: unknown): GameLibraryVersion { const version = requireImportOwner(hostId, id); - const files = validateManifest(input); const checksum = manifestChecksum(files); + const files = validateManifest(input); + const checksum = manifestChecksum(files); if (version.manifestSha256) { - if (version.manifestSha256 !== checksum) throw new Error('An import manifest cannot change. Start a new version.'); + if (version.manifestSha256 !== checksum) + throw new Error('An import manifest cannot change. Start a new version.'); return version; } if (version.state !== 'importing') throw new Error('This import is not writable.'); database().transaction(() => { - database().prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=?,size_bytes=? WHERE id=? AND state=\'importing\'').run(JSON.stringify(files), checksum, files.reduce((sum, file) => sum + file.size, 0), id); - const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)'); + database() + .prepare( + "UPDATE game_file_versions SET manifest=?,manifest_sha256=?,size_bytes=? WHERE id=? AND state='importing'" + ) + .run( + JSON.stringify(files), + checksum, + files.reduce((sum, file) => sum + file.size, 0), + id + ); + const insert = database().prepare( + 'INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)' + ); for (const file of files) insert.run(id, file.path, file.size, file.sha256); })(); return getLibraryVersion(id)!; } -export async function uploadImportFile(hostId: string, id: string, path: string, body: ReadableStream | null): Promise { +export async function uploadImportFile( + hostId: string, + id: string, + path: string, + body: ReadableStream | null +): Promise { const file = getLibraryFile(id, path); - if (!file || file.sourceHostId !== hostId) throw new Error('File is not listed in a manifest belonging to this host.'); - if (file.state !== 'importing' || publishing.has(id)) throw new Error('This import is not writable.'); + if (!file || file.sourceHostId !== hostId) + throw new Error('File is not listed in a manifest belonging to this host.'); + if (file.state !== 'importing' || publishing.has(id)) + throw new Error('This import is not writable.'); await receiveGameFile(stagingRoot(id), file, body); } @@ -277,78 +618,143 @@ const publishing = new Map>(); export async function publishImport(hostId: string, id: string): Promise { const version = requireImportOwner(hostId, id); if (version.state === 'ready') return version; - if (version.state !== 'importing' || !version.files.length || !version.manifestSha256) throw new Error('Import needs a complete manifest before publication.'); - const existing = publishing.get(id); if (existing) return existing; + if (version.state !== 'importing' || !version.files.length || !version.manifestSha256) + throw new Error('Import needs a complete manifest before publication.'); + const existing = publishing.get(id); + if (existing) return existing; const task = (async () => { let root = stagingRoot(id); // A restart between the atomic directory rename and metadata commit is recoverable. - try { if ((await stat(version.storageLocation)).isDirectory()) root = version.storageLocation; } - catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; } + try { + if ((await stat(version.storageLocation)).isDirectory()) root = version.storageLocation; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; + } for (const file of version.files) await verifyGameFile(root, file); if (root !== version.storageLocation) { await mkdir(dirname(version.storageLocation), { recursive: true }); await rename(root, version.storageLocation); } - database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id); + database() + .prepare( + "UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'" + ) + .run(new Date().toISOString(), id); return getLibraryVersion(id)!; })(); publishing.set(id, task); - try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); } + try { + const result = await task; + await cleanupControllerLibraries(); + return result; + } finally { + publishing.delete(id); + } } /** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */ export function importPublicationStatus(hostId: string, id: string) { const version = requireImportOwner(hostId, id); return { - status: version.state === 'ready' ? 'published' as const : version.state === 'failed' ? 'failed' as const : publishing.has(id) ? 'verifying' as const : 'pending' as const, - version: { id: version.id, game: version.game, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 }, + status: + version.state === 'ready' + ? ('published' as const) + : version.state === 'failed' + ? ('failed' as const) + : publishing.has(id) + ? ('verifying' as const) + : ('pending' as const), + version: { + id: version.id, + game: version.game, + sizeBytes: version.sizeBytes, + manifestSha256: version.manifestSha256 + }, error: version.error }; } export function requestImportPublication(hostId: string, id: string) { const version = requireImportOwner(hostId, id); if (version.state === 'importing' && !publishing.has(id)) { - if (!version.files.length || !version.manifestSha256) throw new Error('Import requires a manifest before publication.'); - void publishImport(hostId, id).catch(cause => { - database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'") - .run((cause instanceof Error ? cause.message : 'Library publication failed.').slice(0, 2000), id); + if (!version.files.length || !version.manifestSha256) + throw new Error('Import requires a manifest before publication.'); + void publishImport(hostId, id).catch((cause) => { + database() + .prepare( + "UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'" + ) + .run( + (cause instanceof Error ? cause.message : 'Library publication failed.').slice( + 0, + 2000 + ), + id + ); }); } return importPublicationStatus(hostId, id); } -function transferProgress(value: unknown, status: LibraryCacheState): LibraryTransferProgress | null { +function transferProgress( + value: unknown, + status: LibraryCacheState +): LibraryTransferProgress | null { if (value === null || value === undefined) return null; - if (typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid transfer progress.'); + if (typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid transfer progress.'); const progress = value as Record; - const nonnegative = (amount: unknown): amount is number => typeof amount === 'number' && Number.isFinite(amount) && amount >= 0 && amount <= Number.MAX_SAFE_INTEGER; - if (!['downloading', 'verifying'].includes(progress.phase as string) - || typeof progress.startedAt !== 'string' || progress.startedAt.length > 80 - || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/.test(progress.startedAt) - || !Number.isFinite(Date.parse(progress.startedAt)) - || !nonnegative(progress.elapsedSeconds) || !nonnegative(progress.phaseElapsedSeconds) - || progress.phaseElapsedSeconds > progress.elapsedSeconds - || (progress.bytesPerSecond != null && !nonnegative(progress.bytesPerSecond)) - || (progress.etaSeconds != null && !nonnegative(progress.etaSeconds)) - || (['downloading', 'verifying'].includes(status) && progress.phase !== status) - || (['ready', 'outdated'].includes(status) && progress.phase !== 'verifying')) { + const nonnegative = (amount: unknown): amount is number => + typeof amount === 'number' && + Number.isFinite(amount) && + amount >= 0 && + amount <= Number.MAX_SAFE_INTEGER; + if ( + !['downloading', 'verifying'].includes(progress.phase as string) || + typeof progress.startedAt !== 'string' || + progress.startedAt.length > 80 || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/.test( + progress.startedAt + ) || + !Number.isFinite(Date.parse(progress.startedAt)) || + !nonnegative(progress.elapsedSeconds) || + !nonnegative(progress.phaseElapsedSeconds) || + progress.phaseElapsedSeconds > progress.elapsedSeconds || + (progress.bytesPerSecond != null && !nonnegative(progress.bytesPerSecond)) || + (progress.etaSeconds != null && !nonnegative(progress.etaSeconds)) || + (['downloading', 'verifying'].includes(status) && progress.phase !== status) || + (['ready', 'outdated'].includes(status) && progress.phase !== 'verifying') + ) { throw new Error('Invalid transfer progress.'); } - return { phase: progress.phase as LibraryTransferProgress['phase'], startedAt: progress.startedAt, - elapsedSeconds: progress.elapsedSeconds, phaseElapsedSeconds: progress.phaseElapsedSeconds, - bytesPerSecond: progress.bytesPerSecond as number | null ?? null, etaSeconds: progress.etaSeconds as number | null ?? null }; + return { + phase: progress.phase as LibraryTransferProgress['phase'], + startedAt: progress.startedAt, + elapsedSeconds: progress.elapsedSeconds, + phaseElapsedSeconds: progress.phaseElapsedSeconds, + bytesPerSecond: (progress.bytesPerSecond as number | null) ?? null, + etaSeconds: (progress.etaSeconds as number | null) ?? null + }; } type LibraryCacheMetadata = Pick; -const publishedCacheMetadata = new WeakMap, Map>(); +const publishedCacheMetadata = new WeakMap< + ReturnType, + Map +>(); -export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void { - if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.'); +export function updateHostLibraryCaches( + hostId: string, + input: unknown, + options: { snapshot?: boolean } = {} +): void { + if (!Array.isArray(input) || input.length > 2000) + throw new Error('Invalid host library report.'); const handle = database(); // Published metadata is immutable. Keep caches local to the connection and // avoid retaining a publication from an outer transaction that may roll back. - const versions = handle.inTransaction ? new Map() - : publishedCacheMetadata.get(handle) ?? new Map(); + const versions = handle.inTransaction + ? new Map() + : (publishedCacheMetadata.get(handle) ?? new Map()); if (!handle.inTransaction) publishedCacheMetadata.set(handle, versions); const now = new Date().toISOString(); handle.transaction(() => { @@ -359,138 +765,339 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options: if (typeof id !== 'string') throw new Error('Invalid library version ID.'); if (reported.has(id)) throw new Error('Duplicate version in host cache report.'); reported.add(id); - const version = versions.get(id) ?? handle.prepare(`SELECT state, size_bytes AS sizeBytes, manifest_sha256 AS manifestSha256 - FROM game_file_versions WHERE id=?`).get(id) as LibraryCacheMetadata | undefined; + const version = + versions.get(id) ?? + (handle + .prepare( + `SELECT state, size_bytes AS sizeBytes, manifest_sha256 AS manifestSha256 + FROM game_file_versions WHERE id=?` + ) + .get(id) as LibraryCacheMetadata | undefined); if (!version || version.state !== 'ready') continue; versions.set(id, version); let status = raw.status ?? raw.state; - if (!['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes(status)) throw new Error('Invalid library cache state.'); + if ( + !['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes( + status + ) + ) + throw new Error('Invalid library cache state.'); const checksum = typeof raw.manifestSha256 === 'string' ? raw.manifestSha256 : null; let error = typeof raw.error === 'string' ? raw.error.slice(0, 2000) : null; - if (['ready', 'outdated'].includes(status) && checksum !== version.manifestSha256) { status = 'failed'; error = 'Host manifest checksum does not match the published version.'; } - const bytes = raw.bytesDownloaded ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0); - if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) throw new Error('Invalid library download progress.'); - if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { status = 'failed'; error = 'Host reported an incomplete library as verified.'; } + if (['ready', 'outdated'].includes(status) && checksum !== version.manifestSha256) { + status = 'failed'; + error = 'Host manifest checksum does not match the published version.'; + } + const bytes = + raw.bytesDownloaded ?? + (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0); + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) + throw new Error('Invalid library download progress.'); + if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { + status = 'failed'; + error = 'Host reported an incomplete library as verified.'; + } const prior = getHostLibraryCache(hostId, id); const terminal = ['ready', 'outdated', 'failed'].includes(status); // Older agents do not report timing. Keep a completed measurement on // later inventory heartbeats, but never carry an ETA into a new phase. - let transfer = raw.transfer === undefined && terminal ? prior?.transfer ?? null : transferProgress(raw.transfer, status); - let verified = raw.bytesVerified ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : terminal ? prior?.bytesVerified ?? null : null); - if (verified !== null && (!Number.isSafeInteger(verified) || verified < 0 || verified > version.sizeBytes)) throw new Error('Invalid library verification progress.'); - if (transfer && status === 'verifying' && bytes !== version.sizeBytes) throw new Error('Download must finish before verification starts.'); - if (['ready', 'outdated'].includes(status) && verified !== version.sizeBytes) throw new Error('Host reported incomplete verification as ready.'); - if (status === 'missing') { transfer = null; verified = null; } - if (terminal && transfer) transfer = { ...transfer, bytesPerSecond: null, etaSeconds: null }; - handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at,bytes_verified,transfer_progress) + let transfer = + raw.transfer === undefined && terminal + ? (prior?.transfer ?? null) + : transferProgress(raw.transfer, status); + let verified = + raw.bytesVerified ?? + (['ready', 'outdated'].includes(status) + ? version.sizeBytes + : terminal + ? (prior?.bytesVerified ?? null) + : null); + if ( + verified !== null && + (!Number.isSafeInteger(verified) || verified < 0 || verified > version.sizeBytes) + ) + throw new Error('Invalid library verification progress.'); + if (transfer && status === 'verifying' && bytes !== version.sizeBytes) + throw new Error('Download must finish before verification starts.'); + if (['ready', 'outdated'].includes(status) && verified !== version.sizeBytes) + throw new Error('Host reported incomplete verification as ready.'); + if (status === 'missing') { + transfer = null; + verified = null; + } + if (terminal && transfer) + transfer = { ...transfer, bytesPerSecond: null, etaSeconds: null }; + handle + .prepare( + `INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at,bytes_verified,transfer_progress) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status, bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at, - bytes_verified=excluded.bytes_verified,transfer_progress=excluded.transfer_progress`) - .run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now, verified, transfer ? JSON.stringify(transfer) : null); + bytes_verified=excluded.bytes_verified,transfer_progress=excluded.transfer_progress` + ) + .run( + hostId, + id, + status, + bytes, + version.sizeBytes, + checksum, + error, + now, + verified, + transfer ? JSON.stringify(transfer) : null + ); } if (options.snapshot) { - const cached = handle.prepare('SELECT version_id FROM host_library_caches WHERE host_id=?').all(hostId) as { version_id: string }[]; - for (const cache of cached) if (!reported.has(cache.version_id)) { - handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=?") - .run(now, hostId, cache.version_id); - } + const cached = handle + .prepare('SELECT version_id FROM host_library_caches WHERE host_id=?') + .all(hostId) as { version_id: string }[]; + for (const cache of cached) + if (!reported.has(cache.version_id)) { + handle + .prepare( + "UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=?" + ) + .run(now, hostId, cache.version_id); + } } })(); } -export function queueLibraryDownload(hostId: string, versionId: string, options: { automatic?: boolean } = {}) { +export function queueLibraryDownload( + hostId: string, + versionId: string, + options: { automatic?: boolean } = {} +) { const version = getLibraryVersion(versionId); - if (!version || version.state !== 'ready') throw new Error('Choose a published library version.'); - if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.'); - if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.'); + if (!version || version.state !== 'ready') + throw new Error('Choose a published library version.'); + if (libraryFilesRetired(versionId)) + throw new Error( + 'These outdated game files have been removed. Upload them again to use them.' + ); + if ( + database() + .prepare( + "SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')" + ) + .get(hostId, versionId) + ) + throw new Error('This host copy is being removed.'); const host = getHost(hostId); - if (!host || !host.connected || !host.credentialActive || (host.mode === 'disabled' && !host.enablingSince)) throw new Error('Choose an online, enrolled host.'); + if ( + !host || + !host.connected || + !host.credentialActive || + (host.mode === 'disabled' && !host.enablingSince) + ) + throw new Error('Choose an online, enrolled host.'); const prior = getHostLibraryCache(hostId, versionId); if (prior?.operationId) { const operation = getHostOperation(prior.operationId); - if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) return operation; + if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) + return operation; } - const retainedBytes = prior?.manifestSha256 === version.manifestSha256 ? prior.bytesDownloaded : 0; - if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes - retainedBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.'); + const retainedBytes = + prior?.manifestSha256 === version.manifestSha256 ? prior.bytesDownloaded : 0; + if ( + Number(host.inventory.storageAvailable ?? 0) < + version.sizeBytes - retainedBytes + host.minimumFreeBytes + ) + throw new Error( + 'This host has insufficient storage for this library and its minimum free-space reserve.' + ); const operation = queueHostOperation(hostId, 'library.download', { - libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`, - filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`, - manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes, + libraryVersionId: versionId, + game: version.game, + manifestUrl: `/api/agent/libraries/${versionId}/manifest`, + filesBaseUrl: `/api/agent/libraries/${versionId}/files`, + statusUrl: `/api/agent/libraries/${versionId}/status`, + manifestSha256: version.manifestSha256, + sizeBytes: version.sizeBytes, ...(options.automatic ? { automatic: true } : {}) }); - updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: retainedBytes, manifestSha256: prior?.manifestSha256, bytesVerified: 0, transfer: null }]); - database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId); + updateHostLibraryCaches(hostId, [ + { + libraryVersionId: versionId, + status: 'downloading', + bytesDownloaded: retainedBytes, + manifestSha256: prior?.manifestSha256, + bytesVerified: 0, + transfer: null + } + ]); + database() + .prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?') + .run(operation.id, hostId, versionId); return operation; } export function reconcileLibraryOperations(): void { const handle = database(); - for (const row of handle.prepare("SELECT id,operation_id FROM game_file_versions WHERE state='importing' AND operation_id IS NOT NULL").all() as { id: string; operation_id: string }[]) { + for (const row of handle + .prepare( + "SELECT id,operation_id FROM game_file_versions WHERE state='importing' AND operation_id IS NOT NULL" + ) + .all() as { id: string; operation_id: string }[]) { const operation = getHostOperation(row.operation_id); - if (operation?.state === 'failed') handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'").run(operation.error ?? 'Host import failed.', row.id); + if (operation?.state === 'failed') + handle + .prepare( + "UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'" + ) + .run(operation.error ?? 'Host import failed.', row.id); } - for (const cache of handle.prepare("SELECT * FROM host_library_caches WHERE status IN ('downloading','verifying') AND operation_id IS NOT NULL").all() as CacheRow[]) { + for (const cache of handle + .prepare( + "SELECT * FROM host_library_caches WHERE status IN ('downloading','verifying') AND operation_id IS NOT NULL" + ) + .all() as CacheRow[]) { const operation = getHostOperation(cache.operation_id!); - if (operation?.state === 'failed') handle.prepare("UPDATE host_library_caches SET status='failed',error=?,updated_at=? WHERE host_id=? AND version_id=?").run(operation.error ?? 'Host download failed.', new Date().toISOString(), cache.host_id, cache.version_id); + if (operation?.state === 'failed') + handle + .prepare( + "UPDATE host_library_caches SET status='failed',error=?,updated_at=? WHERE host_id=? AND version_id=?" + ) + .run( + operation.error ?? 'Host download failed.', + new Date().toISOString(), + cache.host_id, + cache.version_id + ); } } /** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */ export function libraryOverview() { const hosts = listHosts(); - return { libraries: listLibraries(true).map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({ - ...version, caches: hosts.map(host => { - const cached = version.caches.find(cache => cache.hostId === host.id); - if (cached) return { ...cached, status: cached.status === 'ready' && library.latestVersionId !== version.id ? 'outdated' as const : cached.status }; - return { - hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0, - bytesTotal: version.sizeBytes, manifestSha256: null, error: null, updatedAt: version.createdAt, operationId: null - }; - }) - })) })), hosts }; + return { + libraries: listLibraries(true).map((library) => ({ + ...library, + versions: library.versions.map(({ files: _files, ...version }) => ({ + ...version, + caches: hosts.map((host) => { + const cached = version.caches.find((cache) => cache.hostId === host.id); + if (cached) + return { + ...cached, + status: + cached.status === 'ready' && library.latestVersionId !== version.id + ? ('outdated' as const) + : cached.status + }; + return { + hostId: host.id, + libraryVersionId: version.id, + status: 'missing' as const, + bytesDownloaded: 0, + bytesTotal: version.sizeBytes, + manifestSha256: null, + error: null, + updatedAt: version.createdAt, + operationId: null + }; + }) + })) + })), + hosts + }; } /** Any live reservation (including stopped or provisioning instances) pins its files. */ function libraryInUse(versionId: string, hostId?: string): boolean { const handle = database(); // Do not infer an empty fleet if its schema is unavailable. - if (!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'").get()) return true; - if (handle.prepare(`SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1`) - .get(versionId, ...(hostId ? [hostId] : []))) return true; + if ( + !handle + .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'") + .get() + ) + return true; + if ( + handle + .prepare( + `SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1` + ) + .get(versionId, ...(hostId ? [hostId] : [])) + ) + return true; // Accepted jobs pin their game files before placement creates an instance. // Automatic intents may still choose any host, so retain each available copy. - if (handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get() - && handle.prepare(`SELECT 1 FROM server_deployments d + if ( + handle + .prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'" + ) + .get() && + handle + .prepare( + `SELECT 1 FROM server_deployments d JOIN native_creation_intents q ON q.deployment_id=d.id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL AND json_extract(q.selection,'$.libraryId')=? - ${hostId ? `AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL - OR json_extract(q.selection,'$.hostId')=?)` : ''} LIMIT 1`) - .get(versionId, ...(hostId ? [hostId] : []))) return true; - return !!handle.prepare(`SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown') + ${ + hostId + ? `AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL + OR json_extract(q.selection,'$.hostId')=?)` + : '' + } LIMIT 1` + ) + .get(versionId, ...(hostId ? [hostId] : [])) + ) + return true; + return !!handle + .prepare( + `SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown') AND (json_extract(payload,'$.libraryVersionId')=? OR json_extract(payload,'$.libraryId')=? OR json_extract(payload,'$.importId')=?) - ${hostId ? 'AND host_id=?' : ''} LIMIT 1`).get(versionId, versionId, versionId, ...(hostId ? [hostId] : [])); + ${hostId ? 'AND host_id=?' : ''} LIMIT 1` + ) + .get(versionId, versionId, versionId, ...(hostId ? [hostId] : [])); } /** Queue serialized, recoverable host cleanup; an offline host keeps its copy pinned. */ export function reconcileHostLibraryCleanup(hostId: string): void { const handle = database(); const host = getHost(hostId); - if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission - || !host.inventory.capabilities?.includes('library.remove')) return; + if ( + !host?.connected || + !host.credentialActive || + host.mode === 'disabled' || + host.decommission || + !host.inventory.capabilities?.includes('library.remove') + ) + return; const latest = retainedGameFileIds(); - const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[]; + const caches = handle + .prepare('SELECT * FROM host_library_caches WHERE host_id=?') + .all(hostId) as CacheRow[]; for (const cache of caches) { - if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue; - const previous = handle.prepare(`SELECT state,updated_at FROM ec_host_operations WHERE host_id=? AND kind='library.remove' - AND json_extract(payload,'$.libraryVersionId')=? ORDER BY rowid DESC LIMIT 1`).get(hostId, cache.version_id) as { state: string; updated_at: string } | undefined; + if ( + latest.has(cache.version_id) || + (cache.status === 'missing' && cache.bytes_downloaded === 0) || + libraryInUse(cache.version_id, hostId) + ) + continue; + const previous = handle + .prepare( + `SELECT state,updated_at FROM ec_host_operations WHERE host_id=? AND kind='library.remove' + AND json_extract(payload,'$.libraryVersionId')=? ORDER BY rowid DESC LIMIT 1` + ) + .get(hostId, cache.version_id) as { state: string; updated_at: string } | undefined; // Wait for fresh inventory after completion, and back off failures. - if (previous && (Date.parse(cache.updated_at) <= Date.parse(previous.updated_at) - || previous.state === 'failed' && Date.now() - Date.parse(previous.updated_at) < 60_000)) continue; + if ( + previous && + (Date.parse(cache.updated_at) <= Date.parse(previous.updated_at) || + (previous.state === 'failed' && + Date.now() - Date.parse(previous.updated_at) < 60_000)) + ) + continue; const version = getLibraryVersion(cache.version_id)!; - queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id, game: version.game }); + queueHostOperation(hostId, 'library.remove', { + libraryVersionId: version.id, + game: version.game + }); } } @@ -498,28 +1105,54 @@ const cleaning = new WeakMap, Promise>(); /** Reserve deletion durably before yielding, so new downloads cannot race filesystem removal. */ export function cleanupControllerLibraries(): Promise { const handle = database(); - const current = cleaning.get(handle); if (current) return current; + const current = cleaning.get(handle); + if (current) return current; const task = (async () => { await cleanupAbandonedUploads(); const latest = retainedGameFileIds(); - const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v + const versions = handle + .prepare( + `SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id - WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[]; + WHERE v.state='ready' AND c.removed_at IS NULL` + ) + .all() as { id: string; storage_location: string; game: string }[]; for (const version of versions) { if (latest.has(version.id) || libraryInUse(version.id)) continue; // Missing partial downloads still occupy disk. Require a full absence report from every host. - if (handle.prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0) LIMIT 1").get(version.id)) continue; + if ( + handle + .prepare( + "SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0) LIMIT 1" + ) + .get(version.id) + ) + continue; // Only delete the controller-owned version directory, never an arbitrary stored path. const expected = join(gameFilesRoot(), version.game, version.id); - if (!/^[a-zA-Z0-9_-]+$/.test(version.id) || resolve(version.storage_location) !== expected) continue; - handle.prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString()); + if ( + !/^[a-zA-Z0-9_-]+$/.test(version.id) || + resolve(version.storage_location) !== expected + ) + continue; + handle + .prepare( + 'INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)' + ) + .run(version.id, new Date().toISOString()); try { for (const path of [gameFilesRoot(), dirname(expected), expected]) { - const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; }); - if (info?.isSymbolicLink()) throw new Error('Refusing game-file cleanup through a symlink.'); + const info = await lstat(path).catch((cause) => { + if (cause.code === 'ENOENT') return null; + throw cause; + }); + if (info?.isSymbolicLink()) + throw new Error('Refusing game-file cleanup through a symlink.'); } await rm(expected, { recursive: true, force: true }); - handle.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id); + handle + .prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?') + .run(new Date().toISOString(), version.id); } catch (cause) { console.error(`Game-file cleanup will retry for ${version.id}:`, cause); } @@ -531,65 +1164,141 @@ export function cleanupControllerLibraries(): Promise { } function touchUpload(id: string) { - database().prepare("UPDATE game_file_versions SET last_activity_at=? WHERE id=? AND state!='ready'").run(new Date().toISOString(),id); + database() + .prepare("UPDATE game_file_versions SET last_activity_at=? WHERE id=? AND state!='ready'") + .run(new Date().toISOString(), id); } /** Latest N published versions are retained per game; grace starts at supersession. */ export function retainedGameFileIds(settings = fleetSettings()): Set { - const versions = database().prepare(`SELECT v.id,l.game,v.published_at FROM game_file_versions v + const versions = database() + .prepare( + `SELECT v.id,l.game,v.published_at FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.state='ready' - ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC`).all() as { id: string; game: string; published_at: string }[]; + ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC` + ) + .all() as { id: string; game: string; published_at: string }[]; const kept = new Set(); - const seen = new Map(); + const seen = new Map(); for (const v of versions) { const prior = seen.get(v.game); - if (!prior || prior.count < settings.gameFileKeepCount - || Date.parse(prior.newerAt) > Date.now() - settings.gameFileGraceDays * 86400_000) kept.add(v.id); - seen.set(v.game,{ count: (prior?.count ?? 0) + 1, newerAt: v.published_at }); + if ( + !prior || + prior.count < settings.gameFileKeepCount || + Date.parse(prior.newerAt) > Date.now() - settings.gameFileGraceDays * 86400_000 + ) + kept.add(v.id); + seen.set(v.game, { count: (prior?.count ?? 0) + 1, newerAt: v.published_at }); } - for (const row of database().prepare("SELECT record_id FROM retention_controls WHERE category='game-file' AND protected=1").all() as { record_id: string }[]) kept.add(row.record_id); + for (const row of database() + .prepare( + "SELECT record_id FROM retention_controls WHERE category='game-file' AND protected=1" + ) + .all() as { record_id: string }[]) + kept.add(row.record_id); return kept; } function abandonedUploads(settings = fleetSettings()) { const days = settings.uploadKeepDays; if (!days) return []; - return database().prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id + return database() + .prepare( + `SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.state IN ('importing','failed') AND v.source_host_id='' AND COALESCE(v.last_activity_at,v.created_at) { // A chunk or resume request may have refreshed the timestamp before lock admission. - if (!abandonedUploads().some(row => row.id === upload.id)) return; + if (!abandonedUploads().some((row) => row.id === upload.id)) return; if (!/^[a-zA-Z0-9_-]+$/.test(upload.id)) return; - const expected = join(gameFilesRoot(),upload.game,upload.id); + const expected = join(gameFilesRoot(), upload.game, upload.id); if (resolve(upload.storage_location) !== expected) return; - database().prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(upload.id,new Date().toISOString()); + database() + .prepare( + 'INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)' + ) + .run(upload.id, new Date().toISOString()); // Paths are controller-derived. Validate parents before recursive removal. - for (const path of [gameFilesRoot(),join(gameFilesRoot(),'.imports'),join(gameFilesRoot(),upload.game),expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) { - const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; }); - if (info?.isSymbolicLink()) throw new Error('Refusing upload cleanup through a symlink.'); + for (const path of [ + gameFilesRoot(), + join(gameFilesRoot(), '.imports'), + join(gameFilesRoot(), upload.game), + expected, + stagingRoot(upload.id), + `${stagingRoot(upload.id)}.chunks` + ]) { + const info = await lstat(path).catch((cause) => { + if (cause.code === 'ENOENT') return null; + throw cause; + }); + if (info?.isSymbolicLink()) + throw new Error('Refusing upload cleanup through a symlink.'); } - for (const path of [expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) await rm(path,{ recursive: true,force: true }); - database().prepare("UPDATE game_file_versions SET state='failed',error='Upload expired under the retention policy.' WHERE id=? AND state!='ready'").run(upload.id); - database().prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(),upload.id); + for (const path of [ + expected, + stagingRoot(upload.id), + `${stagingRoot(upload.id)}.chunks` + ]) + await rm(path, { recursive: true, force: true }); + database() + .prepare( + "UPDATE game_file_versions SET state='failed',error='Upload expired under the retention policy.' WHERE id=? AND state!='ready'" + ) + .run(upload.id); + database() + .prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?') + .run(new Date().toISOString(), upload.id); }); } } export function gameFileRetentionPreview(settings = fleetSettings()) { const kept = retainedGameFileIds(settings); - const versions = database().prepare(`SELECT v.id,v.size_bytes FROM game_file_versions v LEFT JOIN game_file_cleanup c ON c.version_id=v.id - WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; size_bytes: number }[]; - const eligible = versions.filter(v => !kept.has(v.id) && !libraryInUse(v.id) && !database().prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0)").get(v.id)); - const uploads = abandonedUploads(settings).filter(v => !publishing.has(v.id) && !uploadIsActive(stagingRoot(v.id))); + const versions = database() + .prepare( + `SELECT v.id,v.size_bytes FROM game_file_versions v LEFT JOIN game_file_cleanup c ON c.version_id=v.id + WHERE v.state='ready' AND c.removed_at IS NULL` + ) + .all() as { id: string; size_bytes: number }[]; + const eligible = versions.filter( + (v) => + !kept.has(v.id) && + !libraryInUse(v.id) && + !database() + .prepare( + "SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0)" + ) + .get(v.id) + ); + const uploads = abandonedUploads(settings).filter( + (v) => !publishing.has(v.id) && !uploadIsActive(stagingRoot(v.id)) + ); return [ - { category: 'game-file',label: 'Unused controller game files',eligible: eligible.length,payloadBytes: eligible.reduce((n,v)=>n+v.size_bytes,0),records: eligible.slice(0,20).map(v=>({ id: v.id })) }, - { category: 'upload',label: 'Abandoned uploads',eligible: uploads.length,payloadBytes: 0,records: uploads.slice(0,20).map(v=>({ id: v.id })) } + { + category: 'game-file', + label: 'Unused controller game files', + eligible: eligible.length, + payloadBytes: eligible.reduce((n, v) => n + v.size_bytes, 0), + records: eligible.slice(0, 20).map((v) => ({ id: v.id })) + }, + { + category: 'upload', + label: 'Abandoned uploads', + eligible: uploads.length, + payloadBytes: 0, + records: uploads.slice(0, 20).map((v) => ({ id: v.id })) + } ]; } diff --git a/src/lib/server/game-files/schema.ts b/src/lib/server/game-files/schema.ts index 20086c0..680e09c 100644 --- a/src/lib/server/game-files/schema.ts +++ b/src/lib/server/game-files/schema.ts @@ -50,10 +50,17 @@ export function ensureGameFileSchema(handle: Database.Database): void { WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END; CREATE INDEX IF NOT EXISTS idx_host_library_caches_version ON host_library_caches(version_id); `); - const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as { name: string }[]; - if (!versionColumns.some(column => column.name === 'last_activity_at')) handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT'); - const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as { name: string }[]; - if (!cacheColumns.some(column => column.name === 'bytes_verified')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER'); - if (!cacheColumns.some(column => column.name === 'transfer_progress')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT'); + const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as { + name: string; + }[]; + if (!versionColumns.some((column) => column.name === 'last_activity_at')) + handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT'); + const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as { + name: string; + }[]; + if (!cacheColumns.some((column) => column.name === 'bytes_verified')) + handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER'); + if (!cacheColumns.some((column) => column.name === 'transfer_progress')) + handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT'); initialized.add(handle); } diff --git a/src/lib/server/game-files/storage.ts b/src/lib/server/game-files/storage.ts index 524ad03..93a1964 100644 --- a/src/lib/server/game-files/storage.ts +++ b/src/lib/server/game-files/storage.ts @@ -8,39 +8,62 @@ import { randomUUID } from 'node:crypto'; import type { GameFileEntry } from '../../game-files.js'; export function validateGamePath(input: unknown): string { - if (typeof input !== 'string' || !input || input.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(input) || Buffer.from(input).toString('utf8') !== input - || isAbsolute(input) || input.split('/').some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) { + if ( + typeof input !== 'string' || + !input || + input.length > 1024 || + // eslint-disable-next-line no-control-regex -- Reject control characters in game file paths. + /[\\\x00-\x1f\x7f:]/.test(input) || + Buffer.from(input).toString('utf8') !== input || + isAbsolute(input) || + input + .split('/') + .some((part) => !part || part === '.' || part === '..' || /[. ]$/.test(part)) + ) { throw new Error('Game file paths must be relative, normalized paths without traversal.'); } return input; } export function validateManifest(input: unknown): GameFileEntry[] { - if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a manifest containing 1–200000 files.'); + if (!Array.isArray(input) || !input.length || input.length > 200_000) + throw new Error('Provide a manifest containing 1–200000 files.'); const paths = new Set(); let total = 0; - const files = input.map(raw => { - if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.'); - const path = validateGamePath(raw.path); - if (!Number.isSafeInteger(raw.size) || raw.size < 0 || typeof raw.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(raw.sha256)) throw new Error(`Invalid size or SHA256 for ${path}.`); - const key = path.toLowerCase(); - if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`); - paths.add(key); total += raw.size; - if (!Number.isSafeInteger(total)) throw new Error('Library size exceeds the supported limit.'); - return { path, size: raw.size as number, sha256: raw.sha256 as string }; - }).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); + const files = input + .map((raw) => { + if (!raw || typeof raw !== 'object') throw new Error('Invalid manifest entry.'); + const path = validateGamePath(raw.path); + if ( + !Number.isSafeInteger(raw.size) || + raw.size < 0 || + typeof raw.sha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(raw.sha256) + ) + throw new Error(`Invalid size or SHA256 for ${path}.`); + const key = path.toLowerCase(); + if (paths.has(key)) throw new Error(`Duplicate game file path: ${path}.`); + paths.add(key); + total += raw.size; + if (!Number.isSafeInteger(total)) + throw new Error('Library size exceeds the supported limit.'); + return { path, size: raw.size as number, sha256: raw.sha256 as string }; + }) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); for (const file of files) { const parts = file.path.toLowerCase().split('/'); parts.pop(); while (parts.length) { - if (paths.has(parts.join('/'))) throw new Error('A manifest path cannot be both a file and a directory.'); + if (paths.has(parts.join('/'))) + throw new Error('A manifest path cannot be both a file and a directory.'); parts.pop(); } } return files; } -export const manifestChecksum = (files: GameFileEntry[]): string => createHash('sha256').update(JSON.stringify(files)).digest('hex'); +export const manifestChecksum = (files: GameFileEntry[]): string => + createHash('sha256').update(JSON.stringify(files)).digest('hex'); export function gameFilePath(root: string, path: string): string { const base = resolve(root); @@ -50,21 +73,41 @@ export function gameFilePath(root: string, path: string): string { } /** Refuse symlinks at every existing component, including the managed root. */ -export async function assertRegularPath(root: string, path: string, allowMissing = false): Promise { +export async function assertRegularPath( + root: string, + path: string, + allowMissing = false +): Promise { const result = gameFilePath(root, path); let current = resolve(root); - const components = [current, ...validateGamePath(path).split('/').map(part => current = resolve(current, part))]; + const components = [ + current, + ...validateGamePath(path) + .split('/') + .map((part) => (current = resolve(current, part))) + ]; for (let i = 0; i < components.length; i++) { let info; - try { info = await lstat(components[i]); } - catch (cause) { if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue; throw cause; } - if (info.isSymbolicLink() || (i < components.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('Game storage contains a non-regular path.'); + try { + info = await lstat(components[i]); + } catch (cause) { + if (allowMissing && (cause as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw cause; + } + if ( + info.isSymbolicLink() || + (i < components.length - 1 ? !info.isDirectory() : !info.isFile()) + ) + throw new Error('Game storage contains a non-regular path.'); } return result; } /** Compute a file digest without buffering game archives in memory. */ -export async function hashGameFile(root: string, file: Pick): Promise { +export async function hashGameFile( + root: string, + file: Pick +): Promise { const path = await assertRegularPath(root, file.path); const info = await stat(path); if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`); @@ -74,35 +117,61 @@ export async function hashGameFile(root: string, file: Pick { - if (await hashGameFile(root, file) !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`); + if ((await hashGameFile(root, file)) !== file.sha256) + throw new Error(`Checksum verification failed for ${file.path}.`); } /** Stream to a private temporary file; publish only after byte count and SHA256 match. */ -export async function receiveGameFile(root: string, file: GameFileEntry, body: ReadableStream | null): Promise { +export async function receiveGameFile( + root: string, + file: GameFileEntry, + body: ReadableStream | null +): Promise { if (!body) throw new Error('A file body is required.'); const path = await assertRegularPath(root, file.path, true); await mkdir(dirname(path), { recursive: true }); const temp = `${path}.upload-${randomUUID()}`; let size = 0; const hash = createHash('sha256'); - const verifier = new Transform({ transform(chunk, _encoding, done) { - size += chunk.length; - if (size > file.size) return done(new Error('Upload exceeds the manifest size.')); - hash.update(chunk); done(null, chunk); - } }); + const verifier = new Transform({ + transform(chunk, _encoding, done) { + size += chunk.length; + if (size > file.size) return done(new Error('Upload exceeds the manifest size.')); + hash.update(chunk); + done(null, chunk); + } + }); try { - await pipeline(Readable.fromWeb(body as never), verifier, createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true })); - if (size !== file.size || hash.digest('hex') !== file.sha256) throw new Error(`Upload checksum or size mismatch for ${file.path}.`); + await pipeline( + Readable.fromWeb(body as never), + verifier, + createWriteStream(temp, { flags: 'wx', mode: 0o600, flush: true }) + ); + if (size !== file.size || hash.digest('hex') !== file.sha256) + throw new Error(`Upload checksum or size mismatch for ${file.path}.`); await rename(temp, path); - } finally { await rm(temp, { force: true }); } + } finally { + await rm(temp, { force: true }); + } } -export function parseByteRange(header: string | null, size: number): { start: number; end: number } | null { +export function parseByteRange( + header: string | null, + size: number +): { start: number; end: number } | null { if (!header) return null; const match = /^bytes=(\d*)-(\d*)$/.exec(header); if (!match || (!match[1] && !match[2]) || size === 0) throw new Error('Unsatisfiable range.'); - let start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2])); - let end = match[1] && match[2] ? Number(match[2]) : size - 1; - if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start || (!match[1] && Number(match[2]) === 0)) throw new Error('Unsatisfiable range.'); + const start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2])); + const end = match[1] && match[2] ? Number(match[2]) : size - 1; + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + start >= size || + end < start || + (!match[1] && Number(match[2]) === 0) + ) + throw new Error('Unsatisfiable range.'); return { start, end: Math.min(end, size - 1) }; } diff --git a/src/lib/server/game-files/uploads.ts b/src/lib/server/game-files/uploads.ts index 51f8d0c..8545585 100644 --- a/src/lib/server/game-files/uploads.ts +++ b/src/lib/server/game-files/uploads.ts @@ -32,28 +32,47 @@ export class UploadOffsetError extends Error { /** Apply the same path, case, size and file/directory collision rules as published manifests. */ export function validateUploadPlan(input: unknown): UploadFilePlan[] { - if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a folder containing 1–200000 files.'); + if (!Array.isArray(input) || !input.length || input.length > 200_000) + throw new Error('Provide a folder containing 1–200000 files.'); const modified = new Map(); - const manifest = validateManifest(input.map(raw => { - if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.'); - const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified; - if (!Number.isSafeInteger(lastModified) || lastModified < 0) throw new Error('Invalid file modification time.'); - modified.set(raw.path, lastModified); - return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) }; + const manifest = validateManifest( + input.map((raw) => { + if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.'); + const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified; + if (!Number.isSafeInteger(lastModified) || lastModified < 0) + throw new Error('Invalid file modification time.'); + modified.set(raw.path, lastModified); + return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) }; + }) + ); + return manifest.map((file) => ({ + path: file.path, + size: file.size, + lastModified: modified.get(file.path)! })); - return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! })); } -interface UploadLocks { barrier: Promise; writers: number; files: Map>; active: Set> } +interface UploadLocks { + barrier: Promise; + writers: number; + files: Map>; + active: Set>; +} const uploadLocks = new Map(); -export function uploadIsActive(root: string): boolean { return uploadLocks.has(resolve(root)); } +export function uploadIsActive(root: string): boolean { + return uploadLocks.has(resolve(root)); +} function locksFor(key: string): UploadLocks { let locks = uploadLocks.get(key); - if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); } + if (!locks) { + locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; + uploadLocks.set(key, locks); + } return locks; } function releaseLocks(key: string, locks: UploadLocks) { - if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) uploadLocks.delete(key); + if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) + uploadLocks.delete(key); } /** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */ @@ -64,12 +83,15 @@ export async function withUploadLock(root: string, callback: () => Promise // from starting until publication finishes (including failed publication). const previous = Promise.all([locks.barrier, ...locks.active]); let release!: () => void; - const current = new Promise(done => { release = done; }); + const current = new Promise((done) => { + release = done; + }); locks.barrier = current; locks.writers++; await previous; - try { return await callback(); } - finally { + try { + return await callback(); + } finally { release(); locks.writers--; releaseLocks(key, locks); @@ -77,31 +99,47 @@ export async function withUploadLock(root: string, callback: () => Promise } /** Different files may transfer together; offsets for the same file remain serial. */ -export async function withUploadFileLock(root: string, file: string, callback: () => Promise): Promise { - const key = resolve(root), locks = locksFor(key); +export async function withUploadFileLock( + root: string, + file: string, + callback: () => Promise +): Promise { + const key = resolve(root), + locks = locksFor(key); const previous = Promise.all([locks.barrier, locks.files.get(file)]); let release!: () => void; - const current = new Promise(done => { release = done; }); - locks.files.set(file, current); locks.active.add(current); + const current = new Promise((done) => { + release = done; + }); + locks.files.set(file, current); + locks.active.add(current); await previous; - try { return await callback(); } - finally { - release(); locks.active.delete(current); + try { + return await callback(); + } finally { + release(); + locks.active.delete(current); if (locks.files.get(file) === current) locks.files.delete(file); releaseLocks(key, locks); } } /** File size is the resume checkpoint, including after a controller restart during an append. */ -export async function getUploadFileProgress(root: string, file: Pick): Promise { +export async function getUploadFileProgress( + root: string, + file: Pick +): Promise { const path = await assertRegularPath(root, file.path, true); try { const info = await lstat(path); - if (!info.isFile() || info.isSymbolicLink()) throw new Error('Game storage contains a non-regular path.'); - if (info.size > file.size) throw new Error(`Stored upload exceeds the declared size of ${file.path}.`); + if (!info.isFile() || info.isSymbolicLink()) + throw new Error('Game storage contains a non-regular path.'); + if (info.size > file.size) + throw new Error(`Stored upload exceeds the declared size of ${file.path}.`); return { receivedBytes: info.size, complete: info.size === file.size }; } catch (cause) { - if ((cause as NodeJS.ErrnoException).code === 'ENOENT') return { receivedBytes: 0, complete: false }; + if ((cause as NodeJS.ErrnoException).code === 'ENOENT') + return { receivedBytes: 0, complete: false }; throw cause; } } @@ -118,10 +156,17 @@ export async function receiveUploadChunk( body: ReadableStream | null, beforeWrite?: () => Promise | void ): Promise { - if (!Number.isSafeInteger(chunk.offset) || chunk.offset < 0 || !Number.isSafeInteger(chunk.size) - || chunk.size < 0 || chunk.size > MAX_UPLOAD_CHUNK_BYTES || chunk.offset + chunk.size > file.size - || !Number.isSafeInteger(chunk.offset + chunk.size) || !/^[a-f0-9]{64}$/.test(chunk.sha256) - || (chunk.size === 0 && file.size !== 0)) { + if ( + !Number.isSafeInteger(chunk.offset) || + chunk.offset < 0 || + !Number.isSafeInteger(chunk.size) || + chunk.size < 0 || + chunk.size > MAX_UPLOAD_CHUNK_BYTES || + chunk.offset + chunk.size > file.size || + !Number.isSafeInteger(chunk.offset + chunk.size) || + !/^[a-f0-9]{64}$/.test(chunk.sha256) || + (chunk.size === 0 && file.size !== 0) + ) { throw new Error('Invalid upload chunk size, offset or SHA256.'); } if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.'); @@ -129,7 +174,8 @@ export async function receiveUploadChunk( return withUploadFileLock(root, file.path, async () => { await beforeWrite?.(); const progress = await getUploadFileProgress(root, file); - if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes); + if (chunk.offset !== progress.receivedBytes) + throw new UploadOffsetError(progress.receivedBytes); const destination = await assertRegularPath(root, file.path, true); // Keep transient chunks outside the tree that will become the immutable library. const chunkRoot = `${resolve(root)}.chunks`; @@ -137,19 +183,31 @@ export async function receiveUploadChunk( await mkdir(dirname(temporary), { recursive: true }); let received = 0; const hash = createHash('sha256'); - const verifier = new Transform({ transform(data, _encoding, done) { - received += data.length; - if (received > chunk.size) return done(new Error('Upload exceeds the declared chunk size.')); - hash.update(data); done(null, data); - } }); + const verifier = new Transform({ + transform(data, _encoding, done) { + received += data.length; + if (received > chunk.size) + return done(new Error('Upload exceeds the declared chunk size.')); + hash.update(data); + done(null, data); + } + }); try { - await pipeline(body ? Readable.fromWeb(body as never) : Readable.from([]), verifier, - createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true })); - if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) throw new Error('Upload chunk checksum or size mismatch.'); + await pipeline( + body ? Readable.fromWeb(body as never) : Readable.from([]), + verifier, + createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true }) + ); + if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) + throw new Error('Upload chunk checksum or size mismatch.'); await mkdir(dirname(destination), { recursive: true }); await assertRegularPath(root, file.path, true); - const target = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + const target = await open( + destination, + constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, + 0o600 + ); try { const info = await target.stat(); if (!info.isFile()) throw new Error('Game storage contains a non-regular path.'); @@ -160,8 +218,14 @@ export async function receiveUploadChunk( const bytes = data as Buffer; let written = 0; while (written < bytes.length) { - const result = await target.write(bytes, written, bytes.length - written, position); - if (result.bytesWritten === 0) throw new Error('Unable to append the upload chunk.'); + const result = await target.write( + bytes, + written, + bytes.length - written, + position + ); + if (result.bytesWritten === 0) + throw new Error('Unable to append the upload chunk.'); written += result.bytesWritten; position += result.bytesWritten; } @@ -172,15 +236,26 @@ export async function receiveUploadChunk( await target.sync(); throw cause; } - } finally { await target.close(); } - return { receivedBytes: chunk.offset + chunk.size, complete: chunk.offset + chunk.size === file.size }; - } finally { await rm(temporary, { force: true }); } + } finally { + await target.close(); + } + return { + receivedBytes: chunk.offset + chunk.size, + complete: chunk.offset + chunk.size === file.size + }; + } finally { + await rm(temporary, { force: true }); + } }); } /** Caller holds withUploadLock through hashing and publication to prevent concurrent writes. */ -export async function hashUploadedFiles(root: string, files: ReadonlyArray>): Promise { +export async function hashUploadedFiles( + root: string, + files: ReadonlyArray> +): Promise { const manifest: GameFileEntry[] = []; - for (const file of files) manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) }); + for (const file of files) + manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) }); return validateManifest(manifest); } diff --git a/src/lib/server/hosts/availability.ts b/src/lib/server/hosts/availability.ts index 5472f7e..dd4dbb8 100644 --- a/src/lib/server/hosts/availability.ts +++ b/src/lib/server/hosts/availability.ts @@ -29,41 +29,81 @@ const scheduled = new WeakSet>(); function snapshotSchemaReady(handle: ReturnType): boolean { // Host enrollment and game-file maintenance can run before provisioning is // initialized. Missing queue data means unknown capacity, not an empty fleet. - const { count } = handle.prepare(`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table' - AND name IN ('native_instances','native_provisioning','server_deployments','settings')`).get() as { count: number }; + const { count } = handle + .prepare( + `SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table' + AND name IN ('native_instances','native_provisioning','server_deployments','settings')` + ) + .get() as { count: number }; if (count === 4) return true; snapshots.delete(handle); return false; } -function queueSnapshot(handle: ReturnType, hosts: Map): Snapshot { - const counts = handle.prepare(`SELECT +function queueSnapshot( + handle: ReturnType, + hosts: Map +): Snapshot { + const counts = handle + .prepare( + `SELECT COALESCE(SUM(state IN ('running','awaiting-health') OR ${committedRebalanceStartup}),0) AS active, COALESCE(SUM(state='queued'),0) AS queued FROM server_deployments WHERE action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup}) - AND server_deleted_at IS NULL`).get() as { active: number; queued: number }; - return { hosts, ...counts, startupLimit: fleetSettings().createConcurrency, updatedAt: new Date().toISOString() }; + AND server_deleted_at IS NULL` + ) + .get() as { active: number; queued: number }; + return { + hosts, + ...counts, + startupLimit: fleetSettings().createConcurrency, + updatedAt: new Date().toISOString() + }; } function observeHost(host: Host): HostAvailability { const handle = db(); const { running, startup } = capacityCalibration(host.id); - const measured = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample.generation])); - const pending = handle.prepare(`SELECT DISTINCT i.id,i.generation FROM native_instances i + const measured = new Map( + host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample.generation]) + ); + const pending = handle + .prepare( + `SELECT DISTINCT i.id,i.generation FROM native_instances i JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild') - AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[]; - const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id - WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number }; - const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n; + AND d.state IN ('running','awaiting-health')` + ) + .all(host.id) as { id: string; generation: number }[]; + const incoming = handle + .prepare( + `SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id + WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))` + ) + .get(host.id) as { n: number }; + const unmeasured = + pending.filter((instance) => measured.get(instance.id) !== instance.generation).length + + incoming.n; const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory; - const freeCpu = cpuCount && cpuUsagePercent !== undefined - ? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null; - const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal - ? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null; - return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes, + const freeCpu = + cpuCount && cpuUsagePercent !== undefined + ? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) + : null; + const freeMemory = + memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal + ? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) + : null; + return { + host, + freeCpu, + freeMemory, + runningCpu: running.cpuCores, + runningMemory: running.memoryBytes, calibration: { running, startup }, - ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] }; + ports: handle + .prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?') + .all(host.id) as HostAvailability['ports'] + }; } /** Host telemetry updates the estimate once, independently of how many lobbies are previewed. */ @@ -85,20 +125,33 @@ export function scheduleAvailabilityRefresh(): void { if (!handle.open) return; try { if (!snapshotSchemaReady(handle)) return; - const hosts = new Map(listHosts().map(host => [host.id, observeHost(host)])); + const hosts = new Map(listHosts().map((host) => [host.id, observeHost(host)])); snapshots.set(handle, queueSnapshot(handle, hosts)); - } catch (error) { console.error('[availability] snapshot refresh failed:', error); } + } catch (error) { + console.error('[availability] snapshot refresh failed:', error); + } }); } -subscribe(event => { - if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh(); +subscribe((event) => { + if (event.type === 'fleet-invalidated' || event.type === 'server-status') + scheduleAvailabilityRefresh(); }); export function freePorts(snapshot: HostAvailability, bindAddress?: string): number { const bind = bindAddress ?? snapshot.host.bindAddress; - const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax - && (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port)); + const used = new Set( + snapshot.ports + .filter( + (row) => + row.port >= snapshot.host.portMin && + row.port <= snapshot.host.portMax && + (row.bind_address === bind || + ['0.0.0.0', '::'].includes(bind) || + ['0.0.0.0', '::'].includes(row.bind_address)) + ) + .map((row) => row.port) + ); return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size); } @@ -110,44 +163,108 @@ export function readAvailabilitySnapshot(): Snapshot | null { } /** Estimates only: no placement, historical scans, reservations, or live host requests. */ -export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } { +export function launchAvailability(selections: NativePlacement[]): { + availability: LaunchAvailability; + warnings: string[]; +} { const snapshot = snapshots.get(db()); if (!snapshot) { scheduleAvailabilityRefresh(); - return { availability: { updatedAt: null, requested: selections.length, estimatedAvailable: null, portsAvailable: null, - cpuCoresAvailable: null, memoryBytesAvailable: null, startupSlots: null, startupLimit: null, queued: null }, - warnings: selections.length ? ['Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'] : [] }; + return { + availability: { + updatedAt: null, + requested: selections.length, + estimatedAvailable: null, + portsAvailable: null, + cpuCoresAvailable: null, + memoryBytesAvailable: null, + startupSlots: null, + startupLimit: null, + queued: null + }, + warnings: selections.length + ? [ + 'Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.' + ] + : [] + }; } - const candidates = [...snapshot.hosts.values()].filter(({ host }) => host.readiness === 'ready' - && host.lastHeartbeat && Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS - && (host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3); - const eligible = (candidate: HostAvailability, selection: NativePlacement) => (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) - && !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') - && !!candidate.host.inventory.libraries?.some(library => library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)); - const relevant = candidates.filter(candidate => selections.some(selection => eligible(candidate, selection))); + const candidates = [...snapshot.hosts.values()].filter( + ({ host }) => + host.readiness === 'ready' && + host.lastHeartbeat && + Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS && + (host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3 + ); + const eligible = (candidate: HostAvailability, selection: NativePlacement) => + (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) && + !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') && + !!candidate.host.inventory.libraries?.some( + (library) => + library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state) + ); + const relevant = candidates.filter((candidate) => + selections.some((selection) => eligible(candidate, selection)) + ); const planned = new Map(); let estimatedAvailable = 0; for (const selection of selections) { - const candidate = relevant.find(candidate => { + const candidate = relevant.find((candidate) => { const used = planned.get(candidate.host.id) ?? 0; - return eligible(candidate, selection) && freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) > used - && candidate.freeCpu !== null && candidate.freeMemory !== null - && candidate.freeCpu >= (used + 1) * candidate.runningCpu - && candidate.freeMemory >= (used + 1) * candidate.runningMemory; + return ( + eligible(candidate, selection) && + freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) > + used && + candidate.freeCpu !== null && + candidate.freeMemory !== null && + candidate.freeCpu >= (used + 1) * candidate.runningCpu && + candidate.freeMemory >= (used + 1) * candidate.runningMemory + ); }); - if (candidate) { estimatedAvailable++; planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1); } + if (candidate) { + estimatedAvailable++; + planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1); + } } const warnings: string[] = []; - const unknown = relevant.some(candidate => candidate.freeCpu === null || candidate.freeMemory === null); - if (selections.length && !relevant.length) warnings.push('Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.'); - else if (unknown) warnings.push('Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.'); - else if (estimatedAvailable < selections.length) warnings.push(`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`); + const unknown = relevant.some( + (candidate) => candidate.freeCpu === null || candidate.freeMemory === null + ); + if (selections.length && !relevant.length) + warnings.push( + 'Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.' + ); + else if (unknown) + warnings.push( + 'Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.' + ); + else if (estimatedAvailable < selections.length) + warnings.push( + `Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.` + ); const startupSlots = Math.max(0, snapshot.startupLimit - snapshot.active); - if (selections.length && !startupSlots) warnings.push('All startup slots are in use. New launches will wait in the queue.'); - if (selections.length && snapshot.queued) warnings.push(`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`); - return { availability: { updatedAt: snapshot.updatedAt, requested: selections.length, estimatedAvailable: unknown ? null : estimatedAvailable, - portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0), - cpuCoresAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0), - memoryBytesAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0), - startupSlots, startupLimit: snapshot.startupLimit, queued: snapshot.queued }, warnings }; + if (selections.length && !startupSlots) + warnings.push('All startup slots are in use. New launches will wait in the queue.'); + if (selections.length && snapshot.queued) + warnings.push( + `${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.` + ); + return { + availability: { + updatedAt: snapshot.updatedAt, + requested: selections.length, + estimatedAvailable: unknown ? null : estimatedAvailable, + portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0), + cpuCoresAvailable: unknown + ? null + : relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0), + memoryBytesAvailable: unknown + ? null + : relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0), + startupSlots, + startupLimit: snapshot.startupLimit, + queued: snapshot.queued + }, + warnings + }; } diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index 92a99da..d4ed2ff 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -1,7 +1,13 @@ import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; -import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInventory, ResourceAllowance } from '../../hosts.js'; +import type { + CapacityCalibration, + CapacitySample, + CapacitySamplePage, + HostInventory, + ResourceAllowance +} from '../../hosts.js'; const BOOTSTRAP_CPU = 0.5; const BOOTSTRAP_MEMORY = 1024 ** 3; @@ -9,59 +15,114 @@ const WINDOW_MS = 24 * 60 * 60_000; const MAX_SAMPLES = 10_000; type Phase = 'startup' | 'running'; interface AllowanceRow { - scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number; - window_start: string; observed_at: string; computed_at: string; + scope: string; + phase: Phase; + cpu_cores: number; + memory_bytes: number; + samples: number; + window_start: string; + observed_at: string; + computed_at: string; +} +function database() { + const handle = db(); + ensureHostSchema(handle); + return handle; } -function database() { const handle = db(); ensureHostSchema(handle); return handle; } /** Read retained minute peaks without contacting agents or recalculating allowances. */ -export function capacitySamples(hostId: string, options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }): CapacitySamplePage { +export function capacitySamples( + hostId: string, + options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number } +): CapacitySamplePage { const handle = database(); const clauses = ['s.bucket >= ?']; const values: (string | number)[] = [new Date(Date.now() - WINDOW_MS).toISOString()]; - if (options.scope === 'host') { clauses.push('s.host_id = ?'); values.push(hostId); } - if (options.phase !== 'all') { clauses.push('s.phase = ?'); values.push(options.phase); } + if (options.scope === 'host') { + clauses.push('s.host_id = ?'); + values.push(hostId); + } + if (options.phase !== 'all') { + clauses.push('s.phase = ?'); + values.push(options.phase); + } const where = clauses.join(' AND '); - const total = (handle.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`).get(...values) as { count: number }).count; + const total = ( + handle + .prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`) + .get(...values) as { count: number } + ).count; const pageSize = 50; const page = Math.min(options.page, Math.max(1, Math.ceil(total / pageSize))); - const samples = handle.prepare(`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId, + const samples = handle + .prepare( + `SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId, i.server, s.generation, s.phase, s.bucket AS timestamp, s.cpu_cores AS cpuCores, s.memory_bytes AS memoryBytes FROM ec_instance_resource_samples s LEFT JOIN ec_hosts h ON h.id=s.host_id LEFT JOIN native_instances i ON i.id=s.instance_id AND i.host_id=s.host_id - WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`) + WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?` + ) .all(...values, pageSize, (page - 1) * pageSize) as CapacitySample[]; return { samples, total, page, pageSize }; } export function validateInstanceResources(inventory: HostInventory): void { - if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string' - || inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.'); + if ( + inventory.instanceResourcesError !== undefined && + (typeof inventory.instanceResourcesError !== 'string' || + inventory.instanceResourcesError.length > 500) + ) + throw new Error('Invalid instance resource sampling error.'); const samples = inventory.instanceResources; if (samples === undefined) return; - if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.'); + if (!Array.isArray(samples) || samples.length > 1000) + throw new Error('Invalid instance resource samples.'); const seen = new Set(); for (const sample of samples) { - if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100 - || !Number.isSafeInteger(sample.generation) || sample.generation < 1 - || !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0 - || typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0 - || !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0 - || (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01) - || (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal) - || seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.'); + if ( + !sample || + typeof sample !== 'object' || + typeof sample.instanceId !== 'string' || + !sample.instanceId || + sample.instanceId.length > 100 || + !Number.isSafeInteger(sample.generation) || + sample.generation < 1 || + !Number.isSafeInteger(sample.uptimeSeconds) || + sample.uptimeSeconds < 0 || + typeof sample.cpuCores !== 'number' || + !Number.isFinite(sample.cpuCores) || + sample.cpuCores < 0 || + !Number.isSafeInteger(sample.memoryBytes) || + sample.memoryBytes <= 0 || + (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01) || + (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal) || + seen.has(sample.instanceId) + ) + throw new Error('Invalid instance resource sample.'); seen.add(sample.instanceId); } } /** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */ -export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void { - if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return; +export function recordInstanceResources( + hostId: string, + inventory: HostInventory, + timestamp: string +): void { + if ( + !inventory.instanceResources?.length || + !inventory.dockerRunning || + !inventory.cpuCount || + !inventory.memoryTotal + ) + return; const handle = database(); const bucket = timestamp.slice(0, 16) + ':00.000Z'; const observed = new Map(); - const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL'); + const owned = handle.prepare( + 'SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL' + ); const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples (host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?) ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET @@ -71,58 +132,125 @@ export function recordInstanceResources(hostId: string, inventory: HostInventory // retired and delayed pre-rebuild samples cannot contaminate the model. if (!owned.get(sample.instanceId, hostId, sample.generation)) continue; const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running'; - insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes); + insert.run( + hostId, + sample.instanceId, + sample.generation, + phase, + bucket, + sample.cpuCores, + sample.memoryBytes + ); const prior = observed.get(phase); - observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) }); + observed.set(phase, { + cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), + memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) + }); } - for (const [phase, peak] of observed) for (const scope of [hostId, '*']) { - const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined; - if (!previous || previous.computed_at < bucket) { - const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples - WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`) - .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as - { cpu_cores: number; memory_bytes: number; bucket: string }[]; - const percentile = (values: number[]) => values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1]; - const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu); - const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory)); - handle.prepare(`INSERT INTO ec_resource_allowances + for (const [phase, peak] of observed) + for (const scope of [hostId, '*']) { + const previous = handle + .prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?') + .get(scope, phase) as AllowanceRow | undefined; + if (!previous || previous.computed_at < bucket) { + const rows = handle + .prepare( + `SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples + WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?` + ) + .all( + ...(scope === '*' ? [] : [scope]), + phase, + new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), + MAX_SAMPLES + ) as { cpu_cores: number; memory_bytes: number; bucket: string }[]; + const percentile = (values: number[]) => + values.sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1]; + const cpu = Math.max(percentile(rows.map((row) => row.cpu_cores)), peak.cpu); + const memory = Math.ceil( + Math.max(percentile(rows.map((row) => row.memory_bytes)), peak.memory) + ); + handle + .prepare( + `INSERT INTO ec_resource_allowances (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples, - window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`) - .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp); - if (scope === '*' && fleetSettings().instanceResourceKeepDays) handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket=?') + const rows = database() + .prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?') .all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[]; const allowance = (phase: Phase): ResourceAllowance => { - const local = rows.find(row => row.scope === hostId && row.phase === phase); - const fleet = rows.find(row => row.scope === '*' && row.phase === phase); - const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20) - && Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000); + const local = rows.find((row) => row.scope === hostId && row.phase === phase); + const fleet = rows.find((row) => row.scope === '*' && row.phase === phase); + const learned = (row?: AllowanceRow) => + !!row && + row.samples >= (phase === 'startup' ? 3 : 20) && + Date.parse(row.observed_at) - Date.parse(row.window_start) >= + (phase === 'startup' ? 120_000 : 300_000); const selected = learned(local) ? local : learned(fleet) ? fleet : undefined; return { // A few higher observations can increase a fleet/bootstrap allowance // immediately. Decreasing it requires the evidence thresholds above. - cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0), - memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0), - source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap', + cpuCores: Math.max( + 0.01, + selected?.cpu_cores ?? BOOTSTRAP_CPU, + local?.cpu_cores ?? 0, + selected ? 0 : (fleet?.cpu_cores ?? 0) + ), + memoryBytes: Math.max( + selected?.memory_bytes ?? BOOTSTRAP_MEMORY, + local?.memory_bytes ?? 0, + selected ? 0 : (fleet?.memory_bytes ?? 0) + ), + source: selected ? (selected.scope === hostId ? 'host' : 'fleet') : 'bootstrap', samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0, observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null }; }; - const running = allowance('running'); const startup = allowance('startup'); - if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap'; + const running = allowance('running'); + const startup = allowance('startup'); + if ( + running.source === 'bootstrap' && + (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes) + ) + startup.source = 'bootstrap'; startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores); startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes); return { running, startup }; diff --git a/src/lib/server/hosts/capacity-history.ts b/src/lib/server/hosts/capacity-history.ts index 0a4bae6..d0a9bee 100644 --- a/src/lib/server/hosts/capacity-history.ts +++ b/src/lib/server/hosts/capacity-history.ts @@ -15,8 +15,11 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap const { duration, bucketMs } = windows[query.range]; const startAt = now - duration; const count = duration / bucketMs; - const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts - WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`) + const hosts = handle + .prepare( + `SELECT id,created_at FROM ec_hosts + WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id` + ) .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[]; // The primary key bounds each host scan to the selected interval. Aggregate // before crossing the worker boundary; do not return raw heartbeat rows. @@ -42,8 +45,14 @@ export function readCapacityHistory(query: CapacityQuery, now = Date.now()): Cap for (const point of points) { if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++; } - const buckets = samples.all(count - 1, startAt, bucketMs, host.id, - new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[]; + const buckets = samples.all( + count - 1, + startAt, + bucketMs, + host.id, + new Date(startAt).toISOString(), + new Date(now).toISOString() + ) as { bucket: number; cpu: number; memory: number }[]; for (const bucket of buckets) { const point = points[bucket.bucket]; if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue; diff --git a/src/lib/server/hosts/capacity-overview.ts b/src/lib/server/hosts/capacity-overview.ts index 1c5d7d5..df6b5a1 100644 --- a/src/lib/server/hosts/capacity-overview.ts +++ b/src/lib/server/hosts/capacity-overview.ts @@ -14,56 +14,161 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise !query.hostId || host.id === query.hostId).map(observation => { - const { host, calibration } = observation; - const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS; - const connected = host.connected && fresh; - const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some( - library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state))); - const reasons: string[] = []; - if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.'); - if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.'); - if (host.decommission) reasons.push('Host is being removed.'); - if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.'); - if (host.readiness !== 'ready') reasons.push(...host.readinessReasons); - if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) reasons.push('The default runtime is not ready.'); - if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.'); - if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.'); - const eligible = connected && host.readiness === 'ready' && reasons.length === 0; - const cpuCoresAvailable = fresh ? observation.freeCpu : null; - const memoryBytesAvailable = fresh ? observation.freeMemory : null; - const portsAvailable = freePorts(observation); - const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)), - memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)), - ports: portsAvailable }; - const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports); - const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null - : limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU'; - if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.'); - else if (eligible && estimatedServers === 0) reasons.push(`No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.`); - return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat, - eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable, - portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration }; - }).sort((first, second) => first.name.localeCompare(second.name)); - const eligible = hosts.filter(host => host.eligible); + const hosts: CapacityHost[] = [...snapshot.hosts.values()] + .filter(({ host }) => !query.hostId || host.id === query.hostId) + .map((observation) => { + const { host, calibration } = observation; + const fresh = + !!host.lastHeartbeat && + now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS; + const connected = host.connected && fresh; + const games = (['t5', 't6'] as const).filter( + (game) => + libraries[game] && + host.inventory.libraries?.some( + (library) => + library.id === libraries[game] && + ['ready', 'outdated'].includes(library.state) + ) + ); + const reasons: string[] = []; + if (!connected) + reasons.push( + host.enrolled + ? 'Waiting for a fresh host report.' + : 'This host has not enrolled yet.' + ); + if (host.mode !== 'active') + reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.'); + if (host.decommission) reasons.push('Host is being removed.'); + if (!host.credentialActive && host.enrolled) + reasons.push('Host credentials are inactive.'); + if (host.readiness !== 'ready') reasons.push(...host.readinessReasons); + if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) + reasons.push('The default runtime is not ready.'); + if (!requestedGames.some((game) => games.includes(game))) + reasons.push('Latest game files for this selection are not ready.'); + if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) + reasons.push('Storage is below the launch reserve.'); + const eligible = connected && host.readiness === 'ready' && reasons.length === 0; + const cpuCoresAvailable = fresh ? observation.freeCpu : null; + const memoryBytesAvailable = fresh ? observation.freeMemory : null; + const portsAvailable = freePorts(observation); + const limits = { + cpu: + cpuCoresAvailable === null + ? null + : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)), + memory: + memoryBytesAvailable === null + ? null + : Math.max( + 0, + Math.floor(memoryBytesAvailable / calibration.running.memoryBytes) + ), + ports: portsAvailable + }; + const estimatedServers = !eligible + ? 0 + : limits.cpu === null || limits.memory === null + ? null + : Math.min(limits.cpu, limits.memory, limits.ports); + const limitingResource: CapacityHost['limitingResource'] = + estimatedServers === null || !eligible + ? null + : limits.ports === estimatedServers + ? 'Ports' + : limits.memory === estimatedServers + ? 'Memory' + : 'CPU'; + if (eligible && (limits.cpu === null || limits.memory === null)) + reasons.push('Waiting for valid CPU and memory measurements.'); + else if (eligible && estimatedServers === 0) + reasons.push( + `No additional instances currently fit: ${limitingResource?.toLowerCase()} capacity is full.` + ); + return { + id: host.id, + name: host.name, + readiness: host.readiness, + connected, + updatedAt: host.lastHeartbeat, + eligible, + games, + reasons: [...new Set(reasons)], + cpuCoresAvailable, + memoryBytesAvailable, + portsAvailable, + totalPorts: host.portMax - host.portMin + 1, + estimatedServers, + limits, + limitingResource, + calibration + }; + }) + .sort((first, second) => first.name.localeCompare(second.name)); + const eligible = hosts.filter((host) => host.eligible); const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') => - eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0); + eligible.some((host) => host[key] === null) + ? null + : eligible.reduce((total, host) => total + (host[key] ?? 0), 0); const warnings: string[] = []; - if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.'); - else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.'); - else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`); - if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.'); - if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.'); - return { updatedAt: snapshot.updatedAt, hosts, - hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)), - warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'), - memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0), - eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active), - startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } }; + if (!hosts.length) + warnings.push( + query.hostId + ? 'The selected host is unavailable.' + : 'Add a host to start tracking launch capacity.' + ); + else if (!eligible.length) + warnings.push('No host is ready for this game selection. Review the host details below.'); + else if (eligible.length < hosts.length) + warnings.push( + `${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.` + ); + if (eligible.some((host) => host.estimatedServers === null)) + warnings.push('Some resource measurements are missing; the total estimate is unknown.'); + if (!requestedGames.some((game) => libraries[game])) + warnings.push('Upload game files before launching this selection.'); + return { + updatedAt: snapshot.updatedAt, + hosts, + hostOptions: [...snapshot.hosts.values()] + .map(({ host }) => ({ id: host.id, name: host.name })) + .sort((a, b) => a.name.localeCompare(b.name)), + warnings, + summary: { + estimatedServers: sum('estimatedServers'), + cpuCoresAvailable: sum('cpuCoresAvailable'), + memoryBytesAvailable: sum('memoryBytesAvailable'), + portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0), + eligibleHosts: eligible.length, + totalHosts: hosts.length, + startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active), + startupLimit: snapshot.startupLimit, + activeStartups: snapshot.active, + queued: snapshot.queued + } + }; } diff --git a/src/lib/server/hosts/capacity-page.ts b/src/lib/server/hosts/capacity-page.ts index 1634456..d12e14b 100644 --- a/src/lib/server/hosts/capacity-page.ts +++ b/src/lib/server/hosts/capacity-page.ts @@ -5,6 +5,7 @@ import type { CapacityData, CapacityQuery } from '../../capacity.js'; export async function capacityPage(query: CapacityQuery): Promise { const current = await readCapacityCurrent(query); - if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.'); + if (query.hostId && !current.hostOptions.some((host) => host.id === query.hostId)) + error(404, 'Host not found.'); return { query, current, history: await capacityHistoryReport(query) }; } diff --git a/src/lib/server/hosts/capacity.ts b/src/lib/server/hosts/capacity.ts index a27cc7a..e757985 100644 --- a/src/lib/server/hosts/capacity.ts +++ b/src/lib/server/hosts/capacity.ts @@ -10,9 +10,13 @@ const MEMORY_HEADROOM = 500_000_000; const CPU_USABLE_FRACTION = 0.95; export class CapacityError extends Error {} export interface PlacementOptions { - hostId?: string; libraryId: string; image: string; bindAddress?: string; + hostId?: string; + libraryId: string; + image: string; + bindAddress?: string; excludeHostIds?: string[]; - thorough?: boolean; excludeInstanceId?: string; + thorough?: boolean; + excludeInstanceId?: string; // Execution only: not-yet-submitted launches wait their turn instead of // requiring their entire backlog to fit before any one can start. dispatch?: boolean; @@ -20,12 +24,24 @@ export interface PlacementOptions { } /** One indexed port read per host, including both protocols and wildcard binds. */ -export function availableHostPort(host: Host, options: Pick = {}): number | null { +export function availableHostPort( + host: Host, + options: Pick = {} +): number | null { const bind = options.bindAddress ?? host.bindAddress; - const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?') + const rows = db() + .prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?') .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[]; - const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) - || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port)); + const used = new Set( + rows + .filter( + (row) => + row.bind_address === bind || + ['0.0.0.0', '::'].includes(bind) || + ['0.0.0.0', '::'].includes(row.bind_address) + ) + .map((row) => row.port) + ); for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port); for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port; return null; @@ -33,38 +49,66 @@ export function availableHostPort(host: Host, options: Pick memoryTotal) { + if ( + !cpuCount || + cpuUsagePercent === undefined || + !memoryTotal || + memoryAvailable === undefined || + memoryAvailable > memoryTotal + ) { throw new Error('Waiting for valid CPU and memory measurements.'); } let historicalCpu = cpuUsagePercent; let historicalAvailable = memoryAvailable; if (options.thorough) { - const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory - FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`) - .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null }; + const history = db() + .prepare( + `SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory + FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?` + ) + .get( + host.id, + new Date(Date.now() - 5 * 60_000).toISOString(), + cpuCount, + memoryTotal + ) as { cpu: number | null; memory: number | null }; historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent); historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable); } - const instances = db().prepare(`SELECT i.id, i.generation, + const instances = db() + .prepare( + `SELECT i.id, i.generation, NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at item.hostId === host.id).length ?? 0; + const incoming = db() + .prepare( + `SELECT COUNT(*) AS n FROM ec_host_migrations + WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?` + ) + .get(host.id, options.excludeInstanceId ?? '') as { n: number }; + const planned = options.planned?.filter((item) => item.hostId === host.id).length ?? 0; // Admission budgets the eventual running collection. Dispatch budgets only // the next startup; the rest of the queue waits for live capacity to recover. const launch = options.dispatch ? startup : running; let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores; - let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes; + let extraMemory = + planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes; let reservations = planned + 1 + incoming.n; - const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample])); + const measurements = new Map( + host.inventory.instanceResources?.map((sample) => [sample.instanceId, sample]) + ); for (const instance of instances) { if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue; if (instance.unsubmitted) { @@ -82,28 +126,45 @@ function projectedLoad(host: Host, options: PlacementOptions): number { extraMemory += startup.memoryBytes; reservations++; } - const cores = cpuCount * cpuUsagePercent / 100 + extraCpu; + const cores = (cpuCount * cpuUsagePercent) / 100 + extraCpu; const usedMemory = memoryTotal - memoryAvailable; const memory = usedMemory + extraMemory; const usableMemory = memoryTotal - MEMORY_HEADROOM; const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`; - if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`); - if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` - + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`); + if (cores > cpuCount * CPU_USABLE_FRACTION) + throw new Error( + `Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.` + ); + if (memory > usableMemory) + throw new Error( + `Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` + + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).` + ); // Recovered resources must not remain unavailable until the historical // average catches up. History ranks hosts after current capacity passes. const historicalMemory = memoryTotal - historicalAvailable + extraMemory; - const historicalCores = cpuCount * historicalCpu / 100 + extraCpu; + const historicalCores = (cpuCount * historicalCpu) / 100 + extraCpu; return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal); } /** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */ -export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } { +export function choosePlacement(options: PlacementOptions): { + host: Host; + port: number; + load: number; +} { const candidates: { host: Host; port: number; load: number }[] = []; const reasons: string[] = []; - for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) { + for (const host of listHosts().filter( + (host) => + (!options.hostId || host.id === options.hostId) && + !options.excludeHostIds?.includes(host.id) + )) { try { - requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES }); + requireReadyHost(host.id, { + image: options.image, + requiredBytes: INSTANCE_STORAGE_BYTES + }); assertReadyLibrary(host.id, options.libraryId); const port = availableHostPort(host, options); if (port === null) throw new Error('No available UDP/TCP game port.'); @@ -111,9 +172,13 @@ export function choosePlacement(options: PlacementOptions): { host: Host; port: const candidate = { host, port, load }; if (!options.thorough) return candidate; candidates.push(candidate); - } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); } + } catch (error) { + reasons.push(`${host.name}: ${(error as Error).message}`); + } } candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id)); if (candidates.length) return candidates[0]; - throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`); + throw new CapacityError( + `No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}` + ); } diff --git a/src/lib/server/hosts/delivery.ts b/src/lib/server/hosts/delivery.ts index ed37c73..88e0d04 100644 --- a/src/lib/server/hosts/delivery.ts +++ b/src/lib/server/hosts/delivery.ts @@ -6,14 +6,23 @@ const CHUNK_CHARACTERS = 16 * 1024; const CHUNK_BYTES = 64 * 1024; const encoder = new TextEncoder(); -function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Generator { +function* jsonParts( + operations: OperationDeliveryRow[], + serverTime: string +): Generator { yield '{"operations":['; for (const [index, operation] of operations.entries()) { if (index) yield ','; const metadata = JSON.stringify({ - id: operation.id, hostId: operation.host_id, kind: operation.kind, - instanceId: operation.instance_id, generation: operation.generation, state: operation.state, - error: operation.error, createdAt: operation.created_at, updatedAt: operation.updated_at + id: operation.id, + hostId: operation.host_id, + kind: operation.kind, + instanceId: operation.instance_id, + generation: operation.generation, + state: operation.state, + error: operation.error, + createdAt: operation.created_at, + updatedAt: operation.updated_at }); yield metadata.slice(0, -1) + ',"payload":'; // Load only the immutable payload currently being sent, after the @@ -26,10 +35,14 @@ function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Gen yield '],"serverTime":' + JSON.stringify(serverTime) + '}'; } -function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): Generator { +function* encodedChunks( + operations: OperationDeliveryRow[], + serverTime: string +): Generator { for (const part of jsonParts(operations, serverTime)) { if (Buffer.isBuffer(part)) { - for (let start = 0; start < part.length; start += CHUNK_BYTES) yield part.subarray(start, start + CHUNK_BYTES); + for (let start = 0; start < part.length; start += CHUNK_BYTES) + yield part.subarray(start, start + CHUNK_BYTES); continue; } for (let start = 0; start < part.length;) { @@ -45,7 +58,10 @@ function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): } /** Bound encoding work and yield between chunks so health requests keep flowing. */ -export function hostOperationsResponse(operations: OperationDeliveryRow[], serverTime = new Date().toISOString()): Response { +export function hostOperationsResponse( + operations: OperationDeliveryRow[], + serverTime = new Date().toISOString() +): Response { const chunks = encodedChunks(operations, serverTime); let cancelled = false; const stream = new ReadableStream({ @@ -56,7 +72,12 @@ export function hostOperationsResponse(operations: OperationDeliveryRow[], serve if (next.done) controller.close(); else controller.enqueue(next.value); }, - cancel() { cancelled = true; chunks.return(undefined); } + cancel() { + cancelled = true; + chunks.return(undefined); + } + }); + return new Response(stream, { + headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } }); - return new Response(stream, { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } }); } diff --git a/src/lib/server/hosts/history-counts.ts b/src/lib/server/hosts/history-counts.ts index a68407a..b7da6c6 100644 --- a/src/lib/server/hosts/history-counts.ts +++ b/src/lib/server/hosts/history-counts.ts @@ -6,7 +6,12 @@ const visible = (alias: string) => `${alias}.history_expired_at IS NULL`; /** Durable totals for global, host, server, and host/server filters; receipts update only changed membership. */ export function installHistoryCounts(handle: Database.Database): void { - if (handle.prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'").get()) return; + if ( + handle + .prepare("SELECT 1 FROM ec_host_schema_migrations WHERE name='history-counts-v1'") + .get() + ) + return; handle.transaction(() => { handle.exec(`CREATE TABLE ec_host_operation_history_counts ( host_id TEXT NOT NULL, server_name TEXT NOT NULL, total INTEGER NOT NULL, @@ -15,7 +20,12 @@ export function installHistoryCounts(handle: Database.Database): void { DROP INDEX ec_host_operations_important_history; CREATE INDEX ec_host_operations_important_history ON ec_host_operations(created_at) WHERE history_expired_at IS NULL AND ${importantOperationSql.replaceAll('o.', '')};`); - for (const [host, server] of [['', ''], ['o.host_id', ''], ['', 'o.server_name'], ['o.host_id', 'o.server_name']]) { + for (const [host, server] of [ + ['', ''], + ['o.host_id', ''], + ['', 'o.server_name'], + ['o.host_id', 'o.server_name'] + ]) { const groups = [host, server].filter(Boolean); handle.exec(`INSERT INTO ec_host_operation_history_counts SELECT ${host || "''"},${server || "''"},COUNT(*),COALESCE(SUM(o.routine=1),0),COALESCE(SUM(${important('o')}),0) @@ -37,6 +47,8 @@ export function installHistoryCounts(handle: Database.Database): void { WHEN OLD.host_id IS NOT NEW.host_id OR OLD.server_name IS NOT NEW.server_name OR OLD.routine IS NOT NEW.routine OR (${visible('OLD')}) IS NOT (${visible('NEW')}) OR ${important('OLD')} IS NOT ${important('NEW')} BEGIN ${adjust('OLD', -1)} ${adjust('NEW', 1)} END;`); - handle.prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')").run(); + handle + .prepare("INSERT INTO ec_host_schema_migrations(name) VALUES('history-counts-v1')") + .run(); })(); } diff --git a/src/lib/server/hosts/http.ts b/src/lib/server/hosts/http.ts index 3f25b08..6094cda 100644 --- a/src/lib/server/hosts/http.ts +++ b/src/lib/server/hosts/http.ts @@ -3,7 +3,8 @@ import type { Host } from '../../hosts.js'; import { authenticateCredential } from './index.js'; export function requireAgentTransport(request: Request): void { - if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') error(426, 'Host agents require HTTPS.'); + if (new URL(request.url).protocol !== 'https:' && process.env.EC_AGENT_ALLOW_HTTP !== 'true') + error(426, 'Host agents require HTTPS.'); } export function authenticateAgent(request: Request): Host { requireAgentTransport(request); @@ -15,33 +16,56 @@ export function authenticateAgent(request: Request): Host { } export function requireManagementOrigin(request: Request): void { const origin = request.headers.get('origin'); - if ((origin && origin !== new URL(request.url).origin) || request.headers.get('sec-fetch-site') === 'cross-site') error(403, 'Cross-origin host management requests are not allowed.'); + if ( + (origin && origin !== new URL(request.url).origin) || + request.headers.get('sec-fetch-site') === 'cross-site' + ) + error(403, 'Cross-origin host management requests are not allowed.'); // Agent credentials only authorize agent endpoints, never host management. - if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) error(403, 'Agent credentials cannot manage the controller.'); + if (request.headers.get('authorization')?.match(/^Bearer ec_(?:host|enroll)_/i)) + error(403, 'Agent credentials cannot manage the controller.'); } -export async function objectBody(request: Request, maxBytes = 1024 * 1024): Promise> { - if (Number(request.headers.get('content-length')) > maxBytes) error(413, 'Request is too large.'); +export async function objectBody( + request: Request, + maxBytes = 1024 * 1024 +): Promise> { + if (Number(request.headers.get('content-length')) > maxBytes) + error(413, 'Request is too large.'); const reader = request.body?.getReader(); if (!reader) error(400, 'Request body must be valid JSON.'); - const chunks: Uint8Array[] = []; let received = 0; + const chunks: Uint8Array[] = []; + let received = 0; try { while (true) { const { done, value } = await reader.read(); if (done) break; received += value.byteLength; - if (received > maxBytes) { void reader.cancel().catch(() => {}); error(413, 'Request is too large.'); } + if (received > maxBytes) { + void reader.cancel().catch(() => {}); + error(413, 'Request is too large.'); + } chunks.push(value); } - } finally { reader.releaseLock(); } + } finally { + reader.releaseLock(); + } const raw = Buffer.concat(chunks, received).toString('utf8'); let body: unknown; - try { body = JSON.parse(raw); } catch { error(400, 'Request body must be valid JSON.'); } - if (!body || typeof body !== 'object' || Array.isArray(body)) error(400, 'Request body must be an object.'); + try { + body = JSON.parse(raw); + } catch { + error(400, 'Request body must be valid JSON.'); + } + if (!body || typeof body !== 'object' || Array.isArray(body)) + error(400, 'Request body must be an object.'); return body as Record; } export function hostApiError(cause: unknown): Response { if (cause && typeof cause === 'object' && 'status' in cause && 'body' in cause) throw cause; - return json({ error: cause instanceof Error ? cause.message : 'Host request failed.' }, { status: 400 }); + return json( + { error: cause instanceof Error ? cause.message : 'Host request failed.' }, + { status: 400 } + ); } export function installerCommand(controller: string, token: string): string { const origin = new URL(controller).origin; diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 8992529..ee9f7d7 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -6,7 +6,13 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt import { isIP } from 'node:net'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; -import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js'; +import { + successfulObservation, + routineObservationSql, + importantOperationSql, + operationTargetSql, + safeOperationTarget +} from './observations.js'; import { currentOperationSource } from './operation-context.js'; import { logResultMetadata } from './log-results.js'; import { parseHostInventory } from './request-context.js'; @@ -14,7 +20,18 @@ import { fleetSettings } from '../settings.js'; import { recordInstanceResources, validateInstanceResources } from './calibration.js'; import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js'; import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js'; -import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationScope, HostOperationState } from '../../hosts.js'; +import type { + Host, + HostDecommission, + HostDeletion, + HostInventory, + HostMode, + HostOperation, + HostOperationFilter, + HostOperationHistoryPage, + HostOperationScope, + HostOperationState +} from '../../hosts.js'; export const AGENT_PROTOCOL_VERSION = 1; export const HEARTBEAT_MAX_AGE_MS = 90_000; @@ -23,67 +40,188 @@ const workEvents = new EventEmitter(); // Hold reply bodies only while a reader is awaiting this operation. No TTL cache or disk copy. const logReaders = new Map>(); workEvents.setMaxListeners(0); -export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise { - return new Promise(resolve => { - const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); }; +export function waitForHostWork( + hostId: string, + signal: AbortSignal, + timeoutMs = 20_000 +): Promise { + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + workEvents.off(hostId, done); + signal.removeEventListener('abort', done); + resolve(); + }; const timer = setTimeout(done, timeoutMs); - workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true }); + workEvents.on(hostId, done); + signal.addEventListener('abort', done, { once: true }); if (signal.aborted) done(); }); } const digest = (token: string) => createHash('sha256').update(token).digest('hex'); -function database() { const handle = db(); ensureHostSchema(handle); return handle; } -type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number }; +function database() { + const handle = db(); + ensureHostSchema(handle); + return handle; +} +type HostRow = { + id: string; + name: string; + hostname: string; + mode: HostMode; + port_min: number; + port_max: number; + bind_address: string; + public_address: string; + minimum_free_bytes: number; + created_at: string; + enrolled_at: string | null; + last_heartbeat: string | null; + inventory: string; + credential_active: number; +}; const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`; -type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null }; +type DecommissionRow = { + host_id: string; + operation_id: string; + token_hash: string; + stage: HostDecommission['stage']; + requested_at: string; + updated_at: string; + cleaned_at: string | null; + error: string | null; +}; function decommissionRow(hostId: string): DecommissionRow | undefined { - return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined; + return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as + DecommissionRow | undefined; } function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null { - return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at, - error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null; + return row + ? { + operationId: row.operation_id, + stage: row.stage, + requestedAt: row.requested_at, + updatedAt: row.updated_at, + error: row.error, + retryable: + row.stage === 'failed' && + getHostOperationStatus(row.operation_id)?.state === 'failed' + } + : null; } function hostFromRow(row: HostRow): Host { - const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined; + const enabling = database() + .prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?') + .get(row.id) as { requested_at: string } | undefined; const inventory = parseHostInventory(row.id, row.inventory); - const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS; + const connected = + !!row.credential_active && + !!row.last_heartbeat && + Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS; const reasons: string[] = []; if (!row.enrolled_at) reasons.push('Agent has not enrolled.'); else if (!row.credential_active) reasons.push('Agent credential has been revoked.'); if (!connected) reasons.push('Agent heartbeat is unavailable or stale.'); - if (connected) for (const lane of inventory.agentLanes ?? []) { - const state = hostAgentLaneState(lane); - if (state === 'stopped' || state === 'stalled') reasons.push(`Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.`); - } - if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.'); + if (connected) + for (const lane of inventory.agentLanes ?? []) { + const state = hostAgentLaneState(lane); + if (state === 'stopped' || state === 'stalled') + reasons.push( + `Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.` + ); + } + if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) + reasons.push('A compatible agent protocol is required.'); if (!inventory.dockerRunning) reasons.push('Docker is not ready.'); if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.'); - if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.'); + if ( + typeof inventory.storageAvailable !== 'number' || + inventory.storageAvailable < row.minimum_free_bytes + ) + reasons.push('Insufficient available game storage.'); const infrastructureReady = reasons.length === 0; const runtimeReady = !!inventory.runtimeImages?.length; - const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready'); + const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready'); if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.'); if (!libraryReady) reasons.push('No verified game-file library is available yet.'); if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.'); - if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.'); + if (row.mode === 'disabled') + reasons.push( + enabling + ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' + : 'Host is disabled.' + ); const decommission = presentDecommission(decommissionRow(row.id)); - if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.'); - return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max, - bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes, - createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline', + if (decommission) + reasons.push( + 'Host deletion is in progress; waiting for confirmed cleanup and agent removal.' + ); + return { + id: row.id, + name: row.name, + hostname: row.hostname, + mode: row.mode, + portMin: row.port_min, + portMax: row.port_max, + bindAddress: row.bind_address, + publicAddress: row.public_address, + minimumFreeBytes: row.minimum_free_bytes, + createdAt: row.created_at, + lastHeartbeat: row.last_heartbeat, + connected, + connectionState: connected ? 'online' : 'offline', enablingSince: enabling?.requested_at ?? null, - readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready', - readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission }; + readiness: decommission + ? 'deleting' + : enabling + ? 'preparing' + : row.mode === 'draining' + ? 'draining' + : row.mode === 'disabled' + ? 'disabled' + : !row.enrolled_at + ? 'preparing' + : !infrastructureReady + ? 'degraded' + : !runtimeReady + ? 'needs-runtime' + : !libraryReady + ? 'needs-game-files' + : 'ready', + readinessReasons: reasons, + inventory, + enrolled: !!row.enrolled_at, + credentialActive: !!row.credential_active, + decommission + }; +} +export function listHosts(): Host[] { + return ( + database() + .prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id') + .all() as HostRow[] + ).map(hostFromRow); } -export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); } export function getHost(id: string): Host | null { - const row = database().prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL').get(id) as HostRow | undefined; + const row = database() + .prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL') + .get(id) as HostRow | undefined; return row ? hostFromRow(row) : null; } -function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; } +function requireHost(id: string): Host { + const host = getHost(id); + if (!host) throw new Error('Host was not found.'); + return host; +} export class HostDeletionError extends Error { - constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); } + constructor( + message: string, + readonly status: 400 | 401 | 404 | 409, + readonly blockers: string[] = [] + ) { + super(message); + } } /** Controller records can be removed only after all remote instance ownership is resolved. */ @@ -93,34 +231,75 @@ export function hostDeletionStatus(id: string): HostDeletion { if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] }; const handle = database(); const blockers: string[] = []; - if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.'); - if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.'); - const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[]; + if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) + blockers.push( + 'Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.' + ); + if (host.enrolled && !host.credentialActive) + blockers.push( + 'Re-enroll this host so its agent can receive and complete the cleanup request.' + ); + const instances = handle + .prepare( + 'SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server' + ) + .all(id) as { server: string }[]; if (instances.length) { - const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : ''); - blockers.push(`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`); + const names = + instances + .slice(0, 5) + .map((instance) => instance.server) + .join(', ') + (instances.length > 5 ? ', …' : ''); + blockers.push( + `Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.` + ); } - const pinned = handle.prepare(`SELECT d.server FROM server_deployments d + const pinned = handle + .prepare( + `SELECT d.server FROM server_deployments d JOIN native_creation_intents q ON q.deployment_id=d.id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL AND json_extract(q.selection,'$.hostId')=? AND COALESCE(json_extract(q.selection,'$.automatic'),0)!=1 - ORDER BY d.server LIMIT 6`).all(id) as { server: string }[]; + ORDER BY d.server LIMIT 6` + ) + .all(id) as { server: string }[]; if (pinned.length) { - const names = pinned.slice(0, 5).map(job => job.server).join(', ') + (pinned.length > 5 ? ', …' : ''); - blockers.push(`Cancel the queued server launches pinned to this host before deleting it (${names}).`); + const names = + pinned + .slice(0, 5) + .map((job) => job.server) + .join(', ') + (pinned.length > 5 ? ', …' : ''); + blockers.push( + `Cancel the queued server launches pinned to this host before deleting it (${names}).` + ); } - const ports = handle.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?').get(id) as { count: number }; - if (ports.count && !instances.length) blockers.push(`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`); + const ports = handle + .prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?') + .get(id) as { count: number }; + if (ports.count && !instances.length) + blockers.push( + `This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.` + ); // A pending command may have reached the agent before its acknowledgement. // Only commands that have never been delivered are safe to cancel locally. - const operations = handle.prepare(`SELECT kind,state FROM ec_host_operations WHERE host_id=? + const operations = handle + .prepare( + `SELECT kind,state FROM ec_host_operations WHERE host_id=? AND (state IN ('accepted','running','unknown') OR (state='pending' AND last_delivered_at IS NOT NULL)) - ORDER BY created_at`).all(id) as { kind: string; state: string }[]; + ORDER BY created_at` + ) + .all(id) as { kind: string; state: string }[]; if (operations.length) { - const work = operations.slice(0, 5).map(operation => `${operation.kind}: ${operation.state}`).join(', ') + (operations.length > 5 ? ', …' : ''); - blockers.push(`${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.`); + const work = + operations + .slice(0, 5) + .map((operation) => `${operation.kind}: ${operation.state}`) + .join(', ') + (operations.length > 5 ? ', …' : ''); + blockers.push( + `${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.` + ); } return { allowed: blockers.length === 0, blockers }; } @@ -128,209 +307,526 @@ export function hostDeletionStatus(id: string): HostDeletion { /** Keep the host visible and authenticated until its independent cleanup worker proves removal. */ export function deleteHost(id: string): { deleted: boolean; operationId?: string } { const handle = database(); - return handle.transaction(() => { - const host = getHost(id); - if (!host) throw new HostDeletionError('Host not found.', 404); - if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId }; - const status = hostDeletionStatus(id); - if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers); - const timestamp = now(); - const reason = 'Host deletion requested before this operation was delivered to the agent.'; - handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) - SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id); - handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id); - handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id); - if (!host.enrolled) { - handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id); - return { deleted: true }; - } - const operationId = randomUUID(); - const token = 'ec_cleanup_' + randomBytes(32).toString('base64url'); - const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` }; - handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)") - .run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200)); - handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp); - handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?) - ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`) - .run(id, operationId, digest(token), timestamp, timestamp); - handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id); - queueMicrotask(() => workEvents.emit(id)); - return { deleted: false, operationId }; - }).immediate(); + return handle + .transaction(() => { + const host = getHost(id); + if (!host) throw new HostDeletionError('Host not found.', 404); + if (host.decommission && !host.decommission.retryable) + return { deleted: false, operationId: host.decommission.operationId }; + const status = hostDeletionStatus(id); + if (!status.allowed) + throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers); + const timestamp = now(); + const reason = + 'Host deletion requested before this operation was delivered to the agent.'; + handle + .prepare( + `INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) + SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL` + ) + .run(timestamp, reason, id); + handle + .prepare( + "UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL" + ) + .run(reason, timestamp, id); + handle + .prepare( + 'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL' + ) + .run(timestamp, id); + if (!host.enrolled) { + handle + .prepare( + "UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL" + ) + .run(timestamp, id); + return { deleted: true }; + } + const operationId = randomUUID(); + const token = 'ec_cleanup_' + randomBytes(32).toString('base64url'); + const payload = { + hostId: id, + operationId, + receiptToken: token, + receiptUrl: `/api/agent/decommission/${operationId}` + }; + handle + .prepare( + "INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)" + ) + .run( + operationId, + id, + JSON.stringify(payload), + timestamp, + timestamp, + (currentOperationSource() ?? 'Controller').slice(0, 200) + ); + handle + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)" + ) + .run(operationId, timestamp); + handle + .prepare( + `INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?) + ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL` + ) + .run(id, operationId, digest(token), timestamp, timestamp); + handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id); + queueMicrotask(() => workEvents.emit(id)); + return { deleted: false, operationId }; + }) + .immediate(); } -export function receiveDecommissionReceipt(operationId: string, token: string, input: Record): { acknowledged: true } { - if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); - return database().transaction(() => { - const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined; - if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); - if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400); - if (row.stage === 'completed') return { acknowledged: true as const }; - const operation = getHostOperationStatus(operationId)!; - if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409); - const timestamp = now(); - if (input.stage === 'failed') { - database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?") - .run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id); +export function receiveDecommissionReceipt( + operationId: string, + token: string, + input: Record +): { acknowledged: true } { + if (!token || token.length > 200) + throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); + return database() + .transaction(() => { + const row = database() + .prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?') + .get(operationId) as DecommissionRow | undefined; + if ( + !row || + !timingSafeEqual( + Buffer.from(row.token_hash, 'hex'), + Buffer.from(digest(token), 'hex') + ) + ) + throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); + if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) + throw new HostDeletionError('Invalid cleanup receipt stage.', 400); + if (row.stage === 'completed') return { acknowledged: true as const }; + const operation = getHostOperationStatus(operationId)!; + if (!['accepted', 'running', 'unknown'].includes(operation.state)) + throw new HostDeletionError('Cleanup has not been accepted by the host.', 409); + const timestamp = now(); + if (input.stage === 'failed') { + database() + .prepare( + "UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?" + ) + .run( + typeof input.error === 'string' + ? input.error.slice(0, 2000) + : 'Host cleanup needs attention. The cleanup worker will retry.', + timestamp, + row.host_id + ); + return { acknowledged: true as const }; + } + if (input.stage === 'cleaned') { + database() + .prepare( + "UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?" + ) + .run(timestamp, timestamp, row.host_id); + return { acknowledged: true as const }; + } + if (!row.cleaned_at) + throw new HostDeletionError( + 'The host must confirm file cleanup before confirming agent removal.', + 409 + ); + database() + .prepare( + "UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?" + ) + .run(timestamp, row.host_id); + database() + .prepare( + "UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?" + ) + .run( + JSON.stringify({ filesRemoved: true, agentRemoved: true }), + timestamp, + operationId + ); + database() + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)" + ) + .run(operationId, timestamp); + database() + .prepare( + 'UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL' + ) + .run(timestamp, row.host_id); + database() + .prepare( + 'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL' + ) + .run(timestamp, row.host_id); + if ( + database() + .prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'" + ) + .get() + ) { + database() + .prepare( + "UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?" + ) + .run(timestamp, row.host_id); + } + database() + .prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?") + .run(timestamp, row.host_id); return { acknowledged: true as const }; - } - if (input.stage === 'cleaned') { - database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id); - return { acknowledged: true as const }; - } - if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409); - database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); - database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId); - database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp); - database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id); - database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id); - if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) { - database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); - } - database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id); - return { acknowledged: true as const }; - }).immediate(); + }) + .immediate(); } function textField(value: unknown, label: string, max = 200): string { - if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`); + if ( + typeof value !== 'string' || + !value.trim() || + value.length > max || + // eslint-disable-next-line no-control-regex -- Reject control characters in host text fields. + /[\x00-\x1f]/.test(value) + ) + throw new Error(`${label} must be a nonempty string up to ${max} characters.`); return value.trim(); } -function port(value: unknown): number { if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535) throw new Error('Ports must be whole numbers between 1 and 65535.'); return value; } +function port(value: unknown): number { + if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535) + throw new Error('Ports must be whole numbers between 1 and 65535.'); + return value; +} export function createHost(input: Record): Host { - const id = randomUUID(); const name = textField(input.name, 'Host name'); - const min = port(input.portMin ?? 28960); const max = port(input.portMax ?? 28980); + const id = randomUUID(); + const name = textField(input.name, 'Host name'); + const min = port(input.portMin ?? 28960); + const max = port(input.portMax ?? 28980); if (max < min) throw new Error('The last game port must not precede the first.'); - const bind = input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address'); + const bind = + input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address'); if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); - database().prepare('INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)').run(id, name, min, max, bind, now()); + database() + .prepare( + 'INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)' + ) + .run(id, name, min, max, bind, now()); scheduleAvailabilityRefresh(); return requireHost(id); } export function updateHost(id: string, input: Record): Host { const host = requireHost(id); - if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.'); + if (host.decommission) + throw new Error('Host cleanup is in progress; its settings cannot change.'); const name = input.name === undefined ? host.name : textField(input.name, 'Host name'); let mode = input.mode ?? host.mode; - if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.'); - const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax); + if (!['active', 'draining', 'disabled'].includes(mode as string)) + throw new Error('Host mode must be active, draining, or disabled.'); + const min = port(input.portMin ?? host.portMin); + const max = port(input.portMax ?? host.portMax); if (max < min) throw new Error('The last game port must not precede the first.'); - const bind = input.bindAddress === undefined ? host.bindAddress : textField(input.bindAddress, 'Bind address'); + const bind = + input.bindAddress === undefined + ? host.bindAddress + : textField(input.bindAddress, 'Bind address'); if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); - const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address'); + const publicAddress = + input.publicAddress === undefined + ? host.publicAddress + : input.publicAddress === '' + ? '' + : textField(input.publicAddress, 'Public address'); const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes; - if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.'); + if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) + throw new Error('Minimum free storage must be a nonnegative byte count.'); database().transaction(() => { - if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.'); + if (input.mode === 'active' && host.mode === 'draining') + throw new Error('Wait for this host to finish draining before enabling it.'); if (input.mode === 'active' && host.mode === 'disabled') { - database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now()); + database() + .prepare( + 'INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))' + ) + .run(id, now()); mode = 'disabled'; } if (input.mode === 'draining') { - if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.'); - const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id - WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id); - if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.'); + if (host.enablingSince) + throw new Error('Wait for automatic host setup to finish before draining.'); + const incoming = database() + .prepare( + `SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id + WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1` + ) + .get(id); + if (incoming) + throw new Error( + 'Wait for incoming instance migrations to finish before draining this host.' + ); } - database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id); + database() + .prepare( + 'UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?' + ) + .run(name, mode, min, max, bind, publicAddress, minimum, id); })(); scheduleAvailabilityRefresh(); return requireHost(id); } -export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } { +export function createEnrollment( + hostId: string, + ttlMs = 15 * 60_000 +): { id: string; hostId: string; token: string; expiresAt: string } { const host = requireHost(hostId); - if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.'); - if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.'); - const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID(); + if (host.decommission && !host.decommission.retryable) + throw new Error('Host cleanup is in progress; enrollment is unavailable.'); + if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) + throw new Error('Enrollment lifetime must be at most one hour.'); + const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); + const id = randomUUID(); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); database().transaction(() => { - database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(now(), hostId); - database().prepare('INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)').run(id, hostId, digest(token), now(), expiresAt); + database() + .prepare( + 'UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL' + ) + .run(now(), hostId); + database() + .prepare( + 'INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)' + ) + .run(id, hostId, digest(token), now(), expiresAt); })(); return { id, hostId, token, expiresAt }; } -function issueCredential(hostId: string, credential = 'ec_host_' + randomBytes(32).toString('base64url')): string { - database().prepare('INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)').run(randomUUID(), hostId, digest(credential), now()); +function issueCredential( + hostId: string, + credential = 'ec_host_' + randomBytes(32).toString('base64url') +): string { + database() + .prepare( + 'INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)' + ) + .run(randomUUID(), hostId, digest(credential), now()); return credential; } export function validateInventory(value: unknown): HostInventory { - if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Host inventory must be an object.'); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Host inventory must be an object.'); const inventory = value as HostInventory; if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.'); - for (const key of ['hostname','agentVersion','os','architecture','cpu','dockerVersion']) { - if (inventory[key] !== undefined && (typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000)) throw new Error(`Invalid inventory ${key}.`); + for (const key of ['hostname', 'agentVersion', 'os', 'architecture', 'cpu', 'dockerVersion']) { + if ( + inventory[key] !== undefined && + (typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000) + ) + throw new Error(`Invalid inventory ${key}.`); } - for (const key of ['cpuCount','memoryTotal','memoryAvailable','storageTotal','storageAvailable','protocolVersion']) { - if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`); + for (const key of [ + 'cpuCount', + 'memoryTotal', + 'memoryAvailable', + 'storageTotal', + 'storageAvailable', + 'protocolVersion' + ]) { + if ( + inventory[key] !== undefined && + (typeof inventory[key] !== 'number' || + !Number.isSafeInteger(inventory[key]) || + Number(inventory[key]) < 0) + ) + throw new Error(`Invalid inventory ${key}.`); } - if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.'); + if ( + inventory.cpuUsagePercent !== undefined && + (typeof inventory.cpuUsagePercent !== 'number' || + !Number.isFinite(inventory.cpuUsagePercent) || + inventory.cpuUsagePercent < 0 || + inventory.cpuUsagePercent > 100) + ) + throw new Error('Invalid inventory cpuUsagePercent.'); validateInstanceResources(inventory); if (inventory.instanceStates !== undefined) { if (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000) { throw new Error('Invalid instance state inventory.'); } // A malformed server observation must not discard the host heartbeat. - inventory.instanceStates = inventory.instanceStates.filter(state => state && typeof state === 'object' - && typeof state.instanceId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) - && Number.isSafeInteger(state.generation) && state.generation >= 1 - && typeof state.state === 'string' && /^[a-z-]{1,40}$/.test(state.state) - && typeof state.healthy === 'boolean' && (!state.healthy || state.state === 'running') - && typeof state.observedAt === 'string' && state.observedAt.length <= 40 && Number.isFinite(Date.parse(state.observedAt))); + inventory.instanceStates = inventory.instanceStates.filter( + (state) => + state && + typeof state === 'object' && + typeof state.instanceId === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) && + Number.isSafeInteger(state.generation) && + state.generation >= 1 && + typeof state.state === 'string' && + /^[a-z-]{1,40}$/.test(state.state) && + typeof state.healthy === 'boolean' && + (!state.healthy || state.state === 'running') && + typeof state.observedAt === 'string' && + state.observedAt.length <= 40 && + Number.isFinite(Date.parse(state.observedAt)) + ); } - for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); - if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); - if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.'); + for (const key of ['dockerRunning', 'directoriesWritable']) + if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') + throw new Error(`Invalid inventory ${key}.`); + if ( + inventory.runtimeImages !== undefined && + (!Array.isArray(inventory.runtimeImages) || + inventory.runtimeImages.some((image) => typeof image !== 'string')) + ) + throw new Error('Invalid runtime image inventory.'); + if ( + inventory.capabilities !== undefined && + (!Array.isArray(inventory.capabilities) || + inventory.capabilities.length > 32 || + inventory.capabilities.some((value) => typeof value !== 'string' || value.length > 80)) + ) + throw new Error('Invalid agent capability inventory.'); if (inventory.plutoniumCache !== undefined) { const cache = inventory.plutoniumCache; - if (!cache || typeof cache !== 'object' || Array.isArray(cache) || !Number.isSafeInteger(cache.revision) || cache.revision < 1 - || !Number.isSafeInteger(cache.bytes) || cache.bytes < 0) throw new Error('Invalid Plutonium cache inventory.'); + if ( + !cache || + typeof cache !== 'object' || + Array.isArray(cache) || + !Number.isSafeInteger(cache.revision) || + cache.revision < 1 || + !Number.isSafeInteger(cache.bytes) || + cache.bytes < 0 + ) + throw new Error('Invalid Plutonium cache inventory.'); inventory.plutoniumCache = { revision: cache.revision, bytes: cache.bytes }; } if (inventory.agentUpdate !== undefined) { const update = inventory.agentUpdate; - const version = (value: unknown) => typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value); - if (!update || typeof update !== 'object' || Array.isArray(update) || typeof update.enabled !== 'boolean' - || !['current','checking','deferred','updating','updated','failed'].includes(update.state) - || !version(update.currentVersion) || (update.targetVersion !== undefined && !version(update.targetVersion)) - || (update.error !== undefined && (typeof update.error !== 'string' || update.error.length > 2000)) - || (update.checkedAt !== undefined && (typeof update.checkedAt !== 'string' || update.checkedAt.length > 80 || !Number.isFinite(Date.parse(update.checkedAt))))) { + const version = (value: unknown) => + typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value); + if ( + !update || + typeof update !== 'object' || + Array.isArray(update) || + typeof update.enabled !== 'boolean' || + !['current', 'checking', 'deferred', 'updating', 'updated', 'failed'].includes( + update.state + ) || + !version(update.currentVersion) || + (update.targetVersion !== undefined && !version(update.targetVersion)) || + (update.error !== undefined && + (typeof update.error !== 'string' || update.error.length > 2000)) || + (update.checkedAt !== undefined && + (typeof update.checkedAt !== 'string' || + update.checkedAt.length > 80 || + !Number.isFinite(Date.parse(update.checkedAt)))) + ) { throw new Error('Invalid agent update inventory.'); } // Only expose display fields, never the local recovery journal or secrets. - inventory.agentUpdate = { enabled: update.enabled, state: update.state, currentVersion: update.currentVersion, + inventory.agentUpdate = { + enabled: update.enabled, + state: update.state, + currentVersion: update.currentVersion, ...(update.targetVersion === undefined ? {} : { targetVersion: update.targetVersion }), ...(update.error === undefined ? {} : { error: update.error }), - ...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) }; + ...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) + }; } if (inventory.agentLanes !== undefined) { - if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4) throw new Error('Invalid agent lane inventory.'); + if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4) + throw new Error('Invalid agent lane inventory.'); // Keep valid lane diagnostics without discarding a host heartbeat over // one malformed sample. Do not persist thread details or raw errors. const names = new Set(); - inventory.agentLanes = inventory.agentLanes.filter(lane => { - if (!lane || !['health-1','health-2','read','write'].includes(lane.name) || names.has(lane.name) - || typeof lane.alive !== 'boolean' || typeof lane.busy !== 'boolean' - || typeof lane.pollAgeSeconds !== 'number' || !Number.isFinite(lane.pollAgeSeconds) || lane.pollAgeSeconds < 0 - || (lane.lastPollAt !== null && (typeof lane.lastPollAt !== 'string' || lane.lastPollAt.length > 40 || !Number.isFinite(Date.parse(lane.lastPollAt))))) return false; - names.add(lane.name); - return true; - }).map(({name,alive,busy,lastPollAt,pollAgeSeconds}) => ({name,alive,busy,lastPollAt,pollAgeSeconds})); + inventory.agentLanes = inventory.agentLanes + .filter((lane) => { + if ( + !lane || + !['health-1', 'health-2', 'read', 'write'].includes(lane.name) || + names.has(lane.name) || + typeof lane.alive !== 'boolean' || + typeof lane.busy !== 'boolean' || + typeof lane.pollAgeSeconds !== 'number' || + !Number.isFinite(lane.pollAgeSeconds) || + lane.pollAgeSeconds < 0 || + (lane.lastPollAt !== null && + (typeof lane.lastPollAt !== 'string' || + lane.lastPollAt.length > 40 || + !Number.isFinite(Date.parse(lane.lastPollAt)))) + ) + return false; + names.add(lane.name); + return true; + }) + .map(({ name, alive, busy, lastPollAt, pollAgeSeconds }) => ({ + name, + alive, + busy, + lastPollAt, + pollAgeSeconds + })); } - if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library => - !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.'); + if ( + inventory.libraries !== undefined && + (!Array.isArray(inventory.libraries) || + inventory.libraries.length > 2000 || + inventory.libraries.some( + (library) => + !library || + typeof library !== 'object' || + typeof library.id !== 'string' || + !library.id || + library.id.length > 200 || + ![ + 'missing', + 'downloading', + 'verifying', + 'ready', + 'failed', + 'outdated' + ].includes(library.state) + )) + ) + throw new Error('Invalid game library inventory.'); return inventory; } -export function enrollHost(token: unknown, rawInventory: unknown): { hostId: string; credential: string; heartbeatSeconds: number } { - if (typeof token !== 'string' || token.length > 200) throw new Error('Enrollment token is invalid or expired.'); +export function enrollHost( + token: unknown, + rawInventory: unknown +): { hostId: string; credential: string; heartbeatSeconds: number } { + if (typeof token !== 'string' || token.length > 200) + throw new Error('Enrollment token is invalid or expired.'); const inventory = validateInventory(rawInventory); return database().transaction(() => { - const enrollment = database().prepare('SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?').get(digest(token), now()) as { id: string; host_id: string } | undefined; + const enrollment = database() + .prepare( + 'SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?' + ) + .get(digest(token), now()) as { id: string; host_id: string } | undefined; if (!enrollment) throw new Error('Enrollment token is invalid or expired.'); - const changed = database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL').run(now(), enrollment.id); + const changed = database() + .prepare( + 'UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL' + ) + .run(now(), enrollment.id); if (changed.changes !== 1) throw new Error('Enrollment token has already been consumed.'); revokeCredentials(enrollment.host_id); const credential = issueCredential(enrollment.host_id); - database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id); + database() + .prepare( + 'UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?' + ) + .run( + now(), + now(), + inventory.hostname ?? '', + JSON.stringify(inventory), + enrollment.host_id + ); recordResources(enrollment.host_id, inventory, now()); scheduleAvailabilityRefresh(); return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 }; @@ -338,42 +834,67 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str } export function authenticateCredential(token: string): Host | null { if (!token || token.length > 200) return null; - const row = database().prepare('SELECT id,host_id,last_used_at FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL') - .get(digest(token)) as { id: string; host_id: string; last_used_at: string | null } | undefined; + const row = database() + .prepare( + 'SELECT id,host_id,last_used_at FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL' + ) + .get(digest(token)) as + { id: string; host_id: string; last_used_at: string | null } | undefined; if (!row) return null; const timestamp = Date.now(); if (row.last_used_at === null || timestamp - Date.parse(row.last_used_at) >= 60_000) { - database().prepare(`UPDATE ec_host_credentials SET last_used_at=? WHERE id=? AND revoked_at IS NULL - AND (last_used_at IS NULL OR last_used_at<=?)`) - .run(new Date(timestamp).toISOString(), row.id, new Date(timestamp - 60_000).toISOString()); + database() + .prepare( + `UPDATE ec_host_credentials SET last_used_at=? WHERE id=? AND revoked_at IS NULL + AND (last_used_at IS NULL OR last_used_at<=?)` + ) + .run( + new Date(timestamp).toISOString(), + row.id, + new Date(timestamp - 60_000).toISOString() + ); } return getHost(row.host_id); } export function revokeCredentials(hostId: string): void { const host = requireHost(hostId); - if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.'); - database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId); + if (host.decommission && !host.decommission.retryable) + throw new Error('Host cleanup needs its credentials until removal is confirmed.'); + database() + .prepare( + 'UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL' + ) + .run(now(), hostId); scheduleAvailabilityRefresh(); } export function rotateCredential(hostId: string, nextCredential?: string): string { requireHost(hostId); - if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential)) throw new Error('New credentials must contain 32 random bytes encoded as base64url.'); + if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential)) + throw new Error('New credentials must contain 32 random bytes encoded as base64url.'); return database().transaction(() => { if (nextCredential) { - const existing = database().prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?').get(digest(nextCredential)) as { host_id: string; revoked_at: string | null } | undefined; + const existing = database() + .prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?') + .get(digest(nextCredential)) as + { host_id: string; revoked_at: string | null } | undefined; if (existing) { - if (existing.host_id !== hostId || existing.revoked_at) throw new Error('This credential cannot be reused.'); + if (existing.host_id !== hostId || existing.revoked_at) + throw new Error('This credential cannot be reused.'); return nextCredential; } } - revokeCredentials(hostId); return issueCredential(hostId, nextCredential); + revokeCredentials(hostId); + return issueCredential(hostId, nextCredential); })(); } export function heartbeat(hostId: string, value: unknown): Host { - requireHost(hostId); const inventory = validateInventory(value); + requireHost(hostId); + const inventory = validateInventory(value); database().transaction(() => { const timestamp = now(); - database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId); + database() + .prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?') + .run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId); recordResources(hostId, inventory, timestamp); })(); const host = requireHost(hostId); @@ -383,159 +904,412 @@ export function heartbeat(hostId: string, value: unknown): Host { function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void { recordInstanceResources(hostId, inventory, timestamp); - database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples - (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`) - .run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null, - inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null); - if (fleetSettings().hostResourceKeepDays) database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at library.id === options.libraryId && library.state === 'ready')) reasons.push('The selected game library is not ready on this host.'); - if (options.image && !host.inventory.runtimeImages?.includes(options.image)) reasons.push('The selected runtime image is not cached on this host.'); - if (options.port !== undefined && !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)) reasons.push('The selected host port is not available.'); +export function requireReadyHost( + hostId: string, + options: { + libraryId?: string; + requiredBytes?: number; + image?: string; + port?: number; + bindAddress?: string; + protocol?: 'udp' | 'tcp'; + } = {} +): Host { + const host = requireHost(hostId); + const reasons = [...host.readinessReasons]; + if ( + options.requiredBytes !== undefined && + (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes + ) + reasons.push('Insufficient storage for this deployment.'); + if ( + options.libraryId && + !host.inventory.libraries?.some( + (library) => library.id === options.libraryId && library.state === 'ready' + ) + ) + reasons.push('The selected game library is not ready on this host.'); + if (options.image && !host.inventory.runtimeImages?.includes(options.image)) + reasons.push('The selected runtime image is not cached on this host.'); + if ( + options.port !== undefined && + !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol) + ) + reasons.push('The selected host port is not available.'); if (reasons.length) throw new Error(reasons.join(' ')); return host; } -export type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string }; +export type OperationRow = { + id: string; + host_id: string; + kind: string; + instance_id: string | null; + generation: number | null; + payload: string; + state: HostOperationState; + result: string | null; + error: string | null; + created_at: string; + updated_at: string; +}; export type OperationDeliveryRow = Omit; export type HostOperationStatus = Omit; -const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at'; -const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; +const operationStatusColumns = + 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at'; +const operationPayloadSql = + 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; const readOnlyOperationSql = `(o.kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read') OR (o.kind='rcon.execute' AND COALESCE(json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version'),0)))`; /** Optional cache downloads yield their write lane to deployments and host changes. */ export function shouldDeferPlutoniumPreparation(hostId: string): boolean { - return !!database().prepare(`SELECT 1 FROM ec_host_operations o WHERE o.host_id=? + return !!database() + .prepare( + `SELECT 1 FROM ec_host_operations o WHERE o.host_id=? AND o.state IN ('pending','accepted','running','unknown') AND o.kind!='plutonium.prepare' - AND NOT ${readOnlyOperationSql} LIMIT 1`).get(hostId); + AND NOT ${readOnlyOperationSql} LIMIT 1` + ) + .get(hostId); +} +function operationStatusFromRow(row: Omit): HostOperationStatus { + return { + id: row.id, + hostId: row.host_id, + kind: row.kind, + instanceId: row.instance_id, + generation: row.generation, + state: row.state, + result: row.result === null ? null : JSON.parse(row.result), + error: row.error, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} +function operationFromRow(row: OperationRow): HostOperation { + const status = operationStatusFromRow(row); + return { + ...status, + result: unpackFileJson(status.result), + payload: unpackFileJson(JSON.parse(row.payload)) + }; } -function operationStatusFromRow(row: Omit): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } -function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; } export function getHostOperation(id: string): HostOperation | null { - const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`) + const row = database() + .prepare( + `SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?` + ) .get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; } /** Polling and receipts need the outcome, never another copy of launch files. */ export function getHostOperationStatus(id: string): HostOperationStatus | null { - const row = database().prepare('SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?') + const row = database() + .prepare( + 'SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?' + ) .get(id) as Omit | undefined; return row ? operationStatusFromRow(row) : null; } export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { - const columns = summary ? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result` + const columns = summary + ? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result` : `${operationStatusColumns},${operationPayloadSql} AS payload`; - return (database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? AND o.history_expired_at IS NULL ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?`) - .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); + return ( + database() + .prepare( + `SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? AND o.history_expired_at IS NULL ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?` + ) + .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[] + ).map(operationFromRow); } /** Browse retained history without loading command payloads or credential-bearing results. */ -export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important', scope: HostOperationScope = {}): HostOperationHistoryPage { +export function listHostOperationHistory( + page = 1, + filter: HostOperationFilter = 'important', + scope: HostOperationScope = {} +): HostOperationHistoryPage { const handle = database(); const pageSize = 25; - const conditions = filter === 'all' ? [] : [filter === 'routine' ? 'o.routine=1' : `(${importantOperationSql})`]; + const conditions = + filter === 'all' + ? [] + : [filter === 'routine' ? 'o.routine=1' : `(${importantOperationSql})`]; conditions.push('o.history_expired_at IS NULL'); const parameters: string[] = []; - if (scope.host) { conditions.push('o.host_id=?'); parameters.push(scope.host); } - if (scope.server) { conditions.push('o.server_name=?'); parameters.push(scope.server); } - if (scope.operation) { conditions.push('o.id=?'); parameters.push(scope.operation); } + if (scope.host) { + conditions.push('o.host_id=?'); + parameters.push(scope.host); + } + if (scope.server) { + conditions.push('o.server_name=?'); + parameters.push(scope.server); + } + if (scope.operation) { + conditions.push('o.id=?'); + parameters.push(scope.operation); + } const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; return handle.transaction((): HostOperationHistoryPage => { const countColumn = filter === 'all' ? 'total' : filter; // An exact operation seeks its unique ID instead of paging through history. const total = scope.operation - ? (handle.prepare(`SELECT 1 AS total FROM ec_host_operations o ${where}`).get(...parameters) as { total: number } | undefined)?.total ?? 0 - : (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`) - .get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0; + ? (( + handle + .prepare(`SELECT 1 AS total FROM ec_host_operations o ${where}`) + .get(...parameters) as { total: number } | undefined + )?.total ?? 0) + : (( + handle + .prepare( + `SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?` + ) + .get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined + )?.total ?? 0); const totalPages = Math.max(1, Math.ceil(total / pageSize)); const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1); - const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind, + const operations = handle + .prepare( + `SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind, o.instance_id AS instanceId,o.generation,o.state, CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error, o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL ${where} - ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`) - .all(...parameters, pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations']; - return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine, - targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter, - page: currentPage, pageSize, total, totalPages }; + ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?` + ) + .all( + ...parameters, + pageSize, + (currentPage - 1) * pageSize + ) as HostOperationHistoryPage['operations']; + return { + operations: operations.map((operation) => ({ + ...operation, + routine: !!operation.routine, + targetPath: safeOperationTarget(operation.kind, operation.targetPath) + })), + filter, + page: currentPage, + pageSize, + total, + totalPages + }; })(); } /** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */ -export function pruneHostObservations(): number { return pruneHistory('observation'); } +export function pruneHostObservations(): number { + return pruneHistory('observation'); +} /** Heartbeats stop either launch generation at its next safe command boundary. */ export function pendingCreationCancellations(hostId: string): string[] { - const rows = database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS creation_id, + const rows = database() + .prepare( + `SELECT json_extract(payload,'$.createOperationId') AS creation_id, json_extract(payload,'$.recoveryRebuildOperationId') AS recovery_id FROM ec_host_operations - WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`) + WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')` + ) .all(hostId) as { creation_id: string; recovery_id: string | null }[]; - return rows.flatMap(row => row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id]); + return rows.flatMap((row) => + row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id] + ); } -type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string }; -export function queueHostOperation(hostId: string, kind: string, payload: Record, options: QueueHostOperationOptions = {}): HostOperation { - if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.'); +type QueueHostOperationOptions = { + instanceId?: string; + generation?: number; + idempotencyKey?: string; + id?: string; +}; +export function queueHostOperation( + hostId: string, + kind: string, + payload: Record, + options: QueueHostOperationOptions = {} +): HostOperation { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) + throw new Error('Operation payload must be an object.'); const operation = enqueueHostOperation(hostId, kind, JSON.stringify(payload), options); return getHostOperation(operation.id)!; } /** Keep immutable launch bytes separate so receipts and delivery do not rewrite large payloads. */ -export function queueHostOperationSerialized(hostId: string, kind: string, payload: Buffer, options: QueueHostOperationOptions = {}, routingPayload: Record = {}): HostOperationStatus { - if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125) throw new Error('Serialized operation payload must be a JSON object.'); - if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload)) throw new Error('Operation routing payload must be an object.'); +export function queueHostOperationSerialized( + hostId: string, + kind: string, + payload: Buffer, + options: QueueHostOperationOptions = {}, + routingPayload: Record = {} +): HostOperationStatus { + if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125) + throw new Error('Serialized operation payload must be a JSON object.'); + if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload)) + throw new Error('Operation routing payload must be an object.'); return enqueueHostOperation(hostId, kind, payload, options, JSON.stringify(routingPayload)); } -function enqueueHostOperation(hostId: string, kind: string, payload: string | Buffer, options: QueueHostOperationOptions, routingPayload?: string): HostOperationStatus { +function enqueueHostOperation( + hostId: string, + kind: string, + payload: string | Buffer, + options: QueueHostOperationOptions, + routingPayload?: string +): HostOperationStatus { const host = requireHost(hostId); - if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.'); - if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.'); + if (host.decommission) + throw new Error('Host cleanup is in progress; new operations are blocked.'); + if (kind === 'host.decommission') + throw new Error('Use the host deletion workflow to request cleanup.'); if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.'); - if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); + if ( + options.generation !== undefined && + (!Number.isSafeInteger(options.generation) || options.generation < 1) + ) + throw new Error('Instance generation must be a positive integer.'); const operation = database().transaction(() => { if (options.idempotencyKey) { - const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,o.history_expired_at,o.request_hash,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload - FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as - { id: string; kind: string; instance_id: string | null; generation: number | null; history_expired_at: string | null; request_hash: string | null; same_payload: number } | undefined; + const existing = database() + .prepare( + `SELECT o.id,o.kind,o.instance_id,o.generation,o.history_expired_at,o.request_hash,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload + FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?` + ) + .get(payload, hostId, options.idempotencyKey) as + | { + id: string; + kind: string; + instance_id: string | null; + generation: number | null; + history_expired_at: string | null; + request_hash: string | null; + same_payload: number; + } + | undefined; if (existing) { - const samePayload = existing.request_hash ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload; - if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !samePayload) throw new Error('Operation idempotency key is already associated with a different request.'); + const samePayload = existing.request_hash + ? existing.request_hash === createHash('sha256').update(payload).digest('hex') + : !!existing.same_payload; + if ( + existing.kind !== kind || + existing.instance_id !== (options.instanceId ?? null) || + existing.generation !== (options.generation ?? null) || + !samePayload + ) + throw new Error( + 'Operation idempotency key is already associated with a different request.' + ); return getHostOperationStatus(existing.id)!; } } - if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); - if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','plutonium.prepare','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.'); + if (['instance.create', 'create'].includes(kind)) requireReadyHost(hostId); + if ( + host.mode === 'disabled' && + kind !== 'instance.cancel-create' && + !( + host.enablingSince && + ['runtime.pull', 'library.download', 'plutonium.prepare', 'host.inspect'].includes( + kind + ) + ) + ) + throw new Error('Host is disabled. Enable it before requesting operations.'); const requestHash = createHash('sha256').update(payload).digest('hex'); const packed = packFileJson(JSON.parse(payload.toString())); payload = JSON.stringify(packed.value); - const id = options.id ?? randomUUID(); const timestamp = now(); - const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?') - .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null; - database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash) - VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)`) - .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, - routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName, requestHash); + const id = options.id ?? randomUUID(); + const timestamp = now(); + const serverName = options.instanceId + ? (( + database() + .prepare('SELECT server FROM native_instances WHERE id=?') + .get(options.instanceId) as { server: string } | undefined + )?.server ?? null) + : null; + database() + .prepare( + `INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash) + VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)` + ) + .run( + id, + hostId, + kind, + options.instanceId ?? null, + options.generation ?? null, + options.idempotencyKey ?? null, + routingPayload ?? payload, + timestamp, + timestamp, + (currentOperationSource() ?? 'Controller').slice(0, 200), + serverName, + requestHash + ); referenceBlobs('operation', id, packed.hashes); - if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload); - database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, - target_path=${operationTargetSql} WHERE o.id=?`).run(id); - database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); + if (routingPayload !== undefined) + database() + .prepare( + 'INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))' + ) + .run(id, payload); + database() + .prepare( + `UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, + target_path=${operationTargetSql} WHERE o.id=?` + ) + .run(id); + database() + .prepare( + 'INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)' + ) + .run(id, 'pending', timestamp); return getHostOperationStatus(id)!; })(); queueMicrotask(() => workEvents.emit(hostId)); return operation; } -export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] { +export function pendingHostOperations( + hostId: string, + lane: 'all' | 'read' | 'health' | 'write' = 'all' +): HostOperation[] { return pendingHostOperationRows(hostId, lane).map(operationFromRow); } type OperationLane = 'all' | 'read' | 'health' | 'write'; -export function pendingHostOperationRows(hostId: string, lane?: OperationLane, includePayload?: true): OperationRow[]; -export function pendingHostOperationRows(hostId: string, lane: OperationLane, includePayload: false): OperationDeliveryRow[]; +export function pendingHostOperationRows( + hostId: string, + lane?: OperationLane, + includePayload?: true +): OperationRow[]; +export function pendingHostOperationRows( + hostId: string, + lane: OperationLane, + includePayload: false +): OperationDeliveryRow[]; /** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */ -export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] { +export function pendingHostOperationRows( + hostId: string, + lane: OperationLane = 'all', + includePayload = true +): OperationDeliveryRow[] { const host = requireHost(hostId); if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return []; // Older agents hold activity.lock across the entire delivered batch, including @@ -543,23 +1317,42 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = ' // busy indefinitely. Drain the old batch by withholding ALL further work; // heartbeats and receipts still flow, and the new capability releases this hold. // This must live in the controller so the fix also works before an agent update. - if (database().prepare(`SELECT 1 FROM ec_host_operations + if ( + database() + .prepare( + `SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') - AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1`) - .get(hostId, Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), - Number(host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false))) return []; + AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1` + ) + .get( + hostId, + Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), + Number( + host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false + ) + ) + ) + return []; // Accepted/running/unknown commands keep the same immutable ID. The agent // consults its durable journal before acknowledging or executing a replay. const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`; const healthLane = host.inventory.capabilities?.includes('operations.health-lane'); - const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}` - : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`; + const laneFilter = + lane === 'all' + ? '' + : lane === 'health' + ? ` AND ${healthOnly}` + : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`; // Health never queues behind a delivered file batch. Keep background reads // in small batches to amortize delivery without handing out a long backlog. - const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25; + const limit = healthLane ? (lane === 'health' ? 1 : lane === 'read' ? 4 : 25) : 25; return database().transaction(() => { - const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns; - const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') + const columns = includePayload + ? `${operationStatusColumns},${operationPayloadSql} AS payload` + : operationStatusColumns; + const rows = database() + .prepare( + `SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') ${laneFilter} ${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','plutonium.prepare','host.inspect')" : ''} AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create')) @@ -571,92 +1364,183 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = ' WHERE creation.operation_id=o.id) ${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''} ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END, - COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`) + COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}` + ) // A short replay lease avoids handing the idle health lane its busy // sibling's work, but still recovers a lost receipt or agent restart. - .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), - ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])) as OperationDeliveryRow[]; + .all( + hostId, + Number(host.readiness === 'ready' || host.mode === 'draining'), + Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), + ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : []) + ) as OperationDeliveryRow[]; const preparationLimit = fleetSettings().preparationConcurrency; - const libraryKinds = ['library.download','library.import']; - const preparing = preparationLimit && rows.some(operation => libraryKinds.includes(operation.kind)) - ? new Set((database().prepare(`SELECT DISTINCT host_id FROM ec_host_operations + const libraryKinds = ['library.download', 'library.import']; + const preparing = + preparationLimit && rows.some((operation) => libraryKinds.includes(operation.kind)) + ? new Set( + ( + database() + .prepare( + `SELECT DISTINCT host_id FROM ec_host_operations WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL - AND state IN ('pending','accepted','running','unknown')`).all() as { host_id: string }[]).map(row => row.host_id)) - : new Set(); - const admitted = rows.filter(operation => { - if (!libraryKinds.includes(operation.kind) || preparing.has(hostId) || !preparationLimit) return true; + AND state IN ('pending','accepted','running','unknown')` + ) + .all() as { host_id: string }[] + ).map((row) => row.host_id) + ) + : new Set(); + const admitted = rows.filter((operation) => { + if ( + !libraryKinds.includes(operation.kind) || + preparing.has(hostId) || + !preparationLimit + ) + return true; if (preparing.size >= preparationLimit) return false; - preparing.add(hostId); return true; + preparing.add(hostId); + return true; }); const deliveredAt = now(); - for (const operation of admitted) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); + for (const operation of admitted) + database() + .prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?') + .run(deliveredAt, operation.id); return admitted; })(); } /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */ export function getHostOperationPayload(hostId: string, id: string): Buffer { - const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`) + const row = database() + .prepare( + `SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?` + ) .get(id, hostId) as { payload: Buffer } | undefined; if (!row) throw new Error('Host operation was not found for this host.'); - if (!row.payload.includes('"contentSha256"') || getHost(hostId)?.inventory.capabilities?.includes('files.blobs')) return row.payload; + if ( + !row.payload.includes('"contentSha256"') || + getHost(hostId)?.inventory.capabilities?.includes('files.blobs') + ) + return row.payload; return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString())))); } -export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus { +export function reportHostOperationStatus( + hostId: string, + id: string, + input: { state: unknown; result?: unknown; error?: unknown } +): HostOperationStatus { let liveLogResult: { value: unknown } | undefined; const operation = database().transaction(() => { requireHost(hostId); const operation = getHostOperationStatus(id); - if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.'); - if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.'); - if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.'); - if (['completed','failed'].includes(operation.state)) return operation; + if (!operation || operation.hostId !== hostId) + throw new Error('Operation was not found for this host.'); + if (operation.kind === 'host.decommission' && input.state === 'completed') + throw new Error('Host removal requires the cleanup worker completion receipt.'); + if ( + !['accepted', 'running', 'completed', 'failed', 'unknown'].includes( + input.state as string + ) + ) + throw new Error('Invalid operation state.'); + if (['completed', 'failed'].includes(operation.state)) return operation; const state = input.state as HostOperationState; if (operation.state === 'running' && state === 'accepted') return operation; - const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000); + const error = + input.error === undefined || input.error === null + ? null + : String(input.error).slice(0, 4000); let result = input.result === undefined ? null : JSON.stringify(input.result); - if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); + if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) + throw new Error('Operation result exceeds the limit.'); if (result !== null) { - const packed = packFileJson(operation.kind === 'logs.read' ? logResultMetadata(input.result) : input.result); + const packed = packFileJson( + operation.kind === 'logs.read' ? logResultMetadata(input.result) : input.result + ); result = JSON.stringify(packed.value); referenceBlobs('operation-result', id, packed.hashes); } - const observation = state === 'completed' && !error && successfulObservation(operation.kind, - operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result); - database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?') + const observation = + state === 'completed' && + !error && + successfulObservation( + operation.kind, + operation.kind === 'rcon.execute' + ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) + : {}, + input.result + ); + database() + .prepare( + 'UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?' + ) .run(state, result, error, now(), Number(observation), id); - if (operation.kind === 'logs.read' && state === 'completed') liveLogResult = { value: input.result ?? null }; + if (operation.kind === 'logs.read' && state === 'completed') + liveLogResult = { value: input.result ?? null }; if (operation.kind === 'host.decommission') { - if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id); - else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id); + if (state === 'failed') + database() + .prepare( + "UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?" + ) + .run( + error ?? + 'The agent could not start cleanup. Repair the installation, then retry deletion.', + now(), + id + ); + else + database() + .prepare( + "UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')" + ) + .run(now(), id); } if (operation.state !== state) { - database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error); - if (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect' || operation.kind === 'files.stage') { + database() + .prepare( + 'INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)' + ) + .run(id, state, now(), error); + if ( + (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect') || + operation.kind === 'files.stage' + ) { recordAgentReceipt(id, state, now()); } } return getHostOperationStatus(id)!; })(); - if (liveLogResult) for (const reader of logReaders.get(id) ?? []) { - reader.received = true; - reader.result = liveLogResult.value; - } + if (liveLogResult) + for (const reader of logReaders.get(id) ?? []) { + reader.received = true; + reader.result = liveLogResult.value; + } queueMicrotask(() => workEvents.emit(`operation:${id}`)); return operation; } -export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation { +export function reportHostOperation( + hostId: string, + id: string, + input: { state: unknown; result?: unknown; error?: unknown } +): HostOperation { reportHostOperationStatus(hostId, id, input); return getHostOperation(id)!; } -export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): Promise { +export async function waitHostOperationStatus( + id: string, + timeoutMs = 30_000 +): Promise { if (getHostOperationStatus(id)?.kind !== 'logs.read') return waitOperationStatus(id, timeoutMs); const reader: { received: boolean; result?: unknown } = { received: false }; const readers = logReaders.get(id) ?? new Set(); - readers.add(reader); logReaders.set(id, readers); + readers.add(reader); + logReaders.set(id, readers); try { const operation = await waitOperationStatus(id, timeoutMs); if (operation.state !== 'completed') return operation; - if (!reader.received) throw new Error('Log reply is no longer available. Request a new read.'); + if (!reader.received) + throw new Error('Log reply is no longer available. Request a new read.'); return { ...operation, result: reader.result }; } finally { readers.delete(reader); @@ -668,27 +1552,41 @@ async function waitOperationStatus(id: string, timeoutMs: number): Promise= deadline) { // Delivery and expiry both claim the pending row transactionally. // Never cancel a delivered check or a command with side effects. const expired = database().transaction(() => { const timestamp = now(); const error = 'Expired before delivery.'; - const result = database().prepare(`UPDATE ec_host_operations SET state='failed',error=?,updated_at=? + const result = database() + .prepare( + `UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=? AND state='pending' AND last_delivered_at IS NULL - AND kind IN ('instance.inspect','rcon.status-batch')`).run(error, timestamp, id); - if (result.changes) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)') - .run(id, 'failed', timestamp, error); + AND kind IN ('instance.inspect','rcon.status-batch')` + ) + .run(error, timestamp, id); + if (result.changes) + database() + .prepare( + 'INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)' + ) + .run(id, 'failed', timestamp, error); return result.changes > 0; })(); if (expired) { queueMicrotask(() => workEvents.emit(`operation:${id}`)); throw new Error(`Host operation ${id} expired before delivery.`); } - throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`); + throw new Error( + `Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.` + ); } - await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now()))); + await waitForHostWork( + `operation:${id}`, + new AbortController().signal, + Math.min(1000, Math.max(1, deadline - Date.now())) + ); } } export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise { @@ -697,30 +1595,75 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise return status.kind === 'logs.read' ? { ...operation, result: status.result } : operation; } -export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean { - const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress; +export function hostPortAvailable( + hostId: string, + candidate: number, + bindAddress?: string, + protocol: 'udp' | 'tcp' = 'udp', + instanceId?: string +): boolean { + const host = requireHost(hostId); + port(candidate); + const bind = bindAddress ?? host.bindAddress; if (host.decommission) return false; if (candidate < host.portMin || candidate > host.portMax) return false; - const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[]; - return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address))); + const rows = database() + .prepare( + 'SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?' + ) + .all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[]; + return !rows.some( + (row) => + row.instance_id !== instanceId && + (row.bind_address === bind || + ['0.0.0.0', '::'].includes(bind) || + ['0.0.0.0', '::'].includes(row.bind_address)) + ); } -export function nextHostPort(hostId: string, additionalPorts: number[] = [], bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp'): number | null { +export function nextHostPort( + hostId: string, + additionalPorts: number[] = [], + bindAddress?: string, + protocol: 'udp' | 'tcp' = 'udp' +): number | null { const host = requireHost(hostId); - for (let candidate = host.portMin; candidate <= host.portMax; candidate++) if (!additionalPorts.includes(candidate) && hostPortAvailable(hostId, candidate, bindAddress, protocol)) return candidate; + for (let candidate = host.portMin; candidate <= host.portMax; candidate++) + if ( + !additionalPorts.includes(candidate) && + hostPortAvailable(hostId, candidate, bindAddress, protocol) + ) + return candidate; return null; } -export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number { - const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp'; - if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.'); +export function reserveHostPort( + hostId: string, + instanceId: string, + options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {} +): number { + const host = requireHost(hostId); + const bind = options.bindAddress ?? host.bindAddress; + const protocol = options.protocol ?? 'udp'; + if (host.decommission) + throw new Error('Host cleanup is in progress; new port reservations are blocked.'); if (!isIP(bind)) throw new Error('Bind address must be an IP address.'); - if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.'); + if (!['udp', 'tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.'); if (!instanceId) throw new Error('An immutable instance ID is required.'); return database().transaction(() => { - const existing = database().prepare('SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?').get(hostId, instanceId, bind, protocol) as { port: number } | undefined; - if (existing && (options.port === undefined || options.port === existing.port)) return existing.port; + const existing = database() + .prepare( + 'SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?' + ) + .get(hostId, instanceId, bind, protocol) as { port: number } | undefined; + if (existing && (options.port === undefined || options.port === existing.port)) + return existing.port; const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol); - if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) throw new Error('No available game port on the selected host.'); - database().prepare('INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)').run(hostId, instanceId, bind, protocol, allocated, now()); + if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) + throw new Error('No available game port on the selected host.'); + database() + .prepare( + 'INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)' + ) + .run(hostId, instanceId, bind, protocol, allocated, now()); scheduleAvailabilityRefresh(); return allocated; })(); diff --git a/src/lib/server/hosts/instances.ts b/src/lib/server/hosts/instances.ts index 3dfb31e..7859084 100644 --- a/src/lib/server/hosts/instances.ts +++ b/src/lib/server/hosts/instances.ts @@ -5,10 +5,14 @@ import { observationReader } from '../fleet/status.js'; /** Current instances remain visible when their host or game stops reporting. */ export function hostInstances(hostId: string): ServerObservation[] { - const servers = db().prepare(`SELECT s.* FROM servers s + const servers = db() + .prepare( + `SELECT s.* FROM servers s JOIN native_instances i ON i.server = s.name WHERE i.host_id = ? AND i.retired_at IS NULL AND s.destroyed_at IS NULL - ORDER BY s.name`).all(hostId) as ServerRow[]; + ORDER BY s.name` + ) + .all(hostId) as ServerRow[]; if (!servers.length) return []; const observe = observationReader(); return servers.map(observe); diff --git a/src/lib/server/hosts/log-results.ts b/src/lib/server/hosts/log-results.ts index 6f563cb..99b9251 100644 --- a/src/lib/server/hosts/log-results.ts +++ b/src/lib/server/hosts/log-results.ts @@ -9,6 +9,8 @@ export function logResultMetadata(value: unknown): Record | nul ...(typeof result.truncated === 'boolean' ? { truncated: result.truncated } : {}), ...(typeof result.reset === 'boolean' ? { reset: result.reset } : {}), ...(Array.isArray(result.lines) ? { lineCount: result.lines.length } : {}), - ...(typeof result.contentBase64 === 'string' ? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') } : {}) + ...(typeof result.contentBase64 === 'string' + ? { bytesRead: Buffer.byteLength(result.contentBase64, 'base64') } + : {}) }; } diff --git a/src/lib/server/hosts/observations.ts b/src/lib/server/hosts/observations.ts index 5398b7c..e27469e 100644 --- a/src/lib/server/hosts/observations.ts +++ b/src/lib/server/hosts/observations.ts @@ -1,23 +1,50 @@ -export const observationKinds = ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read']; +export const observationKinds = [ + 'host.inspect', + 'instance.inspect', + 'rcon.status-batch', + 'files.read', + 'files.list', + 'files.stat', + 'files.manifest', + 'logs.read' +]; const commands = ['status', 'getstatus', 'version']; export function safeOperationTarget(kind: string, value: string | null): string | null { if (!value || value.length > 1024) return null; const paths = kind === 'files.manifest' ? value.split(', ') : [value]; - return paths.every(path => path === '.' || !/[\\\x00-\x1f\x7f]/.test(path) - && path.split('/').every(part => part !== '' && part !== '.' && part !== '..')) ? value : null; + return paths.every( + (path) => + path === '.' || + // eslint-disable-next-line no-control-regex -- Reject control characters in observed paths. + (!/[\\\x00-\x1f\x7f]/.test(path) && + path.split('/').every((part) => part !== '' && part !== '.' && part !== '..')) + ) + ? value + : null; } -export function successfulObservation(kind: string, payload: Record, result: unknown): boolean { - const routine = observationKinds.includes(kind) || kind === 'rcon.execute' && commands.includes(String(payload.command)); - const value = result && typeof result === 'object' ? result as Record : {}; +export function successfulObservation( + kind: string, + payload: Record, + result: unknown +): boolean { + const routine = + observationKinds.includes(kind) || + (kind === 'rcon.execute' && commands.includes(String(payload.command))); + const value = result && typeof result === 'object' ? (result as Record) : {}; // Older agents set ok=false when any target fails, despite completing the batch. const completedBatch = kind === 'rcon.status-batch' && Array.isArray(value.results); - return routine && (completedBatch || value.ok !== false) && value.healthy !== false && (value.error == null || value.error === ''); + return ( + routine && + (completedBatch || value.ok !== false) && + value.healthy !== false && + (value.error == null || value.error === '') + ); } /** Uses the operation alias o; serialized payloads may live outside the routing row. */ -export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map(kind => `'${kind}'`).join(',')}) THEN 1 +export const routineObservationSql = `(CASE WHEN o.kind IN (${observationKinds.map((kind) => `'${kind}'`).join(',')}) THEN 1 WHEN o.kind='rcon.execute' THEN json_extract(COALESCE( (SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload),'$.command') IN ('status','getstatus','version') ELSE 0 END)`; @@ -32,7 +59,8 @@ export const successfulObservationSql = `(o.state='completed' AND COALESCE(o.err export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unknown') OR (o.state='completed' AND o.observation_success=0) OR COALESCE(o.error,'')!='')`; -const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; +const payloadSql = + 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; /** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */ export const operationTargetSql = `CASE WHEN o.kind='plutonium.prepare' THEN CASE diff --git a/src/lib/server/hosts/overview.ts b/src/lib/server/hosts/overview.ts index e5bbd9d..b79cc9f 100644 --- a/src/lib/server/hosts/overview.ts +++ b/src/lib/server/hosts/overview.ts @@ -5,25 +5,44 @@ import { observationReader } from '../fleet/status.js'; import type { ServerRow } from '../fleet/servers.js'; import type { Host } from '../../hosts.js'; -export interface HostOverview extends Host { serverCount: number; playerCount: number; unreportedInstances: number } +export interface HostOverview extends Host { + serverCount: number; + playerCount: number; + unreportedInstances: number; +} /** Read current placements once and use the fleet's shared telemetry/RCON selection. */ export function hostOverview(): HostOverview[] { const hosts = listHosts(); - const instances = db().prepare(`SELECT s.*, i.host_id FROM native_instances i + const instances = db() + .prepare( + `SELECT s.*, i.host_id FROM native_instances i JOIN servers s ON s.name = i.server - WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL`).all() as (ServerRow & { host_id: string })[]; - const counts = new Map(); + WHERE i.retired_at IS NULL AND s.destroyed_at IS NULL` + ) + .all() as (ServerRow & { host_id: string })[]; + const counts = new Map< + string, + { serverCount: number; playerCount: number; unreportedInstances: number } + >(); if (instances.length) { const observe = observationReader(); for (const instance of instances) { - const count = counts.get(instance.host_id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 }; + const count = counts.get(instance.host_id) ?? { + serverCount: 0, + playerCount: 0, + unreportedInstances: 0 + }; count.serverCount++; const observation = observe(instance); - if (observation.health === 'online' && observation.last_players !== null) count.playerCount += observation.last_players; + if (observation.health === 'online' && observation.last_players !== null) + count.playerCount += observation.last_players; else count.unreportedInstances++; counts.set(instance.host_id, count); } } - return hosts.map(host => ({ ...presentHostPreparation(host), ...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 }) })); + return hosts.map((host) => ({ + ...presentHostPreparation(host), + ...(counts.get(host.id) ?? { serverCount: 0, playerCount: 0, unreportedInstances: 0 }) + })); } diff --git a/src/lib/server/hosts/preparation.ts b/src/lib/server/hosts/preparation.ts index a9669c3..a85a299 100644 --- a/src/lib/server/hosts/preparation.ts +++ b/src/lib/server/hosts/preparation.ts @@ -1,25 +1,65 @@ import { db } from '../db/index.js'; import { config } from '../config.js'; -import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation, shouldDeferPlutoniumPreparation } from './index.js'; -import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js'; +import { + AGENT_PROTOCOL_VERSION, + getHost, + queueHostOperation, + shouldDeferPlutoniumPreparation +} from './index.js'; +import { + getHostLibraryCache, + latestPublishedLibraries, + queueLibraryDownload +} from '../game-files/index.js'; import { GAME_FILE_GAMES } from '../../game-files.js'; -import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js'; +import type { + Host, + HostOperationState, + HostPreparation, + HostPreparationItem +} from '../../hosts.js'; import { scheduleAvailabilityRefresh } from './availability.js'; import { readSetting } from '../maintenance/settings.js'; import type { PlutoniumRelease } from '../../plutonium.js'; const activeStates = ['pending', 'accepted', 'running', 'unknown']; const supersededDownload = 'Superseded by newer game files before delivery to the host.'; -type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null }; -type Target = { item: HostPreparationItem; image?: string; versionId?: string; revision?: number; requiredBytes: number; operation?: PreparationOperation }; +type PreparationOperation = { + id: string; + sequence: number; + last_delivered_at: string | null; + state: HostOperationState; + updated_at: string; + error: string | null; +}; +type Target = { + item: HostPreparationItem; + image?: string; + versionId?: string; + revision?: number; + requiredBytes: number; + operation?: PreparationOperation; +}; type QueuedSelection = { libraryId: string }; -type LibraryTarget = Pick[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean }; +type LibraryTarget = Pick< + ReturnType[number], + 'id' | 'sizeBytes' | 'manifestSha256' +> & { name: string; queued?: boolean }; /** Accepted requests retain their prerequisites until placement owns an instance. */ function queuedSelections(hostId: string): QueuedSelection[] { // Standalone host/library readers can initialize before provisioning migrations. - if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return []; - return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q + if ( + !db() + .prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'" + ) + .get() + ) + return []; + return db() + .prepare( + `SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q JOIN server_deployments d ON d.id=q.deployment_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL @@ -30,169 +70,337 @@ function queuedSelections(hostId: string): QueuedSelection[] { JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') - AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[]; + AND (m.target_host_id IS NULL OR m.target_host_id=?)` + ) + .all(hostId, hostId) as QueuedSelection[]; } function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] { const latest = latestPublishedLibraries(); - const latestIds = new Set(latest.map(version => version.id)); + const latestIds = new Set(latest.map((version) => version.id)); const queued = new Map(); - const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256 + const get = db() + .prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256 FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`); for (const selection of selections) { if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue; - const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined; - if (version) queued.set(version.id, { ...version, queued: true, - name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` }); + const version = get.get(selection.libraryId) as + | { + id: string; + game: string; + version: string; + sizeBytes: number; + manifestSha256: string; + } + | undefined; + if (version) + queued.set(version.id, { + ...version, + queued: true, + name: `${GAME_FILE_GAMES.find((game) => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` + }); } // Finish prerequisites for accepted work before downloading newer optional files. return [...queued.values(), ...latest]; } -function latestOperation(hostId: string, kind: string, field?: string, value?: string): PreparationOperation | undefined { - return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations - WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`) - .get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as PreparationOperation | undefined; +function latestOperation( + hostId: string, + kind: string, + field?: string, + value?: string +): PreparationOperation | undefined { + return db() + .prepare( + `SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations + WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1` + ) + .get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as + PreparationOperation | undefined; } function retryAt(hostId: string, target: Target): number { const operation = target.operation; if (!operation || operation.state !== 'failed') return 0; - const kind = target.image ? 'runtime.pull' : target.versionId ? 'library.download' : 'plutonium.prepare'; + const kind = target.image + ? 'runtime.pull' + : target.versionId + ? 'library.download' + : 'plutonium.prepare'; const field = target.image ? '$.image' : target.versionId ? '$.libraryVersionId' : null; const value = target.image ?? target.versionId; const match = field ? 'AND json_extract(payload,?)=?' : ''; const bindings = [hostId, kind, ...(field ? [field, value!] : [])]; // Count consecutive failures, including across controller restarts. Unknown // operations are never retried under a new ID: their journal must reconcile. - const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations + const { count } = db() + .prepare( + `SELECT COUNT(*) AS count FROM ec_host_operations WHERE host_id=? AND kind=? ${match} AND state='failed' AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations - WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`) + WHERE host_id=? AND kind=? ${match} AND state='completed'),0)` + ) .get(...bindings, ...bindings) as { count: number }; - return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1))); + return ( + Date.parse(operation.updated_at) + + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1))) + ); } function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null { - if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' }; - if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` }; - if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' }; - if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' }; - if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' }; - if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' }; - if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' }; + if (host.decommission) + return { + state: 'waiting', + detail: 'Automatic preparation is paused during host deletion.' + }; + if (host.mode !== 'active' && !host.enablingSince) + return { + state: 'waiting', + detail: `Automatic preparation is paused while this host is ${host.mode}.` + }; + if (!host.enrolled) + return { + state: 'waiting', + detail: 'Automatic preparation starts after agent installation.' + }; + if (!host.connected || !host.credentialActive) + return { state: 'waiting', detail: 'Waiting for the host to reconnect.' }; + if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) + return { state: 'blocked', detail: 'A compatible host agent is required.' }; + if (!host.inventory.dockerRunning) + return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' }; + if (!host.inventory.directoriesWritable) + return { state: 'blocked', detail: 'The host cannot write its managed directories.' }; return null; } function targets(host: Host): Target[] { - const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined; - const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation - && operation.sequence > enabling.operation_floor && operation.state === 'completed' - && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at); + const enabling = db() + .prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?') + .get(host.id) as { operation_floor: number } | undefined; + const verifiedThisEnable = (operation: PreparationOperation | undefined) => + !enabling || + (!!operation && + operation.sequence > enabling.operation_floor && + operation.state === 'completed' && + Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at)); const selections = queuedSelections(host.id); // Queued launches and migrations are not authority to pull custom images. - const result: Target[] = [config.defaultRuntimeImage].map(image => { + const result: Target[] = [config.defaultRuntimeImage].map((image) => { const operation = latestOperation(host.id, 'runtime.pull', 'image', image); - const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); - return { image, requiredBytes: host.minimumFreeBytes, + const runtimeReady = + host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); + return { + image, + requiredBytes: host.minimumFreeBytes, operation, - item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } }; + item: { + kind: 'runtime', + label: image, + state: runtimeReady ? 'ready' : 'waiting', + detail: runtimeReady + ? 'Runtime is cached.' + : 'Runtime will be downloaded automatically.' + } + }; }); for (const version of targetLibraries(selections)) { const cache = getHostLibraryCache(host.id, version.id); const matches = cache?.manifestSha256 === version.manifestSha256; - const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id); - const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation); - result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes, + const operation = latestOperation( + host.id, + 'library.download', + 'libraryVersionId', + version.id + ); + const ready = + matches && + ['ready', 'outdated'].includes(cache!.status) && + cache!.bytesDownloaded === version.sizeBytes && + verifiedThisEnable(operation); + result.push({ + versionId: version.id, + requiredBytes: + version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes, operation, - item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting', + item: { + kind: 'library', + label: version.name, + state: ready ? 'ready' : 'waiting', detail: version.queued - ? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.' - : ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.', - bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } }); + ? ready + ? 'Game files selected by queued launches are downloaded and verified.' + : 'Game files selected by queued launches will be downloaded and verified automatically.' + : ready + ? 'The latest game files are downloaded and verified.' + : 'The latest game files will be downloaded and verified automatically.', + bytesDownloaded: cache?.bytesDownloaded ?? 0, + bytesTotal: version.sizeBytes, + ...(cache ? { cache } : {}) + } + }); } if (host.inventory.capabilities?.includes('plutonium.cache')) { // Read the same production release evidence as freshRelease without // importing its fleet/provisioning observers into host preparation. - const release = readSetting('plutonium_release', { revision: null, checkedAt: null, error: null }); + const release = readSetting('plutonium_release', { + revision: null, + checkedAt: null, + error: null + }); const age = release.checkedAt ? Date.now() - Date.parse(release.checkedAt) : NaN; - const revision = !release.error && age >= 0 && age < 60 * 60_000 - && Number.isSafeInteger(release.revision) && release.revision! > 0 ? release.revision! : 0; + const revision = + !release.error && + age >= 0 && + age < 60 * 60_000 && + Number.isSafeInteger(release.revision) && + release.revision! > 0 + ? release.revision! + : 0; const cache = host.inventory.plutoniumCache; const ready = !!cache && cache.revision >= revision; - result.push({ revision, requiredBytes: host.minimumFreeBytes, + result.push({ + revision, + requiredBytes: host.minimumFreeBytes, operation: latestOperation(host.id, 'plutonium.prepare'), - item: { kind: 'plutonium', label: 'Plutonium cache (optional)', state: ready ? 'ready' : 'waiting', - detail: ready ? `Production revision r${cache.revision} is cached for new deployments.` - : cache ? `Production revision r${revision} will replace cached r${cache.revision} automatically.` - : 'Plutonium files will be cached after the runtime and game files are ready.' } }); + item: { + kind: 'plutonium', + label: 'Plutonium cache (optional)', + state: ready ? 'ready' : 'waiting', + detail: ready + ? `Production revision r${cache.revision} is cached for new deployments.` + : cache + ? `Production revision r${revision} will replace cached r${cache.revision} automatically.` + : 'Plutonium files will be cached after the runtime and game files are ready.' + } + }); } const blocker = hostBlocker(host); for (const target of result) { if (target.item.state === 'ready') continue; const operation = target.operation; if (operation) target.item.operationId = operation.id; - if (blocker) { Object.assign(target.item, blocker); continue; } + if (blocker) { + Object.assign(target.item, blocker); + continue; + } if (operation && activeStates.includes(operation.state)) { target.item.state = 'preparing'; - target.item.detail = target.versionId && operation.state === 'pending' && !operation.last_delivered_at - ? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.' - : operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.' - : target.image ? 'Downloading the runtime automatically.' - : target.revision !== undefined ? 'Downloading and verifying the optional Plutonium cache.' - : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.'; - } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) { - target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.'; - } else if ((target.revision !== undefined || !enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) { + target.item.detail = + target.versionId && operation.state === 'pending' && !operation.last_delivered_at + ? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.' + : operation.state === 'unknown' + ? 'Waiting for the agent to reconcile interrupted preparation.' + : target.image + ? 'Downloading the runtime automatically.' + : target.revision !== undefined + ? 'Downloading and verifying the optional Plutonium cache.' + : target.item.cache?.status === 'verifying' + ? 'Download complete. Verifying all game files.' + : 'Downloading game files. Verification starts after all files arrive.'; + } else if ( + operation && + Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at) + ) { + target.item.state = 'preparing'; + target.item.detail = 'Waiting for a fresh host inventory after preparation.'; + } else if ( + (target.revision !== undefined || + !enabling || + (operation?.sequence ?? 0) > enabling.operation_floor) && + retryAt(host.id, target) > Date.now() + ) { target.item.state = 'blocked'; target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`; } else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) { - target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.'; + target.item.state = 'blocked'; + target.item.detail = + 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.'; } } return result; } export function hostPreparationStatus(host: Host): HostPreparation { - const items = targets(host).map(target => target.item); + const items = targets(host).map((target) => target.item); if (host.enablingSince) { const published = latestPublishedLibraries(); - for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) { - items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` }); - } - } else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting', - detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.' }); + for (const game of GAME_FILE_GAMES) + if (!published.some((version) => version.game === game.id)) { + items.push({ + kind: 'library', + label: game.name, + state: 'blocked', + detail: `Upload and publish ${game.name} game files before this host can be enabled.` + }); + } + } else if (!items.some((item) => item.kind === 'library')) + items.push({ + kind: 'library', + label: 'Game files', + state: 'waiting', + detail: 'Upload Black Ops I or Black Ops II game files; hosts download the latest available files automatically.' + }); // Cache health is visible, but never makes a usable host ineligible or // delays re-enabling: a lobby's normal updater handles cache misses. - const required = items.filter(item => item.kind !== 'plutonium'); - const state = required.some(item => item.state === 'blocked') ? 'blocked' - : required.some(item => item.state === 'preparing') ? 'preparing' - : required.some(item => item.state === 'waiting') ? 'waiting' : 'ready'; + const required = items.filter((item) => item.kind !== 'plutonium'); + const state = required.some((item) => item.state === 'blocked') + ? 'blocked' + : required.some((item) => item.state === 'preparing') + ? 'preparing' + : required.some((item) => item.state === 'waiting') + ? 'waiting' + : 'ready'; return { state, items }; } export function presentHostPreparation(host: Host): Host { const preparation = hostPreparationStatus(host); - if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') { - return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.kind !== 'plutonium' && item.state !== 'ready').map(item => item.detail), preparation }; + if ( + ['needs-runtime', 'needs-game-files'].includes(host.readiness) && + preparation.state === 'preparing' + ) { + return { + ...host, + readiness: 'preparing', + readinessReasons: preparation.items + .filter((item) => item.kind !== 'plutonium' && item.state !== 'ready') + .map((item) => item.detail), + preparation + }; } return { ...host, preparation }; } function supersedeUndeliveredDownloads(hostId: string): void { - const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id)); - const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId + const required = new Set( + targetLibraries(queuedSelections(hostId)).map((version) => version.id) + ); + const pending = db() + .prepare( + `SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending' - AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[]; + AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1` + ) + .all(hostId) as { id: string; versionId: string }[]; const now = new Date().toISOString(); for (const operation of pending) { if (required.has(operation.versionId)) continue; // Delivered operations may already be writing files. Only unsubmitted // automatic work can be replaced without consulting the agent journal. - db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id); - db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload); - db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')") + db() + .prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?") + .run(supersededDownload, now, operation.id); + db() + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)" + ) + .run(operation.id, now, supersededDownload); + db() + .prepare( + "UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')" + ) .run(now, hostId, operation.versionId, operation.id); } } @@ -207,28 +415,55 @@ export function reconcileHostPreparation(hostId: string): HostPreparation { // Queue one required setup mutation at a time with fresh disk // inventory. A running optional cache yields to required work via // the heartbeat flag, then resumes under its original operation ID. - const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=? - AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId); - const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=? + const active = db() + .prepare( + `SELECT 1 FROM ec_host_operations WHERE host_id=? + AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1` + ) + .get(hostId); + const finished = db() + .prepare( + `SELECT updated_at FROM ec_host_operations WHERE host_id=? AND kind IN ('runtime.pull','library.download','library.import','plutonium.prepare') AND state IN ('completed','failed') AND (error IS NULL OR error != ?) - ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined; - if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) { + ORDER BY updated_at DESC LIMIT 1` + ) + .get(hostId, supersededDownload) as { updated_at: string } | undefined; + if ( + !active && + (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at)) + ) { // A failed or oversized library must not prevent other libraries // from preparing. The runtime still comes first when eligible. const candidates = targets(host); - const requiredReady = candidates.every(target => target.item.kind === 'plutonium' || target.item.state === 'ready'); - const target = candidates.find(target => target.item.state === 'waiting' && (target.item.kind !== 'plutonium' || requiredReady)); - if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image }); - else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true }); - else if (target?.revision !== undefined && !shouldDeferPlutoniumPreparation(host.id)) queueHostOperation(host.id, 'plutonium.prepare', { - revision: target.revision, minimumFreeBytes: host.minimumFreeBytes, automatic: true - }); + const requiredReady = candidates.every( + (target) => target.item.kind === 'plutonium' || target.item.state === 'ready' + ); + const target = candidates.find( + (target) => + target.item.state === 'waiting' && + (target.item.kind !== 'plutonium' || requiredReady) + ); + if (target?.image) + queueHostOperation(host.id, 'runtime.pull', { image: target.image }); + else if (target?.versionId) + queueLibraryDownload(host.id, target.versionId, { automatic: true }); + else if ( + target?.revision !== undefined && + !shouldDeferPlutoniumPreparation(host.id) + ) + queueHostOperation(host.id, 'plutonium.prepare', { + revision: target.revision, + minimumFreeBytes: host.minimumFreeBytes, + automatic: true + }); } } const status = hostPreparationStatus(host); if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') { - db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId); + db() + .prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'") + .run(hostId); db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId); scheduleAvailabilityRefresh(); } diff --git a/src/lib/server/hosts/releases.ts b/src/lib/server/hosts/releases.ts index 62cc689..9235e30 100644 --- a/src/lib/server/hosts/releases.ts +++ b/src/lib/server/hosts/releases.ts @@ -2,11 +2,15 @@ import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; /** adapter-node copies public assets to build/client; Vite serves static in development. */ -export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'): Promise { +export async function agentReleaseAsset( + file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json' +): Promise { // A prior production build must never shadow a freshly packaged development release. - if (process.env.NODE_ENV === 'development') return readFile(resolve('static/agent', file), 'utf8'); - try { return await readFile(resolve('build/client/agent', file), 'utf8'); } - catch (cause) { + if (process.env.NODE_ENV === 'development') + return readFile(resolve('static/agent', file), 'utf8'); + try { + return await readFile(resolve('build/client/agent', file), 'utf8'); + } catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; return readFile(resolve('static/agent', file), 'utf8'); } diff --git a/src/lib/server/hosts/request-context.ts b/src/lib/server/hosts/request-context.ts index 4f3aa63..23087d2 100644 --- a/src/lib/server/hosts/request-context.ts +++ b/src/lib/server/hosts/request-context.ts @@ -7,8 +7,9 @@ const requests = new AsyncLocalStorage<{ inventories: InventoryCache | null }>() export function withHostRequestContext(work: () => T | Promise): Promise { const context = { inventories: new Map() as InventoryCache | null }; return requests.run(context, async () => { - try { return await work(); } - finally { + try { + return await work(); + } finally { // Detached work can inherit this context. Release request data and // stop caching when the response is ready, even if that work continues. context.inventories = null; diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 4409865..41af912 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -1,16 +1,29 @@ import { createHash } from 'node:crypto'; -import { ensureFileBlobSchema, migrationRows, packFileJson, referenceBlobs } from '../db/file-blobs.js'; +import { + ensureFileBlobSchema, + migrationRows, + packFileJson, + referenceBlobs +} from '../db/file-blobs.js'; import { retentionClocks } from '../db/retention-clocks.js'; import { installHistoryCounts } from './history-counts.js'; import { migrationStep } from '../db/connection.js'; import type Database from 'better-sqlite3'; -import { successfulObservationSql, routineObservationSql, operationTargetSql, importantOperationSql } from './observations.js'; +import { + successfulObservationSql, + routineObservationSql, + operationTargetSql, + importantOperationSql +} from './observations.js'; const initialized = new WeakSet(); export function ensureHostSchema(handle: Database.Database): void { if (initialized.has(handle)) return; // Background readers open only after controller migrations have completed. - if (handle.readonly) { initialized.add(handle); return; } + if (handle.readonly) { + initialized.add(handle); + return; + } migrationStep('host schema and saved command files', () => migrateHostSchema(handle)); initialized.add(handle); } @@ -105,26 +118,39 @@ function migrateHostSchema(handle: Database.Database): void { CREATE INDEX IF NOT EXISTS idx_ec_host_credentials_host ON ec_host_credentials(host_id); CREATE INDEX IF NOT EXISTS idx_ec_host_enrollments_host ON ec_host_enrollments(host_id); `); - const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[]; - if (!columns.some(column => column.name === 'history_expired_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT'); - if (!columns.some(column => column.name === 'request_hash')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT'); - if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); - if (!columns.some(column => column.name === 'observation_success')) { + const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { + name: string; + }[]; + if (!columns.some((column) => column.name === 'history_expired_at')) + handle.exec('ALTER TABLE ec_host_operations ADD COLUMN history_expired_at TEXT'); + if (!columns.some((column) => column.name === 'request_hash')) + handle.exec('ALTER TABLE ec_host_operations ADD COLUMN request_hash TEXT'); + if (!columns.some((column) => column.name === 'last_delivered_at')) + handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); + if (!columns.some((column) => column.name === 'observation_success')) { handle.transaction(() => { - handle.exec('ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0'); - handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}`); + handle.exec( + 'ALTER TABLE ec_host_operations ADD COLUMN observation_success INTEGER NOT NULL DEFAULT 0' + ); + handle.exec( + `UPDATE ec_host_operations AS o SET observation_success=1 WHERE ${successfulObservationSql}` + ); })(); } handle.exec(`CREATE INDEX IF NOT EXISTS ec_host_operations_observation_retention ON ec_host_operations(updated_at) WHERE observation_success=1 AND state='completed'`); - if (!columns.some(column => column.name === 'routine')) { + if (!columns.some((column) => column.name === 'routine')) { handle.transaction(() => { handle.exec(`ALTER TABLE ec_host_operations ADD COLUMN routine INTEGER NOT NULL DEFAULT 0; ALTER TABLE ec_host_operations ADD COLUMN server_name TEXT; ALTER TABLE ec_host_operations ADD COLUMN target_path TEXT; ALTER TABLE ec_host_operations ADD COLUMN source TEXT;`); - const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { name: string }[]; - const nativeInstances = ['id', 'server'].every(name => nativeColumns.some(column => column.name === name)); + const nativeColumns = handle.prepare('PRAGMA table_info(native_instances)').all() as { + name: string; + }[]; + const nativeInstances = ['id', 'server'].every((name) => + nativeColumns.some((column) => column.name === name) + ); handle.exec(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, target_path=${operationTargetSql}, server_name=${nativeInstances ? '(SELECT n.server FROM native_instances n WHERE n.id=o.instance_id)' : 'NULL'}`); })(); @@ -135,59 +161,99 @@ function migrateHostSchema(handle: Database.Database): void { CREATE INDEX IF NOT EXISTS ec_host_operations_important_history ON ec_host_operations(created_at) WHERE ${importantOperationSql.replaceAll('o.', '')}`); handle.exec('CREATE TABLE IF NOT EXISTS ec_host_schema_migrations (name TEXT PRIMARY KEY)'); - if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('staged-operation-payloads')) { + if ( + !handle + .prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?') + .get('staged-operation-payloads') + ) { handle.transaction(() => { handle.exec(`INSERT OR IGNORE INTO ec_host_operation_payloads(operation_id,payload) SELECT id,payload FROM ec_host_operations WHERE kind='files.stage' AND json_type(payload,'$.files')='array'; UPDATE ec_host_operations SET payload=json_object('stageId',json_extract(payload,'$.stageId')) WHERE kind='files.stage' AND json_type(payload,'$.files')='array';`); - handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('staged-operation-payloads'); + handle + .prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)') + .run('staged-operation-payloads'); })(); } - if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('completed-status-batches')) { + if ( + !handle + .prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?') + .get('completed-status-batches') + ) { handle.transaction(() => { // Reclassify existing completed batches without discarding their per-server replies. handle.exec(`UPDATE ec_host_operations AS o SET observation_success=1 WHERE o.kind='rcon.status-batch' AND o.observation_success=0 AND ${successfulObservationSql}`); - handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('completed-status-batches'); + handle + .prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)') + .run('completed-status-batches'); })(); } const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[]; - if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT'); - const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { name: string }[]; - if (!allowanceColumns.some(column => column.name === 'usage_margin')) { + if (!hostColumns.some((column) => column.name === 'deleted_at')) + handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT'); + const allowanceColumns = handle.prepare('PRAGMA table_info(ec_resource_allowances)').all() as { + name: string; + }[]; + if (!allowanceColumns.some((column) => column.name === 'usage_margin')) { // Existing caches include 25% padding. Preserve learned evidence while // removing that padding exactly once, including dormant startup models. handle.transaction(() => { - handle.exec('ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1'); - handle.exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)'); + handle.exec( + 'ALTER TABLE ec_resource_allowances ADD COLUMN usage_margin REAL NOT NULL DEFAULT 1' + ); + handle.exec( + 'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores/1.25, memory_bytes=CAST(memory_bytes/1.25 AS INTEGER)' + ); })(); } - if (!handle.prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?').get('operation-file-blobs')) { + if ( + !handle + .prepare('SELECT 1 FROM ec_host_schema_migrations WHERE name=?') + .get('operation-file-blobs') + ) { handle.transaction(() => { - for (const row of migrationRows<{ id: string; payload: string; result: string | null }>(handle, + for (const row of migrationRows<{ id: string; payload: string; result: string | null }>( + handle, `SELECT o.rowid AS rid,o.id,COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.rowid>? AND o.kind IN ('files.stage','instance.create','instance.rebuild','instance.restart','instance.start','files.read') - ORDER BY o.rowid LIMIT 1`)) { + ORDER BY o.rowid LIMIT 1` + )) { const packed = packFileJson(JSON.parse(row.payload), handle); - handle.prepare('UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?') + handle + .prepare( + 'UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?) WHERE id=?' + ) .run(createHash('sha256').update(row.payload).digest('hex'), row.id); if (packed.hashes.size) { - handle.prepare(`INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?) - ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload`).run(row.id, JSON.stringify(packed.value)); - handle.prepare("UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?").run(row.id); + handle + .prepare( + `INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?) + ON CONFLICT(operation_id) DO UPDATE SET payload=excluded.payload` + ) + .run(row.id, JSON.stringify(packed.value)); + handle + .prepare( + "UPDATE ec_host_operations SET payload=json_remove(payload,'$.files') WHERE id=?" + ) + .run(row.id); referenceBlobs('operation', row.id, packed.hashes, handle); } if (row.result) { const result = packFileJson(JSON.parse(row.result), handle); if (result.hashes.size) { - handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run(JSON.stringify(result.value), row.id); + handle + .prepare('UPDATE ec_host_operations SET result=? WHERE id=?') + .run(JSON.stringify(result.value), row.id); referenceBlobs('operation-result', row.id, result.hashes, handle); } } } - handle.prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)').run('operation-file-blobs'); + handle + .prepare('INSERT INTO ec_host_schema_migrations(name) VALUES(?)') + .run('operation-file-blobs'); })(); } handle.exec(`CREATE TRIGGER IF NOT EXISTS release_operation_blobs AFTER DELETE ON ec_host_operations BEGIN diff --git a/src/lib/server/integrations/github-releases.ts b/src/lib/server/integrations/github-releases.ts index a43f673..721226c 100644 --- a/src/lib/server/integrations/github-releases.ts +++ b/src/lib/server/integrations/github-releases.ts @@ -6,25 +6,48 @@ const CACHE_MS = 5 * 60 * 1000; const cache = new Map }>(); export class GithubReleaseError extends Error { - constructor(message: string, public readonly status = 400) { super(message); } + constructor( + message: string, + public readonly status = 400 + ) { + super(message); + } } export function parseGithubReleaseUrl(input: string): { repository: string; tag: string | null } { let url: URL; - try { url = new URL(input); } catch { throw new GithubReleaseError('Enter a GitHub repository or release URL.'); } - if (url.protocol !== 'https:' || url.hostname !== 'github.com' || url.port || url.username || url.password || url.search) { + try { + url = new URL(input); + } catch { + throw new GithubReleaseError('Enter a GitHub repository or release URL.'); + } + if ( + url.protocol !== 'https:' || + url.hostname !== 'github.com' || + url.port || + url.username || + url.password || + url.search + ) { throw new GithubReleaseError('Use an https://github.com repository or release URL.'); } const match = /^\/([\w-]+)\/([\w.-]+)(?:\/(.*?))?\/?$/.exec(url.pathname); - if (!match || match[2] === '.' || match[2] === '..') throw new GithubReleaseError('Invalid GitHub repository URL.'); + if (!match || match[2] === '.' || match[2] === '..') + throw new GithubReleaseError('Invalid GitHub repository URL.'); const repository = `${match[1]}/${match[2]}`; const path = (match[3] ?? '').replace(/\/$/, ''); let tag: string | null = null; try { if (path.startsWith('releases/tag/')) tag = decodeURIComponent(path.slice(13)); - else if (path === 'releases' && url.hash.startsWith('#release-')) tag = decodeURIComponent(url.hash.slice(9)); + else if (path === 'releases' && url.hash.startsWith('#release-')) + tag = decodeURIComponent(url.hash.slice(9)); else if (!['', 'releases', 'releases/latest'].includes(path) || url.hash) throw new Error(); - } catch { throw new GithubReleaseError('Use a repository URL or a link to a specific GitHub release.'); } + } catch { + throw new GithubReleaseError( + 'Use a repository URL or a link to a specific GitHub release.' + ); + } + // eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names. if (tag !== null && (!tag || tag.length > 80 || /[\u0000-\u001f\u007f]/.test(tag))) { throw new GithubReleaseError('Release tags must contain 1–80 characters on one line.'); } @@ -32,7 +55,12 @@ export function parseGithubReleaseUrl(input: string): { repository: string; tag: } function safeAssetName(name: unknown): name is string { - return typeof name === 'string' && /^[^<>:"/\\|?*\u0000-\u001f]+\.dll$/i.test(name) && name.length <= 200; + return ( + typeof name === 'string' && + // eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names. + /^[^<>:"/\\|?*\u0000-\u001f]+\.dll$/i.test(name) && + name.length <= 200 + ); } async function limitedBody(response: Response, limit: number): Promise { @@ -49,45 +77,102 @@ async function limitedBody(response: Response, limit: number): Promise { const next = await reader.read(); if (next.done) break; size += next.value.byteLength; - if (size > limit) throw new GithubReleaseError('GitHub response exceeds the download size limit.', 502); + if (size > limit) + throw new GithubReleaseError( + 'GitHub response exceeds the download size limit.', + 502 + ); chunks.push(next.value); } - } finally { await reader.cancel(); } + } finally { + await reader.cancel(); + } return Buffer.concat(chunks); } async function fetchRelease(repository: string, tag: string | null): Promise { const suffix = tag === null ? 'latest' : `tags/${encodeURIComponent(tag)}`; try { - const response = await fetch(`https://api.github.com/repos/${repository}/releases/${suffix}`, { - headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Electric-Cherry', 'X-GitHub-Api-Version': '2022-11-28' }, - signal: AbortSignal.timeout(15_000), redirect: 'error' - }); + const response = await fetch( + `https://api.github.com/repos/${repository}/releases/${suffix}`, + { + headers: { + Accept: 'application/vnd.github+json', + 'User-Agent': 'Electric-Cherry', + 'X-GitHub-Api-Version': '2022-11-28' + }, + signal: AbortSignal.timeout(15_000), + redirect: 'error' + } + ); if (!response.ok) { await response.body?.cancel(); - if (response.status === 404) throw new GithubReleaseError('No published release found. Check the repository and tag; only public repositories are supported.', 404); - if ([403, 429].includes(response.status)) throw new GithubReleaseError('GitHub is limiting requests. Try checking again later.', 429); - throw new GithubReleaseError(`GitHub could not load this release (HTTP ${response.status}).`, 502); + if (response.status === 404) + throw new GithubReleaseError( + 'No published release found. Check the repository and tag; only public repositories are supported.', + 404 + ); + if ([403, 429].includes(response.status)) + throw new GithubReleaseError( + 'GitHub is limiting requests. Try checking again later.', + 429 + ); + throw new GithubReleaseError( + `GitHub could not load this release (HTTP ${response.status}).`, + 502 + ); } const data = JSON.parse((await limitedBody(response, 2 * 1024 * 1024)).toString('utf8')); - if (!Number.isSafeInteger(data.id) || data.id < 1 || data.draft || typeof data.tag_name !== 'string' || - !data.tag_name.trim() || data.tag_name.length > 80 || /[\u0000-\u001f\u007f]/.test(data.tag_name) || - !Number.isFinite(Date.parse(data.published_at)) || !Array.isArray(data.assets)) { + if ( + !Number.isSafeInteger(data.id) || + data.id < 1 || + data.draft || + typeof data.tag_name !== 'string' || + !data.tag_name.trim() || + data.tag_name.length > 80 || + // eslint-disable-next-line no-control-regex -- Reject control characters in release metadata and asset names. + /[\u0000-\u001f\u007f]/.test(data.tag_name) || + !Number.isFinite(Date.parse(data.published_at)) || + !Array.isArray(data.assets) + ) { throw new GithubReleaseError('GitHub returned invalid release metadata.', 502); } return { - repository, id: data.id, tag: data.tag_name, + repository, + id: data.id, + tag: data.tag_name, url: `https://github.com/${repository}/releases/tag/${encodeURIComponent(data.tag_name)}`, name: typeof data.name === 'string' ? data.name : data.tag_name, - notes: typeof data.body === 'string' ? data.body : '', publishedAt: data.published_at, + notes: typeof data.body === 'string' ? data.body : '', + publishedAt: data.published_at, prerelease: data.prerelease === true, - assets: data.assets.filter((a: any) => a && Number.isSafeInteger(a.id) && a.id > 0 && safeAssetName(a.name) && - a.state === 'uploaded' && Number.isSafeInteger(a.size) && a.size > 0 && a.size <= MAX_DLL_BYTES) - .map((a: any) => ({ id: a.id, name: a.name, size: a.size, digest: typeof a.digest === 'string' ? a.digest : null })) + assets: data.assets + .filter( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Validate dynamically decoded GitHub asset fields below. + (a: any) => + a && + Number.isSafeInteger(a.id) && + a.id > 0 && + safeAssetName(a.name) && + a.state === 'uploaded' && + Number.isSafeInteger(a.size) && + a.size > 0 && + a.size <= MAX_DLL_BYTES + ) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- The preceding filter validates the decoded asset fields. + .map((a: any) => ({ + id: a.id, + name: a.name, + size: a.size, + digest: typeof a.digest === 'string' ? a.digest : null + })) }; } catch (error) { if (error instanceof GithubReleaseError) throw error; - throw new GithubReleaseError('Could not reach GitHub or read the release. Try again shortly.', 502); + throw new GithubReleaseError( + 'Could not reach GitHub or read the release. Try again shortly.', + 502 + ); } } @@ -99,20 +184,46 @@ export async function getGithubRelease(url: string, fresh = false): Promise= 100) cache.delete(cache.keys().next().value!); const release = fetchRelease(repository, tag); cache.set(key, { expires: Date.now() + CACHE_MS, release }); - try { return await release; } catch (error) { cache.delete(key); throw error; } + try { + return await release; + } catch (error) { + cache.delete(key); + throw error; + } } -export async function downloadGithubAsset(release: GithubPluginRelease, asset: GithubPluginAsset): Promise { +export async function downloadGithubAsset( + release: GithubPluginRelease, + asset: GithubPluginAsset +): Promise { // Construct the URL from validated metadata. Never fetch a client-supplied download URL. - let url = new URL(`https://github.com/${release.repository}/releases/download/${encodeURIComponent(release.tag)}/${encodeURIComponent(asset.name)}`); + let url = new URL( + `https://github.com/${release.repository}/releases/download/${encodeURIComponent(release.tag)}/${encodeURIComponent(asset.name)}` + ); const signal = AbortSignal.timeout(60_000); try { for (let i = 0; i < 5; i++) { - if (url.protocol !== 'https:' || url.port || url.username || url.password || - !['github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com'].includes(url.hostname)) { - throw new GithubReleaseError('GitHub redirected this download to an unsupported host.', 502); + if ( + url.protocol !== 'https:' || + url.port || + url.username || + url.password || + ![ + 'github.com', + 'release-assets.githubusercontent.com', + 'objects.githubusercontent.com' + ].includes(url.hostname) + ) { + throw new GithubReleaseError( + 'GitHub redirected this download to an unsupported host.', + 502 + ); } - const response = await fetch(url, { redirect: 'manual', signal, headers: { 'User-Agent': 'Electric-Cherry' } }); + const response = await fetch(url, { + redirect: 'manual', + signal, + headers: { 'User-Agent': 'Electric-Cherry' } + }); if ([301, 302, 303, 307, 308].includes(response.status)) { await response.body?.cancel(); const location = response.headers.get('location'); @@ -122,19 +233,35 @@ export async function downloadGithubAsset(release: GithubPluginRelease, asset: G } if (!response.ok) { await response.body?.cancel(); - throw new GithubReleaseError(`GitHub asset download failed (HTTP ${response.status}).`, 502); + throw new GithubReleaseError( + `GitHub asset download failed (HTTP ${response.status}).`, + 502 + ); } const content = await limitedBody(response, MAX_DLL_BYTES); - if (content.length !== asset.size) throw new GithubReleaseError('Downloaded size differs from the release asset. Preview the release again.', 502); + if (content.length !== asset.size) + throw new GithubReleaseError( + 'Downloaded size differs from the release asset. Preview the release again.', + 502 + ); const sha = createHash('sha256').update(content).digest('hex'); - if (asset.digest?.startsWith('sha256:') && asset.digest.toLowerCase() !== `sha256:${sha}`) { - throw new GithubReleaseError('Downloaded checksum does not match GitHub’s asset digest.', 502); + if ( + asset.digest?.startsWith('sha256:') && + asset.digest.toLowerCase() !== `sha256:${sha}` + ) { + throw new GithubReleaseError( + 'Downloaded checksum does not match GitHub’s asset digest.', + 502 + ); } return content; } throw new GithubReleaseError('GitHub asset download redirected too many times.', 502); } catch (error) { if (error instanceof GithubReleaseError) throw error; - throw new GithubReleaseError('Could not download the DLL from GitHub. Try again shortly.', 502); + throw new GithubReleaseError( + 'Could not download the DLL from GitHub. Try again shortly.', + 502 + ); } } diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index 869b413..4bb0ed4 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -22,53 +22,119 @@ export function freshRelease(): number | null { } export function runningVersions(serverName?: string): PlutoniumVersion[] { const bindings = serverName === undefined ? [] : [serverName]; - const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`) + const versions = db() + .prepare( + `SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}` + ) .all(...bindings) as PlutoniumVersion[]; const target = freshRelease(); - const byServer = new Map(versions.map(version => [version.server, version])); - const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]); - const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`) - .all(...bindings) as { server: string }[]).map(row => row.server)); + const byServer = new Map(versions.map((version) => [version.server, version])); + const epochs = + serverName === undefined + ? latestGameSubmissions() + : new Map([[serverName, latestGameSubmission(serverName)]]); + const recovering = new Set( + ( + db() + .prepare( + `SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}` + ) + .all(...bindings) as { server: string }[] + ).map((row) => row.server) + ); const selected = serverName === undefined ? undefined : getServer(serverName); - const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : []; - return servers.map(server => { - const value = byServer.get(server.name) - ?? { server: server.name, revision: null, checked_at: null, error: null }; + const servers = + serverName === undefined + ? listServers() + : selected && !selected.destroyed_at + ? [selected] + : []; + return servers.map((server) => { + const value = byServer.get(server.name) ?? { + server: server.name, + revision: null, + checked_at: null, + error: null + }; const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN; const epoch = epochs.get(server.name); const stopped = server.desired_state === 'stopped'; - const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name); + const valid = + !value.error && + age >= 0 && + age < HOUR && + (!epoch || value.checked_at! > epoch) && + !recovering.has(server.name); const revision = valid && !stopped ? value.revision : null; - return { ...value, revision, stopped, - update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown' - : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' }; + return { + ...value, + revision, + stopped, + update_status: stopped + ? 'stopped' + : revision === null || target === null + ? 'unknown' + : revision < target + ? 'outdated' + : revision > target + ? 'ahead' + : 'current' + }; }); } -export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise { +export async function probeVersion( + server: string, + options: { maintenance?: boolean; operationId?: string } = {} +): Promise { // Startup verification calls this directly; only maintenance observers yield // ownership, without replacing provisioning's version evidence with an error. - const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined - && !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL - AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server)); + const suppressed = () => + (options.maintenance && hasActiveProvisioning(server)) || + (options.operationId !== undefined && + !db() + .prepare( + `SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL + AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL` + ) + .get(options.operationId, server)); // The verifying operation keeps its recovery slot until every readiness // check passes. Waiting on its own slot would deadlock native rebuilds. - const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId - && db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId)); + const recoveryBlocked = () => + restartRecoveryPending(server) && + !( + options.operationId && + db() + .prepare( + 'SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL' + ) + .get(server, options.operationId) + ); if (suppressed()) return null; const original = getServer(server); if (!original) return null; const epoch = latestGameSubmission(server); const recovering = recoveryBlocked(); - const health = !recovering && original.desired_state !== 'stopped' - ? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false }; + const health = + !recovering && original.desired_state !== 'stopped' + ? await nativeHealth(server).catch(() => ({ ok: false })) + : { ok: false }; if (suppressed()) return null; - let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' } - : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); + let result = !health.ok + ? { revision: null, error: 'Waiting for a healthy running server.' } + : await rconPlutoniumVersion(server).catch(() => ({ + revision: null, + error: 'Version check failed.' + })); if (suppressed()) return null; const current = getServer(server); // An older probe must not clear evidence already recorded for a new process // or replacement registration, including replacements with no submission yet. - if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null; + if ( + !current || + current.created_at !== original.created_at || + epoch !== latestGameSubmission(server) + ) + return null; const checkedAt = nowIso(); if (recoveryBlocked()) { result = { revision: null, error: 'Waiting for restart recovery.' }; @@ -78,8 +144,11 @@ export async function probeVersion(server: string, options: { maintenance?: bool result = { revision: null, error: 'Waiting for a version observation after startup.' }; } // Clear the old revision on failure: it is no longer evidence of what is running. - db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?) - ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`) + db() + .prepare( + `INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?) + ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error` + ) .run(server, result.revision, checkedAt, result.error); return result.revision; } @@ -87,14 +156,18 @@ export async function probeVersion(server: string, options: { maintenance?: bool let checking: Promise | null = null; export function checkPlutonium(): Promise { if (checking) return checking; - checking = check().finally(() => { checking = null; }); + checking = check().finally(() => { + checking = null; + }); return checking; } async function check(): Promise { await checkRelease(); const servers = listServers(); for (let i = 0; i < servers.length; i += 4) { - await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true }))); + await Promise.all( + servers.slice(i, i + 4).map((s) => probeVersion(s.name, { maintenance: true })) + ); } writeSetting('plutonium_versions_checked_at', nowIso()); } @@ -103,19 +176,33 @@ let checkingRelease: Promise | null = null; /** Refresh the published target without probing unrelated fleet servers. */ export function checkRelease(): Promise { if (checkingRelease) return checkingRelease; - checkingRelease = fetchRelease().finally(() => { checkingRelease = null; }); + checkingRelease = fetchRelease().finally(() => { + checkingRelease = null; + }); return checkingRelease; } async function fetchRelease(): Promise { const previous = release(); try { - const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' }); - if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`); + const response = await fetch(PLUTONIUM_MANIFEST_URL, { + signal: AbortSignal.timeout(config.httpTimeoutMs), + redirect: 'error' + }); + if (!response.ok) + throw new Error(`Updater manifest request failed (HTTP ${response.status}).`); const body = await response.text(); if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.'); - writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null }); + writeSetting('plutonium_release', { + revision: parsePlutoniumManifest(JSON.parse(body)), + checkedAt: nowIso(), + error: null + }); } catch (err) { - writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message }); + writeSetting('plutonium_release', { + ...previous, + checkedAt: nowIso(), + error: (err as Error).message + }); } } diff --git a/src/lib/server/integrations/rcon-diagnostics.ts b/src/lib/server/integrations/rcon-diagnostics.ts index 5e6c48a..329092f 100644 --- a/src/lib/server/integrations/rcon-diagnostics.ts +++ b/src/lib/server/integrations/rcon-diagnostics.ts @@ -2,10 +2,15 @@ import type { RconStatusResult } from './rcon.js'; /** Redact before truncating or hex encoding, including IPv6 and mapped IPv4 endpoints. */ export function redactRconAddresses(text: string): string { - return text.replace(/\[[0-9a-f:.]+(?:%[\w.-]+)?\](?::\d+)?|(? finish({ ok: false, reason: 'timeout', detail: `no reply within ${config.rcon.timeoutMs}ms` }), + () => + finish({ + ok: false, + reason: 'timeout', + detail: `no reply within ${config.rcon.timeoutMs}ms` + }), config.rcon.timeoutMs ); @@ -159,8 +165,8 @@ export function parseStatusHeader(text: string): Record { const out: Record = {}; for (const rawLine of text.split(/\r?\n/)) { const line = rawLine.trim(); - if (/^-+\s+-+/.test(line)) break; // scoreboard row — stop - if (/^\d+\s+/.test(line)) break; // player row that snuck in + if (/^-+\s+-+/.test(line)) break; // scoreboard row — stop + if (/^\d+\s+/.test(line)) break; // player row that snuck in const m = /^([a-z][a-z0-9_]*)\s*:\s*(.+)$/i.exec(line); if (m) out[m[1].toLowerCase()] = m[2].trim(); } @@ -168,10 +174,10 @@ export function parseStatusHeader(text: string): Record { } export interface RconPlayer { - num: number | null; // client slot — print dialect only + num: number | null; // client slot — print dialect only score: number; ping: number; - guid: string | null; // format varies by engine (hex, decimal, steam ID) + guid: string | null; // format varies by engine (hex, decimal, steam ID) name: string; } @@ -186,7 +192,7 @@ export function stripColorCodes(s: string): string { interface Column { name: string; start: number; - end: number; // exclusive + end: number; // exclusive } /** @@ -261,7 +267,8 @@ export function parsePlayerRoster(text: string): RconPlayer[] { const num = Number(numStr); const score = Number(slice(line, scoreI)); const ping = Number(slice(line, pingI)); - if (!Number.isFinite(num) || !Number.isFinite(score) || !Number.isFinite(ping)) continue; + if (!Number.isFinite(num) || !Number.isFinite(score) || !Number.isFinite(ping)) + continue; const guid = guidI >= 0 ? slice(line, guidI) || null : null; const name = stripColorCodes(slice(line, nameI)); @@ -325,10 +332,14 @@ const statusInFlight = new Map>(); async function sendStatus(endpoint: RconEndpoint): Promise { for (let attempt = 0; ; attempt++) { const result = await sendRcon(endpoint.host, endpoint.port, 'status'); - if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) { + if ( + result.ok || + (result.reason !== 'timeout' && result.reason !== 'send-error') || + attempt === 2 + ) { return result; } - await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)); } } @@ -343,8 +354,12 @@ export function formatRawReply(buf: Buffer): string { const lines: string[] = []; for (let i = 0; i < bytes.length; i += 16) { const chunk = bytes.subarray(i, i + 16); - const hex = Array.from(chunk).map((b) => b.toString(16).padStart(2, '0')).join(' '); - const ascii = Array.from(chunk).map((b) => (b >= 0x20 && b < 0x7f ? String.fromCharCode(b) : '.')).join(''); + const hex = Array.from(chunk) + .map((b) => b.toString(16).padStart(2, '0')) + .join(' '); + const ascii = Array.from(chunk) + .map((b) => (b >= 0x20 && b < 0x7f ? String.fromCharCode(b) : '.')) + .join(''); lines.push(`${i.toString(16).padStart(4, '0')} ${hex.padEnd(47, ' ')} ${ascii}`); } if (buf.length > RAW_REPLY_LIMIT) { @@ -353,28 +368,70 @@ export function formatRawReply(buf: Buffer): string { return lines.join('\n'); } -export interface AgentRconReply { ok?: boolean; reason?: string; text: string } +export interface AgentRconReply { + ok?: boolean; + reason?: string; + text: string; +} -export function agentRconStatusResult(reply: AgentRconReply, endpoint: { host: string; port: number }): RconStatusResult { +export function agentRconStatusResult( + reply: AgentRconReply, + endpoint: { host: string; port: number } +): RconStatusResult { if (reply.ok === false) { const failures: Record = { unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' }, - unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' }, - timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' }, + unreachable: { + reason: 'send-error', + detail: 'Agent could not reach the local RCON endpoint.' + }, + timeout: { + reason: 'timeout', + detail: 'Agent received no reply from the local RCON endpoint.' + }, authentication: { reason: 'bad-reply', detail: 'wrong RCON password' }, - 'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' } + 'invalid-reply': { + reason: 'bad-reply', + detail: 'Agent RCON reply is missing the 0xff prefix.' + } + }; + return { + ok: false, + endpoint, + ...(failures[reply.reason ?? ''] ?? { + reason: 'bad-reply' as const, + detail: 'Agent RCON query failed.' + }) }; - return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? { - reason: 'bad-reply' as const, detail: 'Agent RCON query failed.' - }) }; } const players = parsePlayerCount(reply.text); - if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: redactRconAddresses(reply.text).slice(0, 4000) }; - const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text); - return { ok: true, endpoint, status: { players, - capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients), - map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null, - gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) } }; + if (players === null) + return { + ok: false, + endpoint, + reason: 'bad-reply', + detail: 'Agent returned an unrecognized RCON status.', + rawReply: redactRconAddresses(reply.text).slice(0, 4000) + }; + const info = parseInfoLine(reply.text); + const header = parseStatusHeader(reply.text); + return { + ok: true, + endpoint, + status: { + players, + capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients), + map: + info.mapname ?? + info.sv_mapname ?? + header.map ?? + header.mapname ?? + header.sv_mapname ?? + null, + gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, + roster: parsePlayerRoster(reply.text) + } + }; } /** @@ -386,17 +443,29 @@ export function agentRconStatusResult(reply: AgentRconReply, endpoint: { host: s export async function rconStatusResult(server: string): Promise { const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null; if (native) { - const key = JSON.stringify(['agent', native.id, native.generation, latestGameSubmission(server)]); + const key = JSON.stringify([ + 'agent', + native.id, + native.generation, + latestGameSubmission(server) + ]); const existing = statusInFlight.get(key); if (existing) return existing; const endpoint = { host: `agent:${native.host_id}`, port: native.port }; const request = (async (): Promise => { try { - const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' }); + const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>( + native, + 'rcon.execute', + { command: 'status' } + ); return agentRconStatusResult(reply, endpoint); - } catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; } + } catch (error) { + return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; + } })().finally(() => statusInFlight.delete(key)); - statusInFlight.set(key, request); return request; + statusInFlight.set(key, request); + return request; } // Snapshot before any network work. A probe begun after a restart or a // replacement registration must not join one observing the old process. @@ -405,16 +474,28 @@ export async function rconStatusResult(server: string): Promise { statusInFlight.delete(key); }); + const request = probeStatus(ep).finally(() => { + statusInFlight.delete(key); + }); statusInFlight.set(key, request); return request; } @@ -459,7 +540,13 @@ async function probeStatus(ep: RconEndpoint): Promise { } else { detail = `unparseable reply (${send.buf.length} bytes) — see snippet below`; } - return { ok: false, endpoint: ep, reason: 'bad-reply', detail, rawReply: formatRawReply(send.buf) }; + return { + ok: false, + endpoint: ep, + reason: 'bad-reply', + detail, + rawReply: formatRawReply(send.buf) + }; } const header = parseStatusHeader(text); return { @@ -469,8 +556,19 @@ async function probeStatus(ep: RconEndpoint): Promise { // Prefer statusResponse `\info\` line values when present, fall // back to print-body `key: value` header lines otherwise. players, - capacity: positiveCapacity(info.sv_maxclients ?? info.sv_maxClients ?? header.sv_maxclients ?? header.sv_maxClients), - map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null, + capacity: positiveCapacity( + info.sv_maxclients ?? + info.sv_maxClients ?? + header.sv_maxclients ?? + header.sv_maxClients + ), + map: + info.mapname ?? + info.sv_mapname ?? + header.map ?? + header.mapname ?? + header.sv_mapname ?? + null, gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(text) } @@ -483,14 +581,26 @@ function positiveCapacity(value: string | undefined): number | null { } /** Query the running process, not the updater's on-disk files. */ -export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> { +export async function rconPlutoniumVersion( + server: string +): Promise<{ revision: number | null; error: string | null }> { const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null; if (native) { try { - const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'version' }); + const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { + command: 'version' + }); const revision = parsePlutoniumRevision(reply.text); - return { revision, error: revision === null ? 'Agent RCON reply did not identify a runtime revision.' : null }; - } catch (error) { return { revision: null, error: (error as Error).message }; } + return { + revision, + error: + revision === null + ? 'Agent RCON reply did not identify a runtime revision.' + : null + }; + } catch (error) { + return { revision: null, error: (error as Error).message }; + } } const { endpoint } = await endpointFor(server); if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' }; @@ -498,5 +608,9 @@ export async function rconPlutoniumVersion(server: string): Promise<{ revision: if (!response.ok) return { revision: null, error: response.detail }; const body = stripPrefix(response.buf); const revision = body?.trimStart().startsWith('print') ? parsePlutoniumRevision(body) : null; - return { revision, error: revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null }; + return { + revision, + error: + revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null + }; } diff --git a/src/lib/server/maintenance/index.ts b/src/lib/server/maintenance/index.ts index fc755db..4f18b94 100644 --- a/src/lib/server/maintenance/index.ts +++ b/src/lib/server/maintenance/index.ts @@ -4,10 +4,15 @@ import { maintenanceTick, plutoniumStatus } from './runtime.js'; import type { MaintenanceOverview } from '../../maintenance.js'; export function maintenanceOverview(selectedJob: number | null = null): MaintenanceOverview { - return { policies: maintenancePolicies(), recovery: watchdogOverview(), runtime: plutoniumStatus(selectedJob) }; + return { + policies: maintenancePolicies(), + recovery: watchdogOverview(), + runtime: plutoniumStatus(selectedJob) + }; } export async function runMaintenance(): Promise { // One failing observer must not prevent the other capabilities from reconciling. const results = await Promise.allSettled([maintenanceTick(), watchdogTick()]); - for (const result of results) if (result.status === 'rejected') console.error('[maintenance]', result.reason); + for (const result of results) + if (result.status === 'rejected') console.error('[maintenance]', result.reason); } diff --git a/src/lib/server/maintenance/policies.ts b/src/lib/server/maintenance/policies.ts index c059df0..84d475a 100644 --- a/src/lib/server/maintenance/policies.ts +++ b/src/lib/server/maintenance/policies.ts @@ -2,18 +2,31 @@ import { db } from '../db/index.js'; import type { MaintenancePolicy, MaintenancePolicies } from '../../maintenance.js'; function setting(key: string): string | undefined { - return (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value; + return ( + db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as + { value: string } | undefined + )?.value; } function save(key: string, value: boolean) { - db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value ? '1' : '0'); + db() + .prepare( + 'INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value' + ) + .run(key, value ? '1' : '0'); } export function maintenancePolicies(): MaintenancePolicies { // Preserve the old recovery switch without opting anyone into runtime upgrades. const legacy = setting('watchdog_enabled') !== '0'; return { paused: setting('maintenance_paused') === '1', - health: setting('maintenance_health') === undefined ? legacy : setting('maintenance_health') === '1', - drift: setting('maintenance_drift') === undefined ? legacy : setting('maintenance_drift') === '1', + health: + setting('maintenance_health') === undefined + ? legacy + : setting('maintenance_health') === '1', + drift: + setting('maintenance_drift') === undefined + ? legacy + : setting('maintenance_drift') === '1', updates: setting('maintenance_updates') === '1' }; } @@ -21,5 +34,9 @@ export function maintenanceAllowed(policy?: MaintenancePolicy): boolean { const policies = maintenancePolicies(); return !policies.paused && (!policy || policies[policy]); } -export function setMaintenancePolicy(policy: MaintenancePolicy, enabled: boolean) { save(`maintenance_${policy}`, enabled); } -export function pauseMaintenance(paused: boolean) { save('maintenance_paused', paused); } +export function setMaintenancePolicy(policy: MaintenancePolicy, enabled: boolean) { + save(`maintenance_${policy}`, enabled); +} +export function pauseMaintenance(paused: boolean) { + save('maintenance_paused', paused); +} diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 045e0cd..d6101e7 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -3,15 +3,36 @@ import { db } from '../db/index.js'; import { nativeInstance } from '../backends/native.js'; import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; -import { inspectForMonitoring, rconForMonitoring, monitoringStatus, monitoringIdentity, MonitoringChangedError } from '../fleet/health-observations.js'; +import { + inspectForMonitoring, + rconForMonitoring, + monitoringStatus, + monitoringIdentity, + MonitoringChangedError +} from '../fleet/health-observations.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; -import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js'; +import { + latestGameSubmission, + telemetryEvidence, + telemetryReport +} from '../fleet/status-evidence.js'; import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js'; -import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js'; +import { + cancelProvisioning, + requestWatchdogLifecycle, + escalateWatchdogRestart +} from '../provisioning/requests.js'; import { deploymentStage, finishDeployment } from '../provisioning/history.js'; import { recordProgress } from '../provisioning/progress.js'; import { withOperationSource } from '../hosts/operation-context.js'; -import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js'; +import type { + WatchdogAction, + WatchdogDetail, + WatchdogEvidence, + WatchdogIncident, + WatchdogOverview, + WatchdogState +} from '../../watchdog.js'; import { maintenanceAllowed, setMaintenancePolicy } from './policies.js'; @@ -21,8 +42,18 @@ export const WATCHDOG_FAILURE_MS = 120_000; export const WATCHDOG_RECOVERY_MS = 300_000; const iso = () => new Date().toISOString(); const pendingStates = ['queued', 'waiting', 'running', 'awaiting-health']; -const setting = (key: string) => (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value ?? null; -function setSetting(key: string, value: string) { db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value); } +const setting = (key: string) => + ( + db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as + { value: string } | undefined + )?.value ?? null; +function setSetting(key: string, value: string) { + db() + .prepare( + 'INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value' + ) + .run(key, value); +} export const watchdogEnabled = () => maintenanceAllowed('health') || maintenanceAllowed('drift'); export function setWatchdogEnabled(enabled: boolean) { setSetting('watchdog_enabled', enabled ? '1' : '0'); @@ -30,18 +61,37 @@ export function setWatchdogEnabled(enabled: boolean) { setMaintenancePolicy('drift', enabled); } -function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; } -function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; } -function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); } +function incident(id: string) { + return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as + WatchdogIncident | undefined; +} +function active(server: string, kind: 'health' | 'drift') { + return db() + .prepare( + "SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'" + ) + .get(server, kind) as WatchdogIncident | undefined; +} +function event(id: string, message: string) { + db() + .prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)') + .run(id, iso(), message); +} function transition(id: string, state: WatchdogState, summary: string) { const previous = incident(id); if (!previous || (previous.state === state && previous.summary === summary)) return; - db().prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?') + db() + .prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?') .run(state, summary, state === 'resolved' || state === 'dismissed' ? iso() : null, id); event(id, summary); } -function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident { - const sampledAt = kind === 'health' ? details.observedAt ?? iso() : iso(); +function detect( + server: string, + kind: 'health' | 'drift', + summary: string, + details: WatchdogEvidence +): WatchdogIncident { + const sampledAt = kind === 'health' ? (details.observedAt ?? iso()) : iso(); const sampledTime = Date.parse(sampledAt); const previous = active(server, kind); if (previous) { @@ -49,40 +99,92 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai const sampleGap = sampledTime - Date.parse(previous.last_seen_at); const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable; const prior = JSON.parse(previous.details) as WatchdogEvidence & { actionable?: boolean }; - const newSample = sampleGap > 0 - && Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS; + const newSample = + sampleGap > 0 && + Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS; const priorActionable = prior.actionable; // A successful routine/manual probe between watchdog samples breaks // the failure streak, even if our next probe happens to fail again. const after = latestGameSubmission(server); - const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt] - .some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at) - && Date.parse(at) <= sampledTime && (!after || Date.parse(at) > Date.parse(after))); - const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true) - || prior.identity !== details.identity || sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS); - db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?') - .run(JSON.stringify({ ...details, countedAt: newSample || reset ? iso() : prior.countedAt ?? previous.last_seen_at }), newSample || reset ? sampledAt : previous.last_seen_at, - reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? sampledAt : previous.first_seen_at, previous.id); + const successful = [ + getServer(server)?.last_seen_at, + telemetryReport(server)?.receivedAt + ].some( + (at) => + at && + Date.parse(at) > Date.parse(previous.last_seen_at) && + Date.parse(at) <= sampledTime && + (!after || Date.parse(at) > Date.parse(after)) + ); + const reset = + kind === 'health' && + (successful || + (actionable === true && priorActionable !== true) || + prior.identity !== details.identity || + sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS); + db() + .prepare( + 'UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?' + ) + .run( + JSON.stringify({ + ...details, + countedAt: + newSample || reset ? iso() : (prior.countedAt ?? previous.last_seen_at) + }), + newSample || reset ? sampledAt : previous.last_seen_at, + reset ? 1 : previous.failures + (newSample ? 1 : 0), + reset ? sampledAt : previous.first_seen_at, + previous.id + ); if (previous.state === 'open') transition(previous.id, 'open', summary); return incident(previous.id)!; } const id = randomUUID(); - db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) - VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify({ ...details, countedAt: iso() }), sampledAt, sampledAt); + db() + .prepare( + `INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) + VALUES (?,?,?,'open',?,?,?,?)` + ) + .run( + id, + server, + kind, + summary, + JSON.stringify({ ...details, countedAt: iso() }), + sampledAt, + sampledAt + ); event(id, summary); return incident(id)!; } /** Call only after verifying container readiness and current game evidence. */ -export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean { +export function resolveRecoveredHealthOperation( + operationId: string, + players: number, + source: 'rcon' | 'telemetry' | null +): boolean { return db().transaction(() => { - const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id + const operation = db() + .prepare( + `SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL AND i.kind = 'health' AND i.state IN ('open','recovering') - AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`) - .get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined; + AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')` + ) + .get(operationId) as + | { + server: string; + action: string; + state: string; + submitted_at: string | null; + triggered_by: string; + incident_id: string; + } + | undefined; if (!operation || getServer(operation.server)?.desired_state !== 'running') return false; const waiting = operation.state === 'waiting' && operation.submitted_at === null; if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false; @@ -90,17 +192,48 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu // exists, and lets that work proceed. Submitted recovery must retain // exclusive ownership before accepting evidence or releasing its slot. if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false; - if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false; - const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart'; + if ( + !waiting && + (operation.action !== 'restart' || + !['awaiting-health', 'failed'].includes(operation.state)) + ) + return false; + const message = waiting + ? 'Server recovered before automatic recovery was submitted' + : 'Fresh container and game health verified after restart'; if (waiting && !cancelProvisioning(operationId, message)) return false; const now = iso(); - db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL - WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId); - db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId); - recordProgress(operationId, { blocker: null, observedAt: now, + db() + .prepare( + `UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL + WHERE server = ? AND operation_id = ? AND healthy_at IS NULL` + ) + .run(now, now, players, operation.server, operationId); + db() + .prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?') + .run(now, players, operationId); + recordProgress(operationId, { + blocker: null, + observedAt: now, recovery: { state: 'released', at: now, operationId, message }, - checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', checkedAt: now }, - { key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] }); + checks: [ + { + key: 'container', + label: 'Container readiness', + source: 'agent', + result: 'pass', + checkedAt: now + }, + { + key: 'game', + label: 'Game status', + source, + result: 'pass', + actual: players, + checkedAt: now + } + ] + }); if (operation.state === 'awaiting-health') { deploymentStage(operationId, message); finishDeployment(operationId, { ok: true }); @@ -110,112 +243,278 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu })(); } function actions(id: string): WatchdogAction[] { - return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error + return db() + .prepare( + `SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id - WHERE a.incident_id=? ORDER BY a.rowid`).all(id) as WatchdogAction[]; + WHERE a.incident_id=? ORDER BY a.rowid` + ) + .all(id) as WatchdogAction[]; } export function watchdogOverview(): WatchdogOverview { - return { enabled: watchdogEnabled(), lastScanAt: setting('watchdog_last_scan'), scanError: setting('watchdog_scan_error') || null, - incidents: db().prepare(`SELECT i.*, (SELECT d.stage FROM watchdog_actions a + return { + enabled: watchdogEnabled(), + lastScanAt: setting('watchdog_last_scan'), + scanError: setting('watchdog_scan_error') || null, + incidents: db() + .prepare( + `SELECT i.*, (SELECT d.stage FROM watchdog_actions a JOIN server_deployments d ON d.id = a.operation_id WHERE a.incident_id = i.id AND a.cycle = i.cycle ORDER BY a.rowid DESC LIMIT 1) AS operation_stage FROM watchdog_incidents i WHERE state != 'resolved' OR id IN ( SELECT id FROM watchdog_incidents WHERE state = 'resolved' ORDER BY first_seen_at DESC, rowid DESC LIMIT 100) - ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC`).all() as WatchdogIncident[] }; + ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC` + ) + .all() as WatchdogIncident[] + }; } export function watchdogDetail(id: string): WatchdogDetail | null { const row = incident(id); - return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null; + return row + ? { + incident: row, + actions: actions(id), + events: db() + .prepare( + 'SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id' + ) + .all(id) as WatchdogDetail['events'] + } + : null; } function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean { // An explicit retry can reclaim its own earlier attempt's recovery slot. // Keep other findings and every active lifecycle operation suppressed. - const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r + const retained = db() + .prepare( + `SELECT r.operation_id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL JOIN watchdog_actions a ON a.operation_id = d.id JOIN watchdog_incidents i ON i.id = a.incident_id WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled') AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle - AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined; + AND (? IS NULL OR i.kind = ?) LIMIT 1` + ) + .get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined; return hasActiveProvisioning(server, retained?.operation_id); } function protectedWork(server: string): boolean { - return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) - || !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual') - AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server); + return ( + !!db() + .prepare( + `SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1` + ) + .get(server) || + !!db() + .prepare( + `SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual') + AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1` + ) + .get(server) + ); } -function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) { - db().prepare('INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)') +function addAction( + row: WatchdogIncident, + operationId: string, + action: 'restart' | 'rebuild', + campaignId: string | null = null +) { + db() + .prepare( + 'INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)' + ) .run(randomUUID(), row.id, action, operationId, campaignId, iso(), row.cycle); - transition(row.id, 'recovering', action === 'restart' ? 'Restart queued; waiting for safe recovery' : 'Rebuild queued with the latest promoted setup'); - event(row.id, `${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}`); + transition( + row.id, + 'recovering', + action === 'restart' + ? 'Restart queued; waiting for safe recovery' + : 'Rebuild queued with the latest promoted setup' + ); + event( + row.id, + `${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}` + ); } export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') { return db().transaction(() => { const row = incident(id); if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' }; - if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' }; - if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' }; + if (row.server_deleted_at) + return { + ok: false as const, + status: 409, + error: 'This server was deleted; its recovery history cannot be changed.' + }; + if (actions(id).some((a) => pendingStates.includes(a.operation_state ?? ''))) + return { + ok: false as const, + status: 409, + error: 'Provisioning is still active. Complete or cancel that operation first.' + }; if (action === 'dismiss') { - if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' }; - transition(id, 'dismissed', 'Dismissed by operator; automatic attempts remain suppressed until the condition clears'); + if (!['open', 'attention'].includes(row.state)) + return { + ok: false as const, + status: 409, + error: 'Only detected incidents or incidents needing attention can be dismissed.' + }; + transition( + id, + 'dismissed', + 'Dismissed by operator; automatic attempts remain suppressed until the condition clears' + ); } else { - if (!['attention','dismissed'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only incidents needing attention or dismissed incidents can be retried.' }; - if (!getServer(row.server)) return { ok: false as const, status: 409, error: 'The server is no longer registered; inspect its provisioning history.' }; - db().prepare('UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?').run(iso(), iso(), id); - transition(id, 'open', 'Operator requested a fresh recovery attempt; checking current conditions'); + if (!['attention', 'dismissed'].includes(row.state)) + return { + ok: false as const, + status: 409, + error: 'Only incidents needing attention or dismissed incidents can be retried.' + }; + if (!getServer(row.server)) + return { + ok: false as const, + status: 409, + error: 'The server is no longer registered; inspect its provisioning history.' + }; + db() + .prepare( + 'UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?' + ) + .run(iso(), iso(), id); + transition( + id, + 'open', + 'Operator requested a fresh recovery attempt; checking current conditions' + ); } return { ok: true as const }; })(); } async function reconcileActions(): Promise { - const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[]; + const rows = db() + .prepare( + "SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')" + ) + .all() as WatchdogIncident[]; for (const row of rows) { - const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1); + const action = actions(row.id) + .filter((a) => a.cycle === row.cycle) + .at(-1); if (!action) continue; // Continue observing our own recovery, but do not verify or escalate // against files and health owned by another lifecycle operation. if (hasActiveProvisioning(row.server, action.operation_id)) continue; - const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as - { state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined; - if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; } - if (['done','warning'].includes(operation.state)) { + const operation = db() + .prepare( + 'SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?' + ) + .get(action.operation_id) as + | { + state: string; + submitted_at: string | null; + started_at: string; + stage: string; + error: string | null; + } + | undefined; + if (!operation) { + transition( + row.id, + 'attention', + 'Provisioning history is missing; inspect the server before retrying' + ); + continue; + } + if (['done', 'warning'].includes(operation.state)) { if (row.kind === 'drift') { try { const plan = await previewDrift([row.server]); - if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue; - if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; } + if ( + incident(row.id)?.cycle !== row.cycle || + hasActiveProvisioning(row.server, action.operation_id) + ) + continue; + if (plan.files.length) { + transition( + row.id, + 'attention', + 'Rebuild completed but drift remains; inspect the campaign and server files' + ); + continue; + } } catch (err) { if (!hasActiveProvisioning(row.server, action.operation_id)) - transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); + transition( + row.id, + 'attention', + `Cannot verify rebuilt files: ${(err as Error).message}` + ); continue; } } - transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild'); + transition( + row.id, + 'resolved', + action.action === 'restart' + ? 'Server recovered after restart' + : 'Server recovered after rebuild' + ); continue; } if (row.state === 'attention') continue; // Exhausted attempts stay latched, including across app restarts. - if (['cancelled','interrupted'].includes(operation.state)) { - transition(row.id, 'attention', `Provisioning ${operation.state}; inspect the recorded operation before retrying`); continue; - } - if (action.action === 'rebuild') { - if (operation.state === 'failed') transition(row.id, 'attention', `Rebuild failed: ${operation.error ?? operation.stage}`); - else if (operation.state === 'awaiting-health' && Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >= WATCHDOG_RECOVERY_MS) - transition(row.id, 'attention', 'Rebuild has not recovered within five minutes; provisioning continues verification'); + if (['cancelled', 'interrupted'].includes(operation.state)) { + transition( + row.id, + 'attention', + `Provisioning ${operation.state}; inspect the recorded operation before retrying` + ); continue; } - if (!['awaiting-health','failed'].includes(operation.state)) continue; - if (Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) < WATCHDOG_RECOVERY_MS) continue; + if (action.action === 'rebuild') { + if (operation.state === 'failed') + transition( + row.id, + 'attention', + `Rebuild failed: ${operation.error ?? operation.stage}` + ); + else if ( + operation.state === 'awaiting-health' && + Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >= + WATCHDOG_RECOVERY_MS + ) + transition( + row.id, + 'attention', + 'Rebuild has not recovered within five minutes; provisioning continues verification' + ); + continue; + } + if (!['awaiting-health', 'failed'].includes(operation.state)) continue; + if ( + Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) < + WATCHDOG_RECOVERY_MS + ) + continue; if (!maintenanceAllowed(row.kind)) continue; const current = getServer(row.server); if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue; // A recovery deadline is a reason to check again, not evidence that // rebuilding is necessary. The worker may not have polled this tick yet. const observation = await observe(row.server, action.operation_id); - if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue; + if ( + !observation || + incident(row.id)?.cycle !== row.cycle || + hasActiveProvisioning(row.server, action.operation_id) + ) + continue; if (observation.healthy) { - resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null); + resolveRecoveredHealthOperation( + action.operation_id, + observation.players!, + observation.source ?? null + ); continue; } if (!observation.actionable) continue; @@ -224,45 +523,110 @@ async function reconcileActions(): Promise { if (next.ok) addAction(row, next.deploymentId, 'rebuild'); return next; })(); - if (!result.ok && result.status !== 409 && result.status !== 503) transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`); + if (!result.ok && result.status !== 409 && result.status !== 503) + transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`); } } -interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; observedAt: string; identity: string; players?: number; source?: 'rcon' | 'telemetry' } -async function observe(server: string, operationId?: string, fresh = false): Promise { +interface Observation { + healthy: boolean; + actionable: boolean; + reason: string; + containerState: string | null; + observedAt: string; + identity: string; + players?: number; + source?: 'rcon' | 'telemetry'; +} +async function observe( + server: string, + operationId?: string, + fresh = false +): Promise { const epoch = latestGameSubmission(server); - const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server)) - || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server); + const suppressed = () => + (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server)) || + protectedWork(server) || + getServer(server)?.desired_state !== 'running' || + epoch !== latestGameSubmission(server); // Routine watchdog passes only consume the status monitor. Manual checks and // recovery verification still obtain fresh, independently verified evidence. const snapshot = !fresh && !operationId ? monitoringStatus(server) : null; if (!fresh && !operationId && !snapshot) return null; - const inspected = snapshot?.container ?? await inspectForMonitoring(server, true).catch(error => { - if (error instanceof MonitoringChangedError) return null; - throw error; - }); + const inspected = + snapshot?.container ?? + (await inspectForMonitoring(server, true).catch((error) => { + if (error instanceof MonitoringChangedError) return null; + throw error; + })); if (!inspected) return null; const health = inspected.result; if (suppressed()) return null; const identity = snapshot?.identity ?? monitoringIdentity(server); const containerEvidence = { identity, observedAt: inspected.observedAt }; const state = health.state?.toLowerCase() ?? null; - if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['missing','stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state }; + if (!health.ok) + return { + ...containerEvidence, + healthy: false, + actionable: + !!state && + [ + 'missing', + 'stopped', + 'exited', + 'created', + 'dead', + 'unhealthy', + 'restarting', + 'starting' + ].includes(state), + reason: health.reason, + containerState: state + }; const telemetry = telemetryEvidence(server); - if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' }; - const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => { - if (error instanceof MonitoringChangedError) return null; - throw error; - }); + if (telemetry) + return { + identity, + observedAt: telemetry.receivedAt, + healthy: true, + actionable: false, + reason: 'Container healthy with fresh telemetry', + containerState: state, + players: telemetry.players, + source: 'telemetry' + }; + const probed = snapshot + ? snapshot.game + : await rconForMonitoring(server, true).catch((error) => { + if (error instanceof MonitoringChangedError) return null; + throw error; + }); if (!probed) return null; const rcon = probed.result; if (suppressed()) return null; const latest = telemetryEvidence(server); - const gameEvidence = { identity: snapshot?.identity ?? monitoringIdentity(server), observedAt: latest?.receivedAt ?? probed.observedAt }; - if (rcon.ok || latest) return { ...gameEvidence, healthy: true, actionable: false, reason: 'Container healthy with fresh game status', containerState: state, - players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' }; - return { ...gameEvidence, healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error', - reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state }; + const gameEvidence = { + identity: snapshot?.identity ?? monitoringIdentity(server), + observedAt: latest?.receivedAt ?? probed.observedAt + }; + if (rcon.ok || latest) + return { + ...gameEvidence, + healthy: true, + actionable: false, + reason: 'Container healthy with fresh game status', + containerState: state, + players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), + source: latest ? 'telemetry' : 'rcon' + }; + return { + ...gameEvidence, + healthy: false, + actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error', + reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, + containerState: state + }; } async function scanServer(server: string, forceScan: boolean): Promise { const row = getServer(server); @@ -270,16 +634,31 @@ async function scanServer(server: string, forceScan: boolean): Promise { if (scanSuppressed(server) || protectedWork(server)) return; const observation = await observe(server, undefined, forceScan); // Probes can overlap an API request; recheck ownership before queuing anything. - if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; + if ( + !observation || + scanSuppressed(server) || + protectedWork(server) || + getServer(server)?.desired_state === 'stopped' + ) + return; if (observation.identity !== monitoringIdentity(server)) return; if (!scanSuppressed(server, 'health') && observation.healthy) { const current = active(server, 'health'); - if (current) transition(current.id, 'resolved', 'Fresh game status and host agent health verified'); + if (current) + transition(current.id, 'resolved', 'Fresh game status and host agent health verified'); } else if (!scanSuppressed(server, 'health')) { const problem = detect(server, 'health', observation.reason, observation); - const startup = ['starting','restarting'].includes(observation.containerState ?? ''); - const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS); - if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) { + const startup = ['starting', 'restarting'].includes(observation.containerState ?? ''); + const oldEnough = + Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= + (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS); + if ( + problem.state === 'open' && + observation.actionable && + oldEnough && + problem.failures >= 3 && + maintenanceAllowed('health') + ) { // A missing container has nothing to restart. Recreate it through // the normal rebuild queue, with its identity and safety checks. const action = observation.containerState === 'missing' ? 'rebuild' : 'restart'; @@ -288,49 +667,85 @@ async function scanServer(server: string, forceScan: boolean): Promise { if (next.ok) addAction(problem, next.deploymentId, action); return next; })(); - if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error); + if (!result.ok && result.status !== 409 && result.status !== 503) + transition(problem.id, 'attention', result.error); } } // Missing-container recovery belongs to the health policy. Do not turn // absent files into drift or bypass the sustained health-failure window. // Unknown container health is an observation failure, not proof that a rebuild is safe. - if (observation.containerState === 'missing' || (!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return; + if ( + observation.containerState === 'missing' || + (!observation.healthy && !observation.actionable) || + scanSuppressed(server, 'drift') + ) + return; // File trees change far less often than liveness. Persist the last successful // scan per registration/process so health checks do not rescan every minute. // Manual checks and new lifecycle submissions always get fresh file evidence. const driftKey = `watchdog_drift_scan:${server}`; const instance = nativeInstance(server); - const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]); + const identity = JSON.stringify([ + row.created_at, + instance?.id, + instance?.generation, + latestGameSubmission(server) + ]); let previous: { identity?: string; at?: string } = {}; - try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ } + try { + previous = JSON.parse(setting(driftKey) ?? '{}'); + } catch { + /* rescan legacy/invalid state */ + } const age = Date.now() - Date.parse(previous?.at ?? ''); - if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return; + if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) + return; const prepared = await prepareDriftCampaign(server); - if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; + if ( + scanSuppressed(server, 'drift') || + protectedWork(server) || + getServer(server)?.desired_state === 'stopped' + ) + return; setSetting(driftKey, JSON.stringify({ identity, at: iso() })); if (!prepared.plan.files.length) { const current = active(server, 'drift'); if (current) transition(current.id, 'resolved', 'Server files match the promoted setup'); return; } - const problem = detect(server, 'drift', `${prepared.plan.files.length} file difference(s) from the promoted setup`, { files: prepared.plan.files, fingerprint: prepared.plan.fingerprint }); + const problem = detect( + server, + 'drift', + `${prepared.plan.files.length} file difference(s) from the promoted setup`, + { files: prepared.plan.files, fingerprint: prepared.plan.fingerprint } + ); if (problem.state !== 'open' || !maintenanceAllowed('drift')) return; const result = db().transaction(() => { const next = requestWatchdogLifecycle(server, 'rebuild', problem.id); if (next.ok) { - const campaignId = createDriftRebuildCampaign(server, next.deploymentId, problem.id, prepared); + const campaignId = createDriftRebuildCampaign( + server, + next.deploymentId, + problem.id, + prepared + ); addAction(problem, next.deploymentId, 'rebuild', campaignId); } return next; })(); - if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`); + if (!result.ok && result.status !== 409 && result.status !== 503) + transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`); } let inFlight: Promise | null = null; export function watchdogTick(forceScan = false): Promise { if (inFlight) return inFlight; - inFlight = withOperationSource(forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', () => run(forceScan)) - .finally(() => { inFlight = null; }); + inFlight = withOperationSource( + forceScan ? 'Manual maintenance check' : 'Maintenance watchdog', + () => run(forceScan) + ).finally(() => { + inFlight = null; + }); return inFlight; } async function run(forceScan: boolean): Promise { @@ -338,8 +753,12 @@ async function run(forceScan: boolean): Promise { const lastScan = setting('watchdog_last_scan'); if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return; setSetting('watchdog_last_scan', iso()); - if (!listServers().some(server => nativeInstance(server.name))) { - setSetting('watchdog_scan_error', 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'); return; + if (!listServers().some((server) => nativeInstance(server.name))) { + setSetting( + 'watchdog_scan_error', + 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.' + ); + return; } const errors: string[] = []; const servers = listServers(); @@ -347,21 +766,34 @@ async function run(forceScan: boolean): Promise { async function pump() { while (cursor < servers.length) { const server = servers[cursor++]; - try { await scanServer(server.name, forceScan); } - catch (err) { - if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`); + try { + await scanServer(server.name, forceScan); + } catch (err) { + if (!hasActiveProvisioning(server.name)) + errors.push(`${server.name}: ${(err as Error).message}`); } } } await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump)); // Preserve incidents for temporarily absent rebuild instances, but close retired registrations. - const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[]; + const abandoned = db() + .prepare( + "SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'" + ) + .all() as WatchdogIncident[]; for (const row of abandoned) { if (getServer(row.server) || hasActiveProvisioning(row.server)) continue; - const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined; + const last = db() + .prepare( + 'SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1' + ) + .get(row.server) as { action: string; state: string } | undefined; // An absent registry can also mean a failed rebuild. Only a completed // destruction retires that failure without requiring inspection. - if (['open','dismissed'].includes(row.state) || (last?.action === 'destroy' && ['done','warning'].includes(last.state))) + if ( + ['open', 'dismissed'].includes(row.state) || + (last?.action === 'destroy' && ['done', 'warning'].includes(last.state)) + ) transition(row.id, 'resolved', 'Server is no longer registered'); } setSetting('watchdog_scan_error', errors.join('\n')); diff --git a/src/lib/server/maintenance/runtime.ts b/src/lib/server/maintenance/runtime.ts index 4cc7959..d8b6d73 100644 --- a/src/lib/server/maintenance/runtime.ts +++ b/src/lib/server/maintenance/runtime.ts @@ -6,42 +6,77 @@ import { nextMaintenanceAt } from '../../plutonium.js'; import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js'; import { readSetting, writeSetting } from './settings.js'; -import { checkPlutonium, probeVersion, release, freshRelease, runningVersions, HOUR } from '../integrations/plutonium.js'; +import { + checkPlutonium, + probeVersion, + release, + freshRelease, + runningVersions, + HOUR +} from '../integrations/plutonium.js'; import { maintenanceAllowed } from './policies.js'; export { checkPlutonium } from '../integrations/plutonium.js'; const nowIso = () => new Date().toISOString(); function schedule(): MaintenanceSchedule { // Legacy schedules were explicitly UTC. Preserve their timing until saved. - return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) }; + return { + timeZone: 'UTC', + ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) + }; } export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void { const nextAt = nextMaintenanceAt(time, new Date(), timeZone); - writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null }); + writeSetting('plutonium_schedule', { + enabled, + time, + timeZone, + nextAt: enabled ? nextAt : null + }); } export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus { return { - release: release(), schedule: schedule(), servers: runningVersions(), - jobs: db().prepare(`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state + release: release(), + schedule: schedule(), + servers: runningVersions(), + jobs: db() + .prepare( + `SELECT m.*, d.stage AS operation_stage, d.state AS operation_state FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id WHERE m.state IN ('queued','restarting') OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)) - OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`).all(selectedJob) as MaintenanceJob[] + OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC` + ) + .all(selectedJob) as MaintenanceJob[] }; } -export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] = 'legacy', origin: MaintenanceJob['origin'] = 'manual'): number { - const names = new Set(listServers().map(s => s.name)); - if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.'); +export function queueMaintenance( + servers: string[], + kind: MaintenanceJob['kind'] = 'legacy', + origin: MaintenanceJob['origin'] = 'manual' +): number { + const names = new Set(listServers().map((s) => s.name)); + if (!servers.length || servers.some((s) => !names.has(s))) + throw new Error('Choose existing fleet servers.'); return db().transaction(() => { const target = kind === 'schedule' ? null : freshRelease(); - if (kind === 'update' && target === null) throw new Error('Check the published revision before requesting an update.'); + if (kind === 'update' && target === null) + throw new Error('Check the published revision before requesting an update.'); let added = 0; for (const server of new Set(servers)) { if (getServer(server)?.desired_state === 'stopped') continue; if (origin !== 'manual' && hasActiveProvisioning(server)) continue; - added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin) - VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes; - const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number }; + added += db() + .prepare( + `INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin) + VALUES (?, ?, ?, ?, ?)` + ) + .run(server, target, nowIso(), kind, origin).changes; + const job = db() + .prepare( + "SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')" + ) + .get(server) as { id: number }; requestRestart(server, { kind: 'maintenance', id: String(job.id) }); } return added; @@ -50,52 +85,98 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] /** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */ export function queueUpdates(servers?: string[], automatic = false): number { - if (!maintenanceAllowed()) throw new Error('Maintenance is paused. Resume it before requesting updates.'); + if (!maintenanceAllowed()) + throw new Error('Maintenance is paused. Resume it before requesting updates.'); if (automatic && !maintenanceAllowed('updates')) return 0; - if (freshRelease() === null) throw new Error('Check the published revision before requesting an update.'); + if (freshRelease() === null) + throw new Error('Check the published revision before requesting an update.'); const versions = runningVersions(); - if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.'); - const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server))); - const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs - WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server); + if ( + servers && + (!servers.length || servers.some((server) => !versions.some((v) => v.server === server))) + ) + throw new Error('Choose existing fleet servers.'); + const candidates = versions.filter( + (v) => v.update_status === 'outdated' && (!servers || servers.includes(v.server)) + ); + const names = candidates + .filter( + (v) => + !automatic || + !db() + .prepare( + `SELECT 1 FROM maintenance_jobs + WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'` + ) + .get(v.server, freshRelease()) + ) + .map((v) => v.server); return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0; } export function retryMaintenance(id: number): number { - const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined; + const job = db() + .prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'") + .get(id) as MaintenanceJob | undefined; if (!job) throw new Error('Only failed maintenance can be retried.'); - if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.'); + if (job.server_deleted_at) + throw new Error('This server was deleted; its maintenance history cannot be retried.'); if (!maintenanceAllowed()) throw new Error('Maintenance is paused.'); - if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.'); + if (getServer(job.server)?.desired_state === 'stopped') + throw new Error('This server is intentionally stopped.'); // Ambiguous lifecycle failures retain their slot and require diagnostics, not another request. if (job.provisioning_id) { - const operation = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(job.provisioning_id) as { state: string } | undefined; - if (operation?.state !== 'done') throw new Error('Inspect and recover the existing Provisioning operation before retrying.'); + const operation = db() + .prepare('SELECT state FROM server_deployments WHERE id = ?') + .get(job.provisioning_id) as { state: string } | undefined; + if (operation?.state !== 'done') + throw new Error( + 'Inspect and recover the existing Provisioning operation before retrying.' + ); } return queueMaintenance([job.server], job.kind, 'manual'); } export function cancelMaintenance(id: number): boolean { return db().transaction(() => { - const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined; + const job = db() + .prepare( + "SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'" + ) + .get(id) as { provisioning_id: string | null } | undefined; if (!job) return false; if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false; - return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator' - WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0; + return ( + db() + .prepare( + `UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator' + WHERE id = ? AND state = 'queued'` + ) + .run(nowIso(), id).changes > 0 + ); })(); } function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) { - db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL') + db() + .prepare( + 'UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL' + ) .run(state, detail, nowIso(), job.id); } function hold(job: MaintenanceJob, detail: string) { - db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id); + db() + .prepare( + 'UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL' + ) + .run(detail, job.id); } let ticking: Promise | null = null; export function maintenanceTick(): Promise { if (ticking) return ticking; - ticking = runTick().finally(() => { ticking = null; }); + ticking = runTick().finally(() => { + ticking = null; + }); return ticking; } async function runTick(): Promise { @@ -103,51 +184,94 @@ async function runTick(): Promise { // Provisioning refreshes the manifest independently; that must not postpone // the hourly version checks for the rest of the fleet. const versionsCheckedAt = readSetting('plutonium_versions_checked_at', null); - if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR - || !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR) await checkPlutonium(); + if ( + !latest.checkedAt || + Date.now() - Date.parse(latest.checkedAt) >= HOUR || + !versionsCheckedAt || + Date.now() - Date.parse(versionsCheckedAt) >= HOUR + ) + await checkPlutonium(); if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true); db().transaction(() => { const plan = schedule(); if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) { - const names = listServers().filter(s => s.desired_state !== 'stopped').map(s => s.name); + const names = listServers() + .filter((s) => s.desired_state !== 'stopped') + .map((s) => s.name); if (names.length) queueMaintenance(names, 'schedule', 'schedule'); // Coalesce missed days after downtime into one run; no restart backlog. - writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) }); + writeSetting('plutonium_schedule', { + ...plan, + nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) + }); } })(); - const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[]; + const jobs = db() + .prepare( + `SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id` + ) + .all() as MaintenanceJob[]; for (const job of jobs) { - try { await processJob(job); } - catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } + try { + await processJob(job); + } catch { + hold(job, 'Maintenance check failed; waiting for the next check.'); + } } - } async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); - if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); + if (job.server_deleted_at || !server || server.destroyed_at) + return finish(job, 'cancelled', 'Server removed.'); if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) { return hold(job, 'Waiting for another provisioning operation.'); } const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) }); - if (['failed','interrupted','cancelled'].includes(operation.state)) { - return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage); + if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) { + return finish( + job, + operation.state === 'cancelled' ? 'cancelled' : 'failed', + operation.error ?? operation.stage + ); } if (operation.state !== 'done') { if (operation.submitted_at) { - db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id); - db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server); + db() + .prepare( + "UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?" + ) + .run(operation.submitted_at, job.id); + db() + .prepare( + 'UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?' + ) + .run('Waiting for restart recovery.', job.server); } return hold(job, operation.stage); } if (hasActiveProvisioning(job.server, operation.id)) { - return hold(job, 'Waiting for another provisioning operation before verifying the running revision.'); + return hold( + job, + 'Waiting for another provisioning operation before verifying the running revision.' + ); } const revision = await probeVersion(job.server, { maintenance: true }); if (hasActiveProvisioning(job.server, operation.id)) { - return hold(job, 'Waiting for another provisioning operation before verifying the running revision.'); + return hold( + job, + 'Waiting for another provisioning operation before verifying the running revision.' + ); } if (job.target_revision && (revision === null || revision < job.target_revision)) { - return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`); + return finish( + job, + 'failed', + `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.` + ); } - return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`); + return finish( + job, + 'done', + `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.` + ); } diff --git a/src/lib/server/maintenance/settings.ts b/src/lib/server/maintenance/settings.ts index 13a107d..63150d3 100644 --- a/src/lib/server/maintenance/settings.ts +++ b/src/lib/server/maintenance/settings.ts @@ -1,9 +1,13 @@ import { db } from '../db/index.js'; export function readSetting(key: string, fallback: T): T { - const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined; + const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as + { value: string } | undefined; return row ? JSON.parse(row.value) : fallback; } export function writeSetting(key: string, value: unknown): void { - db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value') + db() + .prepare( + 'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value' + ) .run(key, JSON.stringify(value)); } diff --git a/src/lib/server/performance.ts b/src/lib/server/performance.ts index 85f64c9..7798019 100644 --- a/src/lib/server/performance.ts +++ b/src/lib/server/performance.ts @@ -4,10 +4,14 @@ let started = false; export function startPerformanceMonitor() { if (started || process.env.EC_PERF_LOG !== 'true') return; started = true; - const delay = monitorEventLoopDelay({ resolution: 20 }); delay.enable(); + const delay = monitorEventLoopDelay({ resolution: 20 }); + delay.enable(); setInterval(() => { - console.info('[performance]', { eventLoopP95Ms: Math.round(delay.percentile(95) / 1e6), - eventLoopMaxMs: Math.round(delay.max / 1e6), rssMiB: Math.round(process.memoryUsage().rss / 1048576) }); + console.info('[performance]', { + eventLoopP95Ms: Math.round(delay.percentile(95) / 1e6), + eventLoopMaxMs: Math.round(delay.max / 1e6), + rssMiB: Math.round(process.memoryUsage().rss / 1048576) + }); delay.reset(); }, 60_000).unref(); } diff --git a/src/lib/server/pipeline/activity.ts b/src/lib/server/pipeline/activity.ts index 80fec85..638bcbf 100644 --- a/src/lib/server/pipeline/activity.ts +++ b/src/lib/server/pipeline/activity.ts @@ -6,12 +6,19 @@ let sequence = 0; /** Live diagnostics only. Durable operation milestones belong in provisioning history. */ export function workerActivity(): WorkerActivity[] { - return [...active.values()].map(value => ({ ...value })); + return [...active.values()].map((value) => ({ ...value })); } -export async function trackActivity(key: string, label: string, work: () => Promise): Promise { +export async function trackActivity( + key: string, + label: string, + work: () => Promise +): Promise { if (active.has(key)) key = `${key}:${++sequence}`; active.set(key, { key, label, startedAt: new Date().toISOString() }); - try { return await withOperationSource(currentOperationSource() ?? label, work); } - finally { active.delete(key); } + try { + return await withOperationSource(currentOperationSource() ?? label, work); + } finally { + active.delete(key); + } } diff --git a/src/lib/server/pipeline/beta-plan-runner.ts b/src/lib/server/pipeline/beta-plan-runner.ts index 311537c..f71310d 100644 --- a/src/lib/server/pipeline/beta-plan-runner.ts +++ b/src/lib/server/pipeline/beta-plan-runner.ts @@ -1,35 +1,67 @@ import { db } from '../db/index.js'; import { currentBetaRevision } from '../fleet/beta-revisions.js'; -import { betaOperationInFlight, deployBetaServers, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js'; -import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js'; +import { + betaOperationInFlight, + deployBetaServers, + isActiveBetaServer, + listBetaServers, + rollbackBeta +} from '../fleet/beta.js'; +import { + activity, + currentRun, + observeRun, + prepareRun, + readMetrics, + saveMetrics, + setOutcome, + storedPlan, + type StoredPlan +} from '../fleet/beta-tests.js'; import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { emit, type BetaStage } from './bus.js'; import { requestRestart, releaseRestartRequest } from '../provisioning/index.js'; function stage(id: number, server: string, next: BetaStage, error: string | null = null): void { - const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL') + const result = db() + .prepare( + 'UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL' + ) .run(next, error, new Date().toISOString(), id, server); if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error }); } export function betaState(id: number, ready: boolean): void { const state = ready ? 'ready-promote' : 'running'; - const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state); + const result = db() + .prepare( + "UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?" + ) + .run(state, id, state); if (result.changes) emit({ type: 'beta-state', betaId: id, state }); // A revoked pass must not leave this version labelled tested unless another beta still proves it. - db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?) + db() + .prepare( + `UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?) AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL) AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`) + WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))` + ) .run(ready ? 'tested' : 'untested', id, id); } export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise { const result = await rollbackBeta(id, { restart: plan.rollbackRestart }); - if (!result.deploymentId) activity(id, 'Cancelled queued changes; no deployed files need restoration'); - else if (plan.rollbackRestart === 'when-empty') activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers'); - else activity(id, 'File restoration is held; resume the rollback campaign to apply it with a safe restart'); + if (!result.deploymentId) + activity(id, 'Cancelled queued changes; no deployed files need restoration'); + else if (plan.rollbackRestart === 'when-empty') + activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers'); + else + activity( + id, + 'File restoration is held; resume the rollback campaign to apply it with a safe restart' + ); if (result.warnings.length) activity(id, result.warnings.join('; ')); } @@ -37,27 +69,41 @@ async function fail(id: number, plan: BetaPlan, reason: string): Promise { betaState(id, false); setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason); if (plan.failureAction === 'rollback') { - try { await rollbackTestPlan(id, plan); } - catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); } + try { + await rollbackTestPlan(id, plan); + } catch (error) { + setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); + } } } let inFlight: Promise | null = null; export function plannedBetaTick(): Promise { if (inFlight) return inFlight; - inFlight = tick().finally(() => { inFlight = null; }); + inFlight = tick().finally(() => { + inFlight = null; + }); return inFlight; } async function tick(): Promise { - const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id - WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[]; + const plans = db() + .prepare( + `SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id + WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id` + ) + .all() as StoredPlan[]; for (const stored of plans) { if (betaOperationInFlight(stored.beta_id)) continue; - try { await advancePlan(stored); } - catch (error) { + try { + await advancePlan(stored); + } catch (error) { const reason = `Test runner: ${(error as Error).message}`; - try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); } - catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); } + try { + await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); + } catch { + betaState(stored.beta_id, false); + setOutcome(stored.beta_id, 'failed', reason); + } } } } @@ -68,13 +114,20 @@ async function advancePlan(stored: StoredPlan): Promise { const servers = listBetaServers(id); // Preserve the original coverage and evidence. A replacement with the same // name cannot complete a test that began on a different server. - const missing = servers.filter(server => !isActiveBetaServer(id, server.server)); + const missing = servers.filter((server) => !isActiveBetaServer(id, server.server)); if (missing.length) { betaState(id, false); - setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`); + setOutcome( + id, + 'paused', + `Test server deleted: ${missing.map((server) => server.server).join(', ')}. Roll back this beta and start a new test.` + ); return; } - if (stored.outcome !== 'passed' && Date.now() - Date.parse(currentBetaRevision(id).started_at) >= plan.timeoutHours * 3_600_000) { + if ( + stored.outcome !== 'passed' && + Date.now() - Date.parse(currentBetaRevision(id).started_at) >= plan.timeoutHours * 3_600_000 + ) { betaState(id, false); setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence'); return; @@ -83,19 +136,38 @@ async function advancePlan(stored: StoredPlan): Promise { if (!isActiveBetaServer(id, server.server)) return; const freshPlan = storedPlan(id)!; if (!['active', 'passed'].includes(freshPlan.outcome)) return; - if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; } + if (server.stage === 'failed') { + await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); + return; + } if (server.stage === 'deploying' || server.stage === 'restarting') { - stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`); - await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return; + stage( + id, + server.server, + 'failed', + `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt` + ); + await fail(id, plan, `${server.server}: interrupted ${server.stage}`); + return; } if (server.stage === 'queued') continue; if (server.stage === 'awaiting-empty') { const waitingRun = currentRun(id, server.server); if (waitingRun?.observing_at) { await observeRun(waitingRun, plan); - if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } - const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed); - if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } + if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { + betaState(id, false); + return; + } + const broken = evaluateRequirements( + plan, + readMetrics(currentRun(id, server.server)!), + waitingRun.phase + ).find((p) => p.failed); + if (broken) { + await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); + return; + } } // Prepare evidence once before handing off the restart. Empty-server // checks and remote execution belong entirely to Provisioning. @@ -113,61 +185,116 @@ async function advancePlan(stored: StoredPlan): Promise { if (!isActiveBetaServer(id, server.server)) return; if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') { const operation = requestRestart(server.server, { kind: 'beta', id: String(id) }); - if (['failed','interrupted','cancelled'].includes(operation.state)) { + if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) { const reason = operation.error ?? operation.stage; - stage(id, server.server, 'failed', reason); await fail(id, plan, reason); return; + stage(id, server.server, 'failed', reason); + await fail(id, plan, reason); + return; } if (!operation.submitted_at || operation.state === 'running') { stage(id, server.server, 'awaiting-empty', operation.stage); continue; } - db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?') + db() + .prepare( + 'UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?' + ) .run(operation.submitted_at, id, server.server); - if (operation.state !== 'done') { stage(id, server.server, 'awaiting-health', operation.stage); continue; } + if (operation.state !== 'done') { + stage(id, server.server, 'awaiting-health', operation.stage); + continue; + } const now = operation.finished_at ?? new Date().toISOString(); - db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server); - db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server); + db() + .prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?') + .run(now, id, server.server); + db() + .prepare( + 'UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL' + ) + .run(now, id, server.server); stage(id, server.server, 'observing'); } const run = currentRun(id, server.server); if (!run?.observing_at) continue; await observeRun(run, plan); if (!isActiveBetaServer(id, server.server)) return; - if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } + if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { + betaState(id, false); + return; + } const metrics = readMetrics(currentRun(id, server.server)!); const progress = evaluateRequirements(plan, metrics, run.phase); - const broken = progress.find(p => p.failed); - if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } - const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target)); - if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) { + const broken = progress.find((p) => p.failed); + if (broken) { + stage(id, server.server, 'failed', broken.detail); + await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); + return; + } + const recoveryTarget = Math.max( + 0, + ...plan.requirements.filter((r) => r.kind === 'recovery').map((r) => r.target) + ); + if ( + run.phase < recoveryTarget && + progress + .filter( + (p) => !plan.requirements.some((r) => r.id === p.id && r.kind === 'recovery') + ) + .every((p) => p.passed) + ) { releaseRestartRequest(server.server, { kind: 'beta', id: String(id) }); - db().prepare('UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?').run(id, server.server); + db() + .prepare( + 'UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?' + ) + .run(id, server.server); stage(id, server.server, 'awaiting-empty'); - activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`); - } else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing'); + activity( + id, + `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated` + ); + } else stage(id, server.server, progress.every((p) => p.passed) ? 'ready' : 'observing'); } const updated = listBetaServers(id); - if (updated.some(server => !isActiveBetaServer(id, server.server))) return; - const queued = updated.filter(s => s.stage === 'queued'); - if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) { - const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server); + if (updated.some((server) => !isActiveBetaServer(id, server.server))) return; + const queued = updated.filter((s) => s.stage === 'queued'); + if ( + queued.length && + updated.filter((s) => s.stage !== 'queued').every((s) => s.stage === 'ready') + ) { + const wave = queued.slice(0, plan.waveSize || queued.length).map((s) => s.server); activity(id, `Previous wave passed; deploying ${wave.join(', ')}`); await deployBetaServers(id, wave); betaState(id, false); return; } - const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>; + const registry = db() + .prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?') + .all(id) as Array<{ name: string; game: string | null; map: string | null }>; // All waves must be activated, even when the final coverage rule is a quorum. // A passing server proves only its assigned files. A quorum or coverage // group must not pass an unrelated game/map's untested bundle member. - const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at) - && coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' }))) - && !db().prepare(`SELECT 1 FROM beta_scripts files WHERE files.beta_id = ? AND NOT EXISTS ( + const ready = + !queued.length && + updated.every((s) => !!currentRun(id, s.server)?.observing_at) && + coveragePassed( + plan, + updated.map((s) => ({ + ...registry.find((r) => r.name === s.server)!, + passed: s.stage === 'ready' + })) + ) && + !db() + .prepare( + `SELECT 1 FROM beta_scripts files WHERE files.beta_id = ? AND NOT EXISTS ( SELECT 1 FROM beta_server_scripts members JOIN beta_servers servers ON servers.beta_id = members.beta_id AND servers.server = members.server WHERE members.beta_id = files.beta_id AND members.script_id = files.script_id AND servers.server_deleted_at IS NULL AND servers.stage = 'ready' - ) LIMIT 1`).get(id); + ) LIMIT 1` + ) + .get(id); betaState(id, ready); setOutcome(id, ready ? 'passed' : 'active', null); } diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index accac78..abc58d2 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -72,7 +72,9 @@ function reconcileBetaState(betaId: number): void { const versionRows = db() .prepare(`SELECT script_version_id FROM beta_scripts WHERE beta_id = ?`) .all(betaId) as Array<{ script_version_id: number }>; - const markTested = db().prepare(`UPDATE script_versions SET state = 'tested' WHERE id = ?`); + const markTested = db().prepare( + `UPDATE script_versions SET state = 'tested' WHERE id = ?` + ); for (const v of versionRows) markTested.run(v.script_version_id); } emit({ type: 'beta-state', betaId, state: 'ready-promote' }); @@ -83,7 +85,9 @@ let betaInFlight: Promise<{ processed: number }> | null = null; export function betaTick(): Promise<{ processed: number }> { if (betaInFlight) return betaInFlight; - betaInFlight = runBetaTick().finally(() => { betaInFlight = null; }); + betaInFlight = runBetaTick().finally(() => { + betaInFlight = null; + }); return betaInFlight; } @@ -127,7 +131,7 @@ async function advance(row: BetaServerActiveRow): Promise { case 'restarting': case 'awaiting-health': { const operation = requestRestart(row.server, { kind: 'beta', id: String(row.beta_id) }); - if (['failed','interrupted','cancelled'].includes(operation.state)) { + if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) { transition(row.beta_id, row.server, 'failed', operation.error ?? operation.stage); return; } @@ -135,7 +139,10 @@ async function advance(row: BetaServerActiveRow): Promise { transition(row.beta_id, row.server, 'awaiting-empty', operation.stage); return; } - db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?') + db() + .prepare( + 'UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?' + ) .run(operation.submitted_at, row.beta_id, row.server); if (row.wait_for_health && operation.state !== 'done') { transition(row.beta_id, row.server, 'awaiting-health', operation.stage); diff --git a/src/lib/server/pipeline/bus.ts b/src/lib/server/pipeline/bus.ts index d6587f2..c4a8615 100644 --- a/src/lib/server/pipeline/bus.ts +++ b/src/lib/server/pipeline/bus.ts @@ -85,22 +85,32 @@ const changedBetas = new Set(); export function invalidateBeta(betaId: number): void { const scheduled = changedBetas.size > 0; changedBetas.add(betaId); - if (!scheduled) queueMicrotask(() => { - const ids = [...changedBetas]; changedBetas.clear(); - for (const betaId of ids) bus.emit('event', { type: 'beta-changed', betaId }); - }); + if (!scheduled) + queueMicrotask(() => { + const ids = [...changedBetas]; + changedBetas.clear(); + for (const betaId of ids) bus.emit('event', { type: 'beta-changed', betaId }); + }); } let invalidationPending = false; export function invalidateFleet(): void { if (invalidationPending) return; invalidationPending = true; // Flush after nested database transactions have committed. - queueMicrotask(() => { invalidationPending = false; bus.emit('event', { type: 'fleet-invalidated' }); }); + queueMicrotask(() => { + invalidationPending = false; + bus.emit('event', { type: 'fleet-invalidated' }); + }); } export function emit(event: PipelineEvent): void { bus.emit('event', event); - if (event.type === 'beta-opened' || event.type === 'beta-state' || event.type === 'beta-server-stage') invalidateFleet(); + if ( + event.type === 'beta-opened' || + event.type === 'beta-state' || + event.type === 'beta-server-stage' + ) + invalidateFleet(); if ('betaId' in event && event.type !== 'beta-changed') invalidateBeta(event.betaId); } diff --git a/src/lib/server/pipeline/event-stream.ts b/src/lib/server/pipeline/event-stream.ts index 33c11d6..1022875 100644 --- a/src/lib/server/pipeline/event-stream.ts +++ b/src/lib/server/pipeline/event-stream.ts @@ -13,7 +13,11 @@ interface EventStreamOptions { initialFrames?: Uint8Array[]; } -export function eventStreamResponse({ accepts, encode, initialFrames = [] }: EventStreamOptions): Response { +export function eventStreamResponse({ + accepts, + encode, + initialFrames = [] +}: EventStreamOptions): Response { let heartbeat: ReturnType | undefined; let unsubscribe: (() => void) | undefined; let closed = false; @@ -25,30 +29,37 @@ export function eventStreamResponse({ accepts, encode, initialFrames = [] }: Eve unsubscribe = undefined; }; - const stream = new ReadableStream({ - start(controller) { - const send = (frame: () => Uint8Array) => { + const stream = new ReadableStream( + { + start(controller) { + const send = (frame: () => Uint8Array) => { + if (closed) return; + if ((controller.desiredSize ?? 0) <= 0) { + // Reconnect for fresh state instead of silently losing deltas. + // close() does not invoke cancel(), so release resources here. + cleanup(); + controller.close(); + return; + } + try { + controller.enqueue(frame()); + } catch (cause) { + cleanup(); + controller.error(cause); + } + }; + send(() => hello); + for (const frame of initialFrames) send(() => frame); if (closed) return; - if ((controller.desiredSize ?? 0) <= 0) { - // Reconnect for fresh state instead of silently losing deltas. - // close() does not invoke cancel(), so release resources here. - cleanup(); - controller.close(); - return; - } - try { controller.enqueue(frame()); } - catch (cause) { cleanup(); controller.error(cause); } - }; - send(() => hello); - for (const frame of initialFrames) send(() => frame); - if (closed) return; - unsubscribe = subscribe(event => { - if (accepts(event)) send(() => encode(event)); - }); - heartbeat = setInterval(() => send(() => ping), HEARTBEAT_MS); + unsubscribe = subscribe((event) => { + if (accepts(event)) send(() => encode(event)); + }); + heartbeat = setInterval(() => send(() => ping), HEARTBEAT_MS); + }, + cancel: cleanup }, - cancel: cleanup - }, { highWaterMark: MAX_QUEUED_FRAMES }); + { highWaterMark: MAX_QUEUED_FRAMES } + ); return new Response(stream, { headers: { diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index 645145a..da107b0 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -3,47 +3,97 @@ import { emit } from './bus.js'; import { requestRestart } from '../provisioning/index.js'; interface PendingRow { - deployment_id: string; server: string; restart_state: string; - restarted_at: string | null; files_changed: number; - players: number | null; error: string | null; + deployment_id: string; + server: string; + restart_state: string; + restarted_at: string | null; + files_changed: number; + players: number | null; + error: string | null; } -type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number }; +type SweepResult = { + checked: number; + restarted: number; + healthy: number; + held: number; + failed: number; +}; let sweepInFlight: Promise | null = null; const nowIso = () => new Date().toISOString(); /** Submit activation requests and mirror their results. Provisioning owns all execution. */ export function sweep(): Promise { if (sweepInFlight) return sweepInFlight; - sweepInFlight = runSweep().finally(() => { sweepInFlight = null; }); + sweepInFlight = runSweep().finally(() => { + sweepInFlight = null; + }); return sweepInFlight; } async function runSweep(): Promise { - const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id + const pending = db() + .prepare( + `SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') - AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=ds.deployment_id AND c.resolved_at IS NOT NULL) ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[]; + AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=ds.deployment_id AND c.resolved_at IS NOT NULL) ORDER BY d.started_at, d.id, ds.server` + ) + .all() as PendingRow[]; const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; for (const row of pending) { if (!row.files_changed) continue; // A failed file write is not a restart request. const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id }); result.checked++; - const state = operation.state === 'done' ? 'done' - : ['failed','interrupted','cancelled'].includes(operation.state) ? 'failed' : 'pending'; - const restartedAt = ['awaiting-health','done'].includes(operation.state) ? operation.submitted_at : row.restarted_at; + const state = + operation.state === 'done' + ? 'done' + : ['failed', 'interrupted', 'cancelled'].includes(operation.state) + ? 'failed' + : 'pending'; + const restartedAt = ['awaiting-health', 'done'].includes(operation.state) + ? operation.submitted_at + : row.restarted_at; if (restartedAt && !row.restarted_at) result.restarted++; if (state === 'done') result.healthy++; if (state === 'pending') result.held++; if (state === 'failed') result.failed++; - const error = state === 'done' ? null : operation.error ?? operation.stage; - db().prepare(`UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?, + const error = state === 'done' ? null : (operation.error ?? operation.stage); + db() + .prepare( + `UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + CASE WHEN ? IS NOT NULL AND checked_at IS NOT ? THEN 1 ELSE 0 END, - restarted_at = COALESCE(restarted_at, ?) WHERE deployment_id = ? AND server = ?`) - .run(state, operation.checked_at, operation.players, error, operation.checked_at, operation.checked_at, restartedAt, row.deployment_id, row.server); - if (state !== row.restart_state || operation.players !== row.players || error !== row.error) { - emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error }); + restarted_at = COALESCE(restarted_at, ?) WHERE deployment_id = ? AND server = ?` + ) + .run( + state, + operation.checked_at, + operation.players, + error, + operation.checked_at, + operation.checked_at, + restartedAt, + row.deployment_id, + row.server + ); + if ( + state !== row.restart_state || + operation.players !== row.players || + error !== row.error + ) { + emit({ + type: 'server-state', + deploymentId: row.deployment_id, + server: row.server, + state, + players: operation.players, + error + }); } } - db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS ( - SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso()); + db() + .prepare( + `UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS ( + SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))` + ) + .run(nowIso()); emit({ type: 'sweep-tick', frozen: false }); return result; } diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 82c003a..0cc28c1 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -10,7 +10,16 @@ import { emit } from './bus.js'; import type { ServerStatusEntry } from './bus.js'; import type { RconFailure } from '../integrations/rcon.js'; import { setServerFields, type ServerRow } from '../fleet/servers.js'; -import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, monitoringIdentity, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js'; +import { + inspectForMonitoring, + rconForMonitoring, + prepareMonitoringRcon, + monitoringStatus, + monitoringStatusWriter, + monitoringIdentity, + MonitoringChangedError, + MonitoringSkippedError +} from '../fleet/health-observations.js'; import { serverObservation } from '../fleet/status.js'; import { successfulStatusFields, activityFields } from '../fleet/status-updates.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js'; @@ -46,10 +55,17 @@ function tick(requestAnother = false): Promise { tickInFlight = (async () => { do { tickRequested = false; - try { await trackActivity('provisioning', 'Processing provisioning requests', () => processProvisioning({ wait: false })); } - catch (err) { console.error('[worker] provisioning error:', err); } + try { + await trackActivity('provisioning', 'Processing provisioning requests', () => + processProvisioning({ wait: false }) + ); + } catch (err) { + console.error('[worker] provisioning error:', err); + } } while (tickRequested); - })().finally(() => { tickInFlight = null; }); + })().finally(() => { + tickInFlight = null; + }); return tickInFlight; } @@ -64,7 +80,15 @@ function reconcileRequesters(): void { ['rebalancing', 'Reconciling rebalance moves', runRebalancing], ['game-files', 'Removing unused game files', cleanupControllerLibraries], ['chat', 'Saving match chat', collectFleetChat], - ['retention', 'Cleaning expired history', async () => { await drainRetention(); pruneAutoscaling(); pruneRebalancing(); }] + [ + 'retention', + 'Cleaning expired history', + async () => { + await drainRetention(); + pruneAutoscaling(); + pruneRebalancing(); + } + ] ]; for (const [key, label, work] of requesters) { if (requesterFlights.has(key)) continue; @@ -73,9 +97,21 @@ function reconcileRequesters(): void { nextRetentionAt = Date.now() + RETENTION_INTERVAL_MS; } const flight = trackActivity(key, label, work) - .catch(err => { console.error(`[worker] ${key} error:`, err); }) - .then(async result => { if (key !== 'game-files' && key !== 'retention' && key !== 'chat' && (!['autoscaling', 'rebalancing'].includes(key) || result === true)) await tick(true); }) - .finally(() => { requesterFlights.delete(key); }); + .catch((err) => { + console.error(`[worker] ${key} error:`, err); + }) + .then(async (result) => { + if ( + key !== 'game-files' && + key !== 'retention' && + key !== 'chat' && + (!['autoscaling', 'rebalancing'].includes(key) || result === true) + ) + await tick(true); + }) + .finally(() => { + requesterFlights.delete(key); + }); requesterFlights.set(key, flight); } } @@ -92,47 +128,73 @@ function encodeError(reason: RconFailure, detail: string): string { // Deduplicate queued/running checks per server. A slow server must not prevent // completed servers from being polled again on the next 30-second tick. const pendingStatus = new Map>(); -interface StatusQueue { active: number; queued: Array<() => Promise> } +interface StatusQueue { + active: number; + queued: Array<() => Promise>; +} const statusQueues = new Map(); const changedStatus = new Map(); function publishStatus(entry: ServerStatusEntry) { const scheduled = changedStatus.size > 0; changedStatus.set(entry.server, entry); - if (!scheduled) queueMicrotask(() => { - const entries = [...changedStatus.values()]; changedStatus.clear(); - emit({ type: 'server-status', delta: true, entries }); - }); + if (!scheduled) + queueMicrotask(() => { + const entries = [...changedStatus.values()]; + changedStatus.clear(); + emit({ type: 'server-status', delta: true, entries }); + }); } function drainStatusQueue(host: string, queue: StatusQueue): void { while (queue.active < STATUS_CONCURRENCY_PER_HOST && queue.queued.length) { const next = queue.queued.shift()!; queue.active++; - void next().finally(() => { queue.active--; drainStatusQueue(host, queue); }); + void next().finally(() => { + queue.active--; + drainStatusQueue(host, queue); + }); } if (!queue.active && !queue.queued.length) statusQueues.delete(host); } -function queueStatus(srv: ServerRow, snapshot: MonitoringSnapshot, probe?: () => ReturnType): Promise { +function queueStatus( + srv: ServerRow, + snapshot: MonitoringSnapshot, + probe?: () => ReturnType +): Promise { const server = srv.name; const pending = pendingStatus.get(server); if (pending) return pending; // Host queues keep slow instances from consuming other hosts' slots. const host = srv.host_id ? `native:${srv.host_id}` : `unregistered:${srv.name}`; let queue = statusQueues.get(host); - if (!queue) { queue = { active: 0, queued: [] }; statusQueues.set(host, queue); } + if (!queue) { + queue = { active: 0, queued: [] }; + statusQueues.set(host, queue); + } let finish!: (capacity: number | null) => void; - const work = new Promise(resolve => { finish = resolve; }); + const work = new Promise((resolve) => { + finish = resolve; + }); pendingStatus.set(server, work); queue.queued.push(async () => { let capacity: number | null = null; try { - capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server, snapshot, probe)); + capacity = await withOperationSource('Status monitor', () => + refreshServerStatus(server, snapshot, probe) + ); // Publish this server's latest evidence without waiting for the fleet. const current = snapshot.get(server)?.server; - if (current) publishStatus(statusEntry(current, snapshot.observe, observedPlayerCapacity(server, snapshot))); - } catch (err) { console.error(`[worker] status error for ${server}:`, err); } - finally { pendingStatus.delete(server); finish(capacity); } + if (current) + publishStatus( + statusEntry(current, snapshot.observe, observedPlayerCapacity(server, snapshot)) + ); + } catch (err) { + console.error(`[worker] status error for ${server}:`, err); + } finally { + pendingStatus.delete(server); + finish(capacity); + } }); drainStatusQueue(host, queue); return work; @@ -146,31 +208,63 @@ async function statusPass(): Promise { const snapshot = new MonitoringSnapshot(); const servers = snapshot.servers(); if (!servers.length) return; - const probes = prepareMonitoringRcon(servers.filter(server => !pendingStatus.has(server.name) - && !snapshot.provisioning(server.name) && !snapshot.settling(server.name) && snapshot.telemetryEvidence(server.name)?.players.length !== 0).map(server => server.name), snapshot); - const results = await Promise.all(servers.map(server => queueStatus(server, snapshot, probes.get(server.name)))); + const probes = prepareMonitoringRcon( + servers + .filter( + (server) => + !pendingStatus.has(server.name) && + !snapshot.provisioning(server.name) && + !snapshot.settling(server.name) && + snapshot.telemetryEvidence(server.name)?.players.length !== 0 + ) + .map((server) => server.name), + snapshot + ); + const results = await Promise.all( + servers.map((server) => queueStatus(server, snapshot, probes.get(server.name))) + ); const capacities = new Map(servers.map((server, i) => [server.name, results[i]])); try { // Metrics remain complete fleet collections, using freshness at capture // time rather than the status saved at the beginning of a long scan. snapshot.refresh(); - recordMetrics(snapshot.servers().map(srv => { - const current = snapshot.observe(srv); - return { server: srv.name, game: srv.game ?? 'unknown', - map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map', - players: current.health === 'online' ? current.last_players : null, - capacity: capacities.get(srv.name) ?? null }; - })); - } catch (err) { console.error('[worker] metrics error:', err); } + recordMetrics( + snapshot.servers().map((srv) => { + const current = snapshot.observe(srv); + return { + server: srv.name, + game: srv.game ?? 'unknown', + map: + selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug) + ?.friendly ?? + current.last_map ?? + srv.map ?? + 'Unknown map', + players: current.health === 'online' ? current.last_players : null, + capacity: capacities.get(srv.name) ?? null + }; + }) + ); + } catch (err) { + console.error('[worker] metrics error:', err); + } } -async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, probeStatus?: () => ReturnType): Promise { +async function refreshServerStatus( + server: string, + snapshot: MonitoringSnapshot, + probeStatus?: () => ReturnType +): Promise { snapshot.refresh(server); const srv = snapshot.get(server)?.server; if (!srv) return null; let capacity: number | null = null; let updates: Partial> = {}; - const flush = () => { setServerFields(srv.name, updates); snapshot.patch(srv.name, updates as Partial); updates = {}; }; + const flush = () => { + setServerFields(srv.name, updates); + snapshot.patch(srv.name, updates as Partial); + updates = {}; + }; if (!snapshot.provisioning(srv.name)) { const previousGame = monitoringStatus(srv.name, snapshot)?.game; const publishObservation = monitoringStatusWriter(srv.name, snapshot); @@ -178,25 +272,57 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, const settling = snapshot.settling(srv.name); const epoch = snapshot.get(srv.name)?.submission; let identity: string | undefined; - try { identity = monitoringIdentity(srv.name, snapshot); } catch { /* Offline hosts still need an unavailable sample. */ } + try { + identity = monitoringIdentity(srv.name, snapshot); + } catch { + /* Offline hosts still need an unavailable sample. */ + } const unchanged = () => { const current = snapshot.get(srv.name)?.server; - if (identity) try { if (identity !== monitoringIdentity(srv.name, snapshot)) return false; } catch { return false; } - return epoch === snapshot.get(srv.name)?.submission && !snapshot.provisioning(srv.name) - && current?.created_at === srv.created_at && current.desired_state === srv.desired_state - && current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation; + if (identity) + try { + if (identity !== monitoringIdentity(srv.name, snapshot)) return false; + } catch { + return false; + } + return ( + epoch === snapshot.get(srv.name)?.submission && + !snapshot.provisioning(srv.name) && + current?.created_at === srv.created_at && + current.desired_state === srv.desired_state && + current.native_instance_id === srv.native_instance_id && + current.instance_generation === srv.instance_generation + ); }; const inspectionStartedAt = new Date().toISOString(); const inspected = settling - ? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt } - : await inspectForMonitoring(srv.name, false, snapshot).catch(err => err instanceof MonitoringChangedError ? null - : { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt }); + ? { + result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, + observedAt: inspectionStartedAt + } + : await inspectForMonitoring(srv.name, false, snapshot).catch((err) => + err instanceof MonitoringChangedError + ? null + : { + result: { + ok: false as const, + state: 'unknown', + reason: (err as Error).message + }, + observedAt: inspectionStartedAt + } + ); snapshot.refresh(srv.name); if (!inspected) return null; const container = inspected.result; // A lifecycle request may be submitted while an inspection is in flight. - if (unchanged() && (snapshot.get(srv.name)?.server?.container_checked_at ?? '') <= inspected.observedAt) { - updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown'; + if ( + unchanged() && + (snapshot.get(srv.name)?.server?.container_checked_at ?? '') <= inspected.observedAt + ) { + updates.container_state = container.ok + ? 'running' + : (container.state?.toLowerCase() ?? 'unknown'); updates.container_checked_at = inspected.observedAt; // An unavailable inspection is not evidence that the game stopped. // Probe it just as a manual RCON check would; explicit lifecycle or @@ -210,7 +336,9 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, // Occupied telemetry rosters still need RCON for player pings. flush(); const probeStartedAt = new Date().toISOString(); - const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name, false, snapshot)).catch(err => { + const probe = await ( + probeStatus ? probeStatus() : rconForMonitoring(srv.name, false, snapshot) + ).catch((err) => { snapshot.refresh(srv.name); if (err instanceof MonitoringSkippedError) { // The container was checked, but game evidence retains its original age. @@ -218,13 +346,26 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, return null; } if (err instanceof MonitoringChangedError) return null; - return { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt }; + return { + result: { + ok: false as const, + endpoint: null, + reason: 'send-error' as RconFailure, + detail: (err as Error).message, + rawReply: undefined + }, + observedAt: probeStartedAt + }; }); snapshot.refresh(srv.name); if (!probe) return null; const result = probe.result; const previous = snapshot.get(srv.name)?.server; - if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) { + if ( + unchanged() && + previous && + (previous.last_checked_at ?? '') <= probe.observedAt + ) { gameObservation = probe; const now = probe.observedAt; updates.last_checked_at = now; @@ -235,10 +376,14 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, } else { // Only distinct accepted checks in the current lifecycle count. // Skipped/superseded checks return above without changing the streak. - const consecutive = previous.last_error && (!epoch || (previous.last_checked_at ?? '') > epoch); + const consecutive = + previous.last_error && + (!epoch || (previous.last_checked_at ?? '') > epoch); const failures = consecutive ? previous.rcon_failures || 1 : 0; - updates.rcon_failures = consecutive && previous.last_checked_at === now - ? failures : Math.min(2, failures + 1); + updates.rcon_failures = + consecutive && previous.last_checked_at === now + ? failures + : Math.min(2, failures + 1); updates.last_error = encodeError(result.reason, result.detail); } } @@ -257,17 +402,37 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, return capacity; } -export function statusEntry(srv: ServerRow, observe = serverObservation, capacity?: PlayerCapacityObservation): ServerStatusEntry { +export function statusEntry( + srv: ServerRow, + observe = serverObservation, + capacity?: PlayerCapacityObservation +): ServerStatusEntry { const final = observe(srv); - return { server: srv.name, players: final.last_players, capacity: capacity?.capacity ?? null, - capacityObservedAt: capacity?.capacity_observed_at ?? null, map: final.last_map ?? srv.map, - gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at, + return { + server: srv.name, + players: final.last_players, + capacity: capacity?.capacity ?? null, + capacityObservedAt: capacity?.capacity_observed_at ?? null, + map: final.last_map ?? srv.map, + gametype: final.last_gametype, + roster: final.roster, + health: final.health, + lastSeenAt: final.last_seen_at, lastError: final.last_error, - endpoint: srv.game_port ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null, - activeSince: final.active_since, inactiveSince: final.inactive_since, - gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at, - containerState: final.container_state, containerCheckedAt: final.container_checked_at, - telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry }; + endpoint: srv.game_port + ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } + : null, + reason: final.last_error ? (final.last_error.split(':')[0] as RconFailure) : null, + activeSince: final.active_since, + inactiveSince: final.inactive_since, + gameStartedAt: final.game_started_at, + gameEndedAt: final.game_ended_at, + containerState: final.container_state, + containerCheckedAt: final.container_checked_at, + telemetryAfter: final.telemetry_after, + rconStatus: final.rcon_status, + telemetry: final.telemetry + }; } export function startWorker(): void { @@ -276,13 +441,26 @@ export function startWorker(): void { void tick(); reconcileRequesters(); - void fanoutStatus().catch(err => { console.error('[worker] status fanout error:', err); }); - tickTimer = setInterval(() => { void tick(); reconcileRequesters(); }, TICK_MS); + void fanoutStatus().catch((err) => { + console.error('[worker] status fanout error:', err); + }); + tickTimer = setInterval(() => { + void tick(); + reconcileRequesters(); + }, TICK_MS); statusTimer = setInterval(() => { - void fanoutStatus().catch(err => { console.error('[worker] status fanout error:', err); }); + void fanoutStatus().catch((err) => { + console.error('[worker] status fanout error:', err); + }); }, STATUS_REFRESH_MS); - console.log('[worker] started — sweep every', TICK_MS, 'ms, status every', STATUS_REFRESH_MS, 'ms'); + console.log( + '[worker] started — sweep every', + TICK_MS, + 'ms, status every', + STATUS_REFRESH_MS, + 'ms' + ); } export function stopWorker(): void { @@ -303,4 +481,6 @@ export async function kick(): Promise { } /** Agent receipts need observation promptly, without launching another fleet scan. */ -export async function kickProvisioning(): Promise { await tick(true); } +export async function kickProvisioning(): Promise { + await tick(true); +} diff --git a/src/lib/server/provisioning/activity.ts b/src/lib/server/provisioning/activity.ts index c9ad121..f034a83 100644 --- a/src/lib/server/provisioning/activity.ts +++ b/src/lib/server/provisioning/activity.ts @@ -12,8 +12,12 @@ export interface ActivityQuery { } export function activitySummary(id: string): { total: number; latestId: number | null } { - return db().prepare(`SELECT COUNT(*) AS total, MAX(id) AS latestId - FROM server_deployment_events WHERE deployment_id = ?`).get(id) as { total: number; latestId: number | null }; + return db() + .prepare( + `SELECT COUNT(*) AS total, MAX(id) AS latestId + FROM server_deployment_events WHERE deployment_id = ?` + ) + .get(id) as { total: number; latestId: number | null }; } /** ID ordering and a fixed upper watermark keep older pages stable as new events arrive. */ @@ -25,32 +29,60 @@ export function deploymentActivity(id: string, options: ActivityQuery = {}): Pro const query = options.query ?? ''; const base = 'deployment_id = ? AND id <= ? AND instr(lower(message), lower(?)) > 0'; const values: (string | number)[] = [id, watermark, query]; - const total = (db().prepare(`SELECT COUNT(*) AS n FROM server_deployment_events WHERE ${base}`) - .get(...values) as { n: number }).n; + const total = ( + db() + .prepare(`SELECT COUNT(*) AS n FROM server_deployment_events WHERE ${base}`) + .get(...values) as { n: number } + ).n; const cursor = options.cursor === undefined ? '' : ` AND id ${newest ? '<' : '>'} ?`; if (options.cursor !== undefined) values.push(options.cursor); - const rows = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json + const rows = db() + .prepare( + `SELECT id, created_at, message, kind, phase_key, details_json FROM server_deployment_events WHERE ${base}${cursor} - ORDER BY id ${newest ? 'DESC' : 'ASC'} LIMIT ?`).all(...values, limit + 1) as ActivityRow[]; + ORDER BY id ${newest ? 'DESC' : 'ASC'} LIMIT ?` + ) + .all(...values, limit + 1) as ActivityRow[]; const more = rows.length > limit; const events = rows.slice(0, limit); - return { events, total, totalEvents: summary.total, latestId: summary.latestId, watermark, - nextCursor: more ? events.at(-1)!.id : null }; + return { + events, + total, + totalEvents: summary.total, + latestId: summary.latestId, + watermark, + nextCursor: more ? events.at(-1)!.id : null + }; } export function deploymentEvents(id: string, recent = false): ProvisioningEvent[] { - if (recent) return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json - FROM server_deployment_events WHERE deployment_id = ? ORDER BY id DESC LIMIT 5`).all(id) as ActivityRow[]).reverse(); - return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json - FROM server_deployment_events WHERE deployment_id = ? ORDER BY id`).all(id) as ActivityRow[]); + if (recent) + return ( + db() + .prepare( + `SELECT id, created_at, message, kind, phase_key, details_json + FROM server_deployment_events WHERE deployment_id = ? ORDER BY id DESC LIMIT 5` + ) + .all(id) as ActivityRow[] + ).reverse(); + return db() + .prepare( + `SELECT id, created_at, message, kind, phase_key, details_json + FROM server_deployment_events WHERE deployment_id = ? ORDER BY id` + ) + .all(id) as ActivityRow[]; } export function latestFailureEvent(id: string): ProvisioningEvent | null { - const row = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json + const row = db() + .prepare( + `SELECT id, created_at, message, kind, phase_key, details_json FROM server_deployment_events WHERE deployment_id = ? AND (kind IN ('failed', 'interrupted') OR (kind = 'outcome' AND CASE WHEN json_valid(details_json) THEN json_extract(details_json, '$.state') END IN ('failed', 'interrupted')) OR (kind IS NULL AND (message LIKE 'Failed:%' OR message LIKE 'Interrupted by application restart.%'))) - ORDER BY id DESC LIMIT 1`).get(id) as ActivityRow | undefined; + ORDER BY id DESC LIMIT 1` + ) + .get(id) as ActivityRow | undefined; return row ?? null; } diff --git a/src/lib/server/provisioning/agent-events.ts b/src/lib/server/provisioning/agent-events.ts index e0e136c..6e13140 100644 --- a/src/lib/server/provisioning/agent-events.ts +++ b/src/lib/server/provisioning/agent-events.ts @@ -4,25 +4,52 @@ import { recordProgress } from './progress.js'; /** Record receipts in the report transaction, independently of the execution worker. */ export function recordAgentReceipt(operationId: string, state: string, at: string): void { // Host operations also serve installations that have no provisioning schema yet. - if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_provisioning'").get()) return; - const job = db().prepare(`SELECT d.id, d.phase_key, COALESCE(n.operation_id,m.remove_operation_id) AS operation_id FROM native_provisioning n + if ( + !db() + .prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_provisioning'" + ) + .get() + ) + return; + const job = db() + .prepare( + `SELECT d.id, d.phase_key, COALESCE(n.operation_id,m.remove_operation_id) AS operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id WHERE (n.operation_id=? OR n.stage_operation_id=? OR m.remove_operation_id=?) AND d.server_deleted_at IS NULL - AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId, operationId) as + AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')` + ) + .get(operationId, operationId, operationId) as { id: string; phase_key: string | null; operation_id: string | null } | undefined; if (!job) return; const staging = job.operation_id !== operationId; const message = `${staging ? 'File staging' : 'Host agent command'} ${state} (${operationId})`; if (!staging && job.phase_key === 'container-action') { - db().prepare('UPDATE server_deployments SET stage=? WHERE id=?').run(`Host agent command ${state}`, job.id); + db() + .prepare('UPDATE server_deployments SET stage=? WHERE id=?') + .run(`Host agent command ${state}`, job.id); } recordProgress(job.id, { blocker: null, event: { kind: 'agent', message, details: { operationId, state, receivedAt: at } }, - ...(['accepted', 'running', 'completed'].includes(state) ? { - checkpoint: { key: `${staging ? 'staging' : 'agent'}-${state}`, label: message, at }, - ...(!staging ? { command: { state: 'accepted' as const, message: 'Host agent acknowledged the command.', at } } : {}) - } : {}) + ...(['accepted', 'running', 'completed'].includes(state) + ? { + checkpoint: { + key: `${staging ? 'staging' : 'agent'}-${state}`, + label: message, + at + }, + ...(!staging + ? { + command: { + state: 'accepted' as const, + message: 'Host agent acknowledged the command.', + at + } + } + : {}) + } + : {}) }); } diff --git a/src/lib/server/provisioning/cancel-creation.ts b/src/lib/server/provisioning/cancel-creation.ts index fa9ee86..28e82c1 100644 --- a/src/lib/server/provisioning/cancel-creation.ts +++ b/src/lib/server/provisioning/cancel-creation.ts @@ -7,7 +7,9 @@ import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } fro import { recordProgress } from './progress.js'; export function canCancelCreation(id: string): boolean { - return !!db().prepare(`SELECT 1 FROM server_deployments d + return !!db() + .prepare( + `SELECT 1 FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL AND (i.generation=n.generation OR (n.recovery_step='rebuild' AND i.generation=n.generation+1 @@ -21,55 +23,110 @@ export function canCancelCreation(id: string): boolean { AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled' AND NOT (other.action='destroy' AND other.state='waiting' - AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id); + AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))` + ) + .get(id); } /** Stops observation immediately; remote cleanup remains a durable, separately visible job. */ export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = true): string | null { return db().transaction(() => { - const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined; + const prior = db() + .prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?') + .get(id) as { cleanup_id: string } | undefined; if (prior) return prior.cleanup_id; if (!canCancelCreation(id)) return null; if (pausePolicy) pauseScalingForOperation(id); - const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id,n.generation,n.recovery_step FROM server_deployments d - LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as - { server: string; operation_id: string | null; stage_operation_id: string | null; generation: number | null; recovery_step: 'restart' | 'rebuild' | null }; + const job = db() + .prepare( + `SELECT d.server,n.operation_id,n.stage_operation_id,n.generation,n.recovery_step FROM server_deployments d + LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?` + ) + .get(id) as { + server: string; + operation_id: string | null; + stage_operation_id: string | null; + generation: number | null; + recovery_step: 'restart' | 'rebuild' | null; + }; const instance = nativeInstance(job.server); // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding. - for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) { + for (const removal of db() + .prepare( + "SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL" + ) + .all(job.server) as { id: string }[]) { cancelRemoval(removal.id); } const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy); - db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId); - db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id); - deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null, - event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } }); + db() + .prepare( + 'INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)' + ) + .run(id, cleanupId); + db() + .prepare( + "UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?" + ) + .run(new Date().toISOString(), id); + deploymentStage(id, 'Creation cancelled; cleanup requested', { + blocker: null, + event: { + kind: 'cancelled', + message: + 'Creation cancelled. Its partial instance will be removed without a player-count check.', + details: { cleanupId } + } + }); if (!job.operation_id && !job.stage_operation_id) { // No remote mutation was ever queued. The reservation is the only resource to remove. if (instance) { - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), instance.id); releaseHostPorts(instance.id); } - deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup', - checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } }); + deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { + phase: 'cleanup', + checkpoint: { + key: 'cleanup', + label: 'No host command submitted; local reservations released' + } + }); finishDeployment(cleanupId, { ok: true }); finishCreationCancellation(cleanupId); } else { if (!instance) throw new Error('The original native instance is unavailable.'); let generation = instance.generation; - const payload: { createOperationId: string; recoveryRebuildOperationId?: string; recoveryGeneration?: number } = { createOperationId: job.operation_id ?? id }; + const payload: { + createOperationId: string; + recoveryRebuildOperationId?: string; + recoveryGeneration?: number; + } = { createOperationId: job.operation_id ?? id }; if (job.recovery_step) { // Recovery replaces operation_id, but cancellation must retain the // immutable ownership label from the original launch. - const original = db().prepare(`SELECT id FROM ec_host_operations WHERE host_id=? AND instance_id=? - AND generation=? AND kind='instance.create' ORDER BY rowid DESC LIMIT 1`) - .get(instance.host_id, instance.id, job.generation) as { id: string } | undefined; - if (!original || !job.generation) throw new Error('The original creation command is unavailable.'); + const original = db() + .prepare( + `SELECT id FROM ec_host_operations WHERE host_id=? AND instance_id=? + AND generation=? AND kind='instance.create' ORDER BY rowid DESC LIMIT 1` + ) + .get(instance.host_id, instance.id, job.generation) as + { id: string } | undefined; + if (!original || !job.generation) + throw new Error('The original creation command is unavailable.'); payload.createOperationId = original.id; generation = job.generation; if (job.recovery_step === 'rebuild') { - const recovery = job.operation_id ? getHostOperationStatus(job.operation_id) : null; - if (!recovery || recovery.kind !== 'instance.rebuild' || recovery.instanceId !== instance.id || recovery.generation !== generation) { + const recovery = job.operation_id + ? getHostOperationStatus(job.operation_id) + : null; + if ( + !recovery || + recovery.kind !== 'instance.rebuild' || + recovery.instanceId !== instance.id || + recovery.generation !== generation + ) { throw new Error('The creation recovery command is unavailable.'); } // A delivered rebuild may have created the next generation @@ -78,33 +135,77 @@ export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = tru payload.recoveryGeneration = generation + 1; } } - const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', payload, - { instanceId: instance.id, generation, idempotencyKey: cleanupId }); - db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId); - db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId); - deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null, - command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } }); + const operation = queueHostOperation( + instance.host_id, + 'instance.cancel-create', + payload, + { instanceId: instance.id, generation, idempotencyKey: cleanupId } + ); + db() + .prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?') + .run(operation.id, cleanupId); + db() + .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') + .run(new Date().toISOString(), cleanupId); + deploymentStage( + cleanupId, + 'Waiting for host agent to cancel creation and remove its partial instance', + { + phase: 'container-action', + blocker: null, + command: { + state: 'attempted', + message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` + } + } + ); } return cleanupId; })(); } export function finishCreationCancellation(cleanupId: string): void { - const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id,d.server FROM native_creation_cancellations c + const cancellation = db() + .prepare( + `SELECT c.creation_id,n.operation_id,d.server FROM native_creation_cancellations c JOIN server_deployments d ON d.id=c.creation_id - LEFT JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as + LEFT JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?` + ) + .get(cleanupId) as { creation_id: string; operation_id: string | null; server: string } | undefined; if (!cancellation) return; releaseServerKey(cancellation.server); const now = new Date().toISOString(); - db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId); - deploymentStage(cancellation.creation_id, 'Creation cancelled; cleanup completed', { blocker: null, - checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } }); + db() + .prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)') + .run(now, cancellation.creation_id, cleanupId); + deploymentStage(cancellation.creation_id, 'Creation cancelled; cleanup completed', { + blocker: null, + checkpoint: { + key: 'cleanup', + phase: 'cleanup', + label: 'Cancellation cleanup confirmed; instance and reservations removed' + } + }); // The original command may have lost its receipt. Cleanup proves that its effects are now gone. if (cancellation.operation_id) { - const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')") + const result = db() + .prepare( + "UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')" + ) .run(now, cancellation.operation_id); - if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now); + if (result.changes) + db() + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')" + ) + .run(cancellation.operation_id, now); } - recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } }); + recordProgress(cleanupId, { + event: { + kind: 'cleanup', + message: 'Cancelled creation cleanup completed.', + details: { creationId: cancellation.creation_id } + } + }); } diff --git a/src/lib/server/provisioning/cancellation.ts b/src/lib/server/provisioning/cancellation.ts index ed81020..6edb1cb 100644 --- a/src/lib/server/provisioning/cancellation.ts +++ b/src/lib/server/provisioning/cancellation.ts @@ -8,12 +8,19 @@ export const rebalanceCancellationGuard = `NOT (d.triggered_by LIKE 'rebalance:% SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id AND m.remove_operation_id IS NOT NULL))`; export function rebalanceCancellationBlocker(id: string): string | null { - return db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND NOT (${rebalanceCancellationGuard})`).get(id) - ? 'This rebalance move has committed source removal and must finish recreating the lobby. Stop the rebalance run to withdraw its remaining moves.' : null; + return db() + .prepare( + `SELECT 1 FROM server_deployments d WHERE d.id=? AND NOT (${rebalanceCancellationGuard})` + ) + .get(id) + ? 'This rebalance move has committed source removal and must finish recreating the lobby. Stop the rebalance run to withdraw its remaining moves.' + : null; } export function provisioningCancelled(id: string): boolean { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id); + return !!db() + .prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'") + .get(id); } export function assertProvisioningNotCancelled(id: string): void { @@ -21,49 +28,102 @@ export function assertProvisioningNotCancelled(id: string): void { } export function canCancelProvisioningJob(id: string): boolean { - return !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND d.server_deleted_at IS NULL - AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND ${rebalanceCancellationGuard}`).get(id); + return !!db() + .prepare( + `SELECT 1 FROM server_deployments d WHERE d.id=? AND d.server_deleted_at IS NULL + AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND ${rebalanceCancellationGuard}` + ) + .get(id); } /** Cancel remaining work, not effects already performed by the host. */ -export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator', options: { preserveRebalanceRun?: boolean } = {}): boolean { +export function cancelProvisioningJob( + id: string, + reason = 'Cancelled by operator', + options: { preserveRebalanceRun?: boolean } = {} +): boolean { ensureHostSchema(db()); return db().transaction(() => { if (provisioningCancelled(id)) return true; if (!canCancelProvisioningJob(id)) return false; - const rebalance = !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id); + const rebalance = !!db() + .prepare( + "SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'" + ) + .get(id); if (rebalance) { if (!options.preserveRebalanceRun) requestRebalanceStopForOperation(id); } else pauseScalingForOperation(id); const at = new Date().toISOString(); // Only commands never delivered can be withdrawn. Delivered commands retain // their immutable IDs and receipts; replacements wait for their resolution. - const commands = db().prepare(`SELECT o.id FROM ec_host_operations o + const commands = db() + .prepare( + `SELECT o.id FROM ec_host_operations o WHERE o.state='pending' AND o.last_delivered_at IS NULL AND o.id IN ( SELECT operation_id FROM native_provisioning WHERE deployment_id=? UNION SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=? - UNION SELECT remove_operation_id FROM ec_host_migrations WHERE deployment_id=?)`).all(id, id, id) as { id: string }[]; + UNION SELECT remove_operation_id FROM ec_host_migrations WHERE deployment_id=?)` + ) + .all(id, id, id) as { id: string }[]; for (const command of commands) { - db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(reason, at, command.id); - db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(command.id, at, reason); + db() + .prepare( + "UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?" + ) + .run(reason, at, command.id); + db() + .prepare( + "INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)" + ) + .run(command.id, at, reason); } - db().prepare(`UPDATE staged_file_changes SET state='cancelled' WHERE state='pending' AND EXISTS ( + db() + .prepare( + `UPDATE staged_file_changes SET state='cancelled' WHERE state='pending' AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id=? AND s.kind=staged_file_changes.source_kind - AND s.source_id=staged_file_changes.source_id AND s.server=staged_file_changes.server)`).run(id); + AND s.source_id=staged_file_changes.source_id AND s.server=staged_file_changes.server)` + ) + .run(id); db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(id); - db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage=?,error=NULL WHERE id=?").run(at, reason, id); - recordProgress(id, { blocker: null, checks: [], - recovery: { state: 'released', at, operationId: id, message: 'Recovery capacity released by cancellation; this does not certify server health.' }, - event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt: at, error: null, warnings: [] } } }); + db() + .prepare( + "UPDATE server_deployments SET state='cancelled',finished_at=?,stage=?,error=NULL WHERE id=?" + ) + .run(at, reason, id); + recordProgress(id, { + blocker: null, + checks: [], + recovery: { + state: 'released', + at, + operationId: id, + message: + 'Recovery capacity released by cancellation; this does not certify server health.' + }, + event: { + kind: 'outcome', + message: reason, + details: { state: 'cancelled', finishedAt: at, error: null, warnings: [] } + } + }); return true; })(); } /** A cancelled controller job must not let a replacement overtake a delivered mutation. */ -export function unsettledCancelledCommand(server: string, replacementId: string): { - deploymentId: string; operationId: string; state: string; error: string | null; +export function unsettledCancelledCommand( + server: string, + replacementId: string +): { + deploymentId: string; + operationId: string; + state: string; + error: string | null; } | null { - const row = db().prepare(`SELECT d.id AS deploymentId,o.id AS operationId,o.state,o.error FROM server_deployments d + const row = db() + .prepare( + `SELECT d.id AS deploymentId,o.id AS operationId,o.state,o.error FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id JOIN ec_host_operations o ON o.id=n.operation_id OR o.id=m.remove_operation_id @@ -73,7 +133,10 @@ export function unsettledCancelledCommand(server: string, replacementId: string) OR (o.state='failed' AND o.id=n.operation_id AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)) AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?) - ORDER BY d.rowid,o.rowid LIMIT 1`).get(server, replacementId) as - { deploymentId: string; operationId: string; state: string; error: string | null } | undefined; + ORDER BY d.rowid,o.rowid LIMIT 1` + ) + .get(server, replacementId) as + | { deploymentId: string; operationId: string; state: string; error: string | null } + | undefined; return row ?? null; } diff --git a/src/lib/server/provisioning/concurrency.ts b/src/lib/server/provisioning/concurrency.ts index 4e13242..2cdc23b 100644 --- a/src/lib/server/provisioning/concurrency.ts +++ b/src/lib/server/provisioning/concurrency.ts @@ -7,14 +7,22 @@ export const preparingJobs = new Set(); /** A committed rebalance keeps its startup reservation while between source removal and creation. */ export const committedRebalanceStartup = `(triggered_by LIKE 'rebalance:%' AND state IN ('waiting','interrupted') AND EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL))`; -interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number } +interface Holder { + id: string; + action: string; + host_id: string | null; + target_host_id: string | null; + removing: number; +} export function lifecycleHolders(): Holder[] { ensureHostSchema(db()); // Find capacity owners through their active state or outstanding command. // A correlated OR over d used to walk every retained job and fetch its agent // receipt on every admission, even when all that history was completed. - return db().prepare(`SELECT d.id,CASE WHEN n.recovery_step='rebuild' THEN 'rebuild' ELSE d.action END AS action,i.host_id,m.target_host_id, + return db() + .prepare( + `SELECT d.id,CASE WHEN n.recovery_step='rebuild' THEN 'rebuild' ELSE d.action END AS action,i.host_id,m.target_host_id, CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id @@ -33,29 +41,64 @@ export function lifecycleHolders(): Holder[] { UNION SELECT m.deployment_id FROM ec_host_operations o JOIN ec_host_migrations m ON m.remove_operation_id=o.id WHERE o.state IN ('pending','accepted','running','unknown') - )`) + )` + ) .all(JSON.stringify([...preparingJobs])) as Holder[]; } /** Recheck inside the command transaction after every asynchronous preflight. */ -export function concurrencyBlocker(id: string, action: string, hostId?: string, removing = false): string | null { +export function concurrencyBlocker( + id: string, + action: string, + hostId?: string, + removing = false +): string | null { const settings = fleetSettings(); const holders = lifecycleHolders(); - const others = holders.filter(row => row.id !== id); - const own = holders.find(row => row.id === id); + const others = holders.filter((row) => row.id !== id); + const own = holders.find((row) => row.id === id); const continuing = own && !preparingJobs.has(id); const full = (limit: number, used: number) => limit > 0 && used >= limit; - if (!continuing && full(settings.lifecycleConcurrency, others.length)) return 'Waiting for combined lifecycle capacity.'; - if (!continuing && action === 'rebuild' && full(settings.rebuildConcurrency, others.filter(row => row.action === 'rebuild').length)) return 'Waiting for rebuild capacity.'; + if (!continuing && full(settings.lifecycleConcurrency, others.length)) + return 'Waiting for combined lifecycle capacity.'; + if ( + !continuing && + action === 'rebuild' && + full(settings.rebuildConcurrency, others.filter((row) => row.action === 'rebuild').length) + ) + return 'Waiting for rebuild capacity.'; if (!hostId || continuing) return null; - if (['create','rebuild'].includes(action) && full(settings.createConcurrencyPerHost, - others.filter(row => ['create','rebuild'].includes(row.action) && (row.host_id === hostId || row.target_host_id === hostId)).length)) return 'Waiting for startup capacity on this host.'; - if ((action === 'destroy' || removing) && full(settings.destroyConcurrencyPerHost, - others.filter(row => (row.action === 'destroy' || row.removing) && row.host_id === hostId).length)) return 'Waiting for destruction capacity on this host.'; - if (['start','restart','rebuild'].includes(action)) { - const slots = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN native_instances i - ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)`).all(hostId, id); - if (full(settings.restartConcurrencyPerHost, slots.length)) return 'Waiting for restart recovery capacity on this host.'; + if ( + ['create', 'rebuild'].includes(action) && + full( + settings.createConcurrencyPerHost, + others.filter( + (row) => + ['create', 'rebuild'].includes(row.action) && + (row.host_id === hostId || row.target_host_id === hostId) + ).length + ) + ) + return 'Waiting for startup capacity on this host.'; + if ( + (action === 'destroy' || removing) && + full( + settings.destroyConcurrencyPerHost, + others.filter( + (row) => (row.action === 'destroy' || row.removing) && row.host_id === hostId + ).length + ) + ) + return 'Waiting for destruction capacity on this host.'; + if (['start', 'restart', 'rebuild'].includes(action)) { + const slots = db() + .prepare( + `SELECT r.operation_id FROM restart_rollout r JOIN native_instances i + ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND i.host_id=? AND (r.operation_id IS NULL OR r.operation_id!=?)` + ) + .all(hostId, id); + if (full(settings.restartConcurrencyPerHost, slots.length)) + return 'Waiting for restart recovery capacity on this host.'; } return null; } diff --git a/src/lib/server/provisioning/health-handoff.ts b/src/lib/server/provisioning/health-handoff.ts index 1064ac7..03e2134 100644 --- a/src/lib/server/provisioning/health-handoff.ts +++ b/src/lib/server/provisioning/health-handoff.ts @@ -1,36 +1,53 @@ import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { monitoringIdentity } from '../fleet/health-observations.js'; import { getServer, setServerFields, type ServerRow } from '../fleet/servers.js'; -import { latestGameSubmission, telemetryReport, type VerifiedGameStatus } from '../fleet/status-evidence.js'; +import { + latestGameSubmission, + telemetryReport, + type VerifiedGameStatus +} from '../fleet/status-evidence.js'; import { serverObservation } from '../fleet/status.js'; import { activityFields, successfulStatusFields } from '../fleet/status-updates.js'; /** Called inside the completion transaction; evidence must still describe this process. */ -export function handoffVerifiedHealth(server: string, identity: string, - container: { state: string; observedAt: string }, game: VerifiedGameStatus): boolean { +export function handoffVerifiedHealth( + server: string, + identity: string, + container: { state: string; observedAt: string }, + game: VerifiedGameStatus +): boolean { const current = getServer(server); - if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) return false; + if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) + return false; const fresh = (at: string) => { const age = Date.now() - Date.parse(at); return age >= 0 && age <= ONLINE_MAX_MS; }; const gameAt = game.source === 'telemetry' ? game.receivedAt : game.observedAt; const submission = latestGameSubmission(server); - if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) return false; + if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) + return false; const updates: Partial = {}; if (!current.container_checked_at || current.container_checked_at <= container.observedAt) { updates.container_state = container.state; updates.container_checked_at = container.observedAt; } - if (game.source === 'rcon' && (!current.last_checked_at || current.last_checked_at <= game.observedAt)) { + if ( + game.source === 'rcon' && + (!current.last_checked_at || current.last_checked_at <= game.observedAt) + ) { Object.assign(updates, successfulStatusFields(game.rcon, game.observedAt)); } const candidate = { ...current, ...updates }; // Evaluate the health readers will see after the operation and recovery slot // complete. Newer contradictory evidence must keep verification pending. - const status = serverObservation(candidate, { provisioning: false, settling: false, - submission, telemetry: telemetryReport(server) }); + const status = serverObservation(candidate, { + provisioning: false, + settling: false, + submission, + telemetry: telemetryReport(server) + }); if (status.health !== 'online') return false; setServerFields(server, { ...updates, ...activityFields(candidate, status) }); return true; diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index f480297..0563fd2 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -4,52 +4,107 @@ import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js'; import { actionLabels, type ProvisioningAction } from '../../provisioning.js'; -import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js'; +import type { + ProvisioningCapacity, + ProvisioningDeployment, + ProvisioningDetail, + ProvisioningSource +} from '../../provisioning-detail.js'; import { fleetSettings } from '../settings.js'; import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js'; import { readProgress, recordProgress, type ProgressUpdate } from './progress.js'; import { bindNativeOperation } from './native.js'; import { workerActivity } from '../pipeline/activity.js'; import { canCancelCreation } from './cancel-creation.js'; -import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js'; +import { + canCancelProvisioningJob, + provisioningCancelled, + rebalanceCancellationGuard +} from './cancellation.js'; import { committedRebalanceStartup } from './concurrency.js'; import { nativeDeployment } from '../backends/native.js'; import { ensureHostSchema } from '../hosts/schema.js'; import { safetyOverride, safetyOverrideAvailability } from './safety-override.js'; import { serverTitle } from '../../ui/server-labels.js'; import { provisioningQueueStatus } from '../../ui/provisioning-queue.js'; -import { provisioningPageSize, provisioningPageOffset, provisioningSourceKinds, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js'; +import { + provisioningPageSize, + provisioningPageOffset, + provisioningSourceKinds, + type ProvisioningListOptions, + type ProvisioningListPage, + type ProvisioningListRow +} from '../../provisioning-list.js'; -export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string { +export function beginDeployment( + server: string, + action: ProvisioningAction, + triggeredBy = 'ui' +): string { const id = randomUUID(); db().transaction(() => { - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by) - VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy, - triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') ? currentRequestUsername() : null); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by) + VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)` + ) + .run( + id, + server, + action, + new Date().toISOString(), + triggeredBy, + triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') + ? currentRequestUsername() + : null + ); bindNativeOperation(id, server); - recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested', - label: action === 'create' ? 'Server creation requested' - : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } }); + recordProgress(id, { + phase: 'requested', + observedAt: null, + checkpoint: { + key: 'requested', + phase: 'requested', + label: + action === 'create' + ? 'Server creation requested' + : action === 'destroy' + ? 'Server destruction requested' + : `${actionLabels[action]} server requested` + } + }); })(); return id; } export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void { db().transaction(() => { - const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined; + const previous = db() + .prepare('SELECT stage FROM server_deployments WHERE id = ?') + .get(id) as { stage: string } | undefined; const changed = !!previous && previous.stage !== stage; - const before = update && changed ? activitySummary(id).latestId ?? 0 : 0; + const before = update && changed ? (activitySummary(id).latestId ?? 0) : 0; db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id); if (update) recordProgress(id, update); // Structured transitions may already carry the exact legacy stage text. - const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events - WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage); + const alreadyRecorded = + update && + changed && + db() + .prepare( + `SELECT 1 FROM server_deployment_events + WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1` + ) + .get(id, before, stage); if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage); })(); } export function recordDeploymentEvent(id: string, message: string): void { - db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)') + db() + .prepare( + 'INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)' + ) .run(id, new Date().toISOString(), message); } @@ -58,103 +113,239 @@ function warningMessages(raw: string): string[] { const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.']; const messages = parsed.filter((value): value is string => typeof value === 'string'); - if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.'); + if (messages.length !== parsed.length) + messages.push('Some recorded warnings could not be read.'); return messages; - } catch { return ['Recorded warnings could not be read.']; } + } catch { + return ['Recorded warnings could not be read.']; + } } function sourceDetails(source: Omit): ProvisioningSource { const id = source.source_id; if (source.kind === 'rebalance') { const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id); - return { ...source, label: exists ? 'Rebalance run' : 'Rebalance history expired', - href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null }; + return { + ...source, + label: exists ? 'Rebalance run' : 'Rebalance history expired', + href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null + }; } if (source.kind === 'autoscaling') { const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id); - return { ...source, label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null }; + return { + ...source, + label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', + href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null + }; } if (source.kind === 'campaign') { - const row = db().prepare(`SELECT d.action, d.beta_id, s.name FROM deployments d + const row = db() + .prepare( + `SELECT d.action, d.beta_id, s.name FROM deployments d LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1)) - WHERE d.id=?`).get(id) as { action: string; beta_id: number | null; name: string | null } | undefined; - const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Maintenance repair' : 'Campaign'; - return { ...source, label: row ? `${action}${row.beta_id ? ` · Beta #${row.beta_id}` : row.name ? ` · ${row.name}` : action === 'Campaign' ? '' : ' campaign'}` : 'Campaign unavailable', href: row ? `/campaigns/${encodeURIComponent(id)}` : null }; + WHERE d.id=?` + ) + .get(id) as { action: string; beta_id: number | null; name: string | null } | undefined; + const action = + row?.action === 'promote' + ? 'Promotion' + : row?.action === 'rollback' + ? 'Rollback' + : row?.action === 'drift' + ? 'Maintenance repair' + : 'Campaign'; + return { + ...source, + label: row + ? `${action}${row.beta_id ? ` · Beta #${row.beta_id}` : row.name ? ` · ${row.name}` : action === 'Campaign' ? '' : ' campaign'}` + : 'Campaign unavailable', + href: row ? `/campaigns/${encodeURIComponent(id)}` : null + }; } if (source.kind === 'beta') { - const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined; + const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as + { notes: string | null } | undefined; const label = row?.notes?.trim().split('\n')[0].slice(0, 120); - return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null }; + return { + ...source, + label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, + href: row ? `/betas/${encodeURIComponent(id)}` : null + }; } if (source.kind === 'watchdog') { - const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined; - return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null }; + const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as + { summary: string } | undefined; + return { + ...source, + label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, + href: row ? `/maintenance/${encodeURIComponent(id)}` : null + }; } - const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined; - return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`, - href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null }; + const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as + { kind: string } | undefined; + return { + ...source, + label: + row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`, + href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null + }; } /** Active operations get current pool snapshots, not a second admission/eligibility engine. */ -function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity { - if (!['queued', 'waiting', 'running', 'awaiting-health'].includes(deployment.state)) return { evaluatedAt, pools: [] }; +function capacityFor( + deployment: ProvisioningDeployment, + evaluatedAt: string +): ProvisioningCapacity { + if (!['queued', 'waiting', 'running', 'awaiting-health'].includes(deployment.state)) + return { evaluatedAt, pools: [] }; const settings = fleetSettings(); const pools: ProvisioningCapacity['pools'] = []; if (deployment.action === 'create' || deployment.action === 'rebuild') { - const jobs = db().prepare(`SELECT id, server, state FROM server_deployments + const jobs = db() + .prepare( + `SELECT id, server, state FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup}) - ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[]; - const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server })); - const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id); - pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders, - requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) }); + ORDER BY started_at, rowid` + ) + .all() as { id: string; server: string; state: string }[]; + const holders = jobs + .filter((job) => job.state !== 'queued') + .map(({ id, server }) => ({ id, server })); + const order = jobs + .filter((job) => job.state === 'queued') + .findIndex((job) => job.id === deployment.id); + pools.push({ + kind: 'startup', + limit: settings.createConcurrency, + used: holders.length, + holders, + requestOrder: order < 0 ? null : order + 1, + ownsSlot: holders.some((holder) => holder.id === deployment.id) + }); } - const creationRecovery = deployment.action === 'create' && db().prepare(`SELECT 1 FROM native_provisioning - WHERE deployment_id=? AND recovery_step IS NOT NULL`).get(deployment.id); + const creationRecovery = + deployment.action === 'create' && + db() + .prepare( + `SELECT 1 FROM native_provisioning + WHERE deployment_id=? AND recovery_step IS NOT NULL` + ) + .get(deployment.id); if (['start', 'restart', 'rebuild'].includes(deployment.action) || creationRecovery) { - const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL - ORDER BY requested_at, server`).all() as { id: string | null; server: string }[]; - pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders, - requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) }); + const holders = db() + .prepare( + `SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL + ORDER BY requested_at, server` + ) + .all() as { id: string | null; server: string }[]; + pools.push({ + kind: 'restart', + limit: settings.restartConcurrency, + used: holders.length, + holders, + requestOrder: null, + ownsSlot: holders.some((holder) => holder.id === deployment.id) + }); } return { evaluatedAt, pools }; } /** The default retains the full legacy event response. Page clients opt into a bounded window. */ -export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null { +export function getDeployment( + id: string, + options: { events?: 'all' | 'recent' } = {} +): ProvisioningDetail | null { return db().transaction(() => { - const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists - FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as - (ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined; + const row = db() + .prepare( + `SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists + FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?` + ) + .get(id) as + | (ProvisioningDeployment & { + observation_json: string | null; + phase_key: string | null; + phase_started_at: string | null; + observed_at: string | null; + }) + | undefined; if (!row) return null; const generatedAt = new Date().toISOString(); const native = nativeDeployment(id); - const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error + const agentOperation = native + ? (db() + .prepare( + `SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error FROM native_provisioning n LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id - JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null; - const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c - JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation']; - const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row; - const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id') - .all(id) as Omit[]).map(sourceDetails); - if (row.triggered_by.startsWith('rebalance:')) sources.push(sourceDetails({ kind: 'rebalance', source_id: row.triggered_by.slice('rebalance:'.length) })); + JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?` + ) + .get(id) as ProvisioningDetail['agentOperation']) + : null; + const cancellation = db() + .prepare( + `SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c + JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?` + ) + .get(id) as ProvisioningDetail['cancellation']; + const { + observation_json: _snapshot, + phase_key: _phase, + phase_started_at: _phaseStarted, + observed_at: _observed, + ...deployment + } = row; + const sources = ( + db() + .prepare( + 'SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id' + ) + .all(id) as Omit[] + ).map(sourceDetails); + if (row.triggered_by.startsWith('rebalance:')) + sources.push( + sourceDetails({ + kind: 'rebalance', + source_id: row.triggered_by.slice('rebalance:'.length) + }) + ); const launch = recipeSummaryForDeployment(id); - const collection = db().prepare(`SELECT c.id, c.name FROM ( + const collection = db() + .prepare( + `SELECT c.id, c.name FROM ( SELECT collection_id, deployment_id FROM server_collection_members UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m - JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined; + JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1` + ) + .get(id) as { id: string; name: string } | undefined; const override = safetyOverrideAvailability(id); - return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null, - canOverrideSafety: override.applicable, safetyOverrideUnavailableReason: override.unavailableReason, safetyOverride: safetyOverride(id), - canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null, - workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'), - activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources, - recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null, - canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), - canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id), - canCancelRemoval: canCancelRemoval(id), collection: collection ?? null, - progress: readProgress(row), capacity: capacityFor(deployment, generatedAt), - warningMessages: warningMessages(deployment.warnings), generatedAt }; + return { + deployment, + backend: 'native' as const, + agentOperation: agentOperation ?? null, + canOverrideSafety: override.applicable, + safetyOverrideUnavailableReason: override.unavailableReason, + safetyOverride: safetyOverride(id), + canCancel: canCancelProvisioningJob(id), + canCancelCreation: canCancelCreation(id), + cancellation: cancellation ?? null, + workerActivity: workerActivity(), + events: deploymentEvents(id, options.events === 'recent'), + activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, + sources, + recipe: launch?.recipe ?? null, + blueprintName: launch?.blueprintName ?? null, + canResumeVerification: canResumeVerification(id), + canResumeCleanup: canResumeCleanup(id), + canResumeRebuild: canResumeRebuild(id), + canRetryWithRebuild: canRetryWithRebuild(id), + canCancelRemoval: canCancelRemoval(id), + collection: collection ?? null, + progress: readProgress(row), + capacity: capacityFor(deployment, generatedAt), + warningMessages: warningMessages(deployment.warnings), + generatedAt + }; })(); } @@ -162,29 +353,46 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } export const REMOVING_SERVER = 'Removing server files and registry'; export function canCancelRemoval(id: string): boolean { - return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER); + return !!db() + .prepare( + "SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?" + ) + .get(id, REMOVING_SERVER); } /** Competes atomically with the worker's claim, before any external removal. */ export function cancelRemoval(id: string, pausePolicy = true): boolean { return db().transaction(() => { const finishedAt = new Date().toISOString(); - const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL - WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER); + const result = db() + .prepare( + `UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL + WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?` + ) + .run(finishedAt, id, REMOVING_SERVER); if (!result.changes) return false; if (pausePolicy) pauseScalingForOperation(id); - recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', - message: 'Queued removal cancelled. The server and its files were kept.', - details: { state: 'cancelled', finishedAt } } }); + recordProgress(id, { + blocker: null, + checks: [], + event: { + kind: 'outcome', + message: 'Queued removal cancelled. The server and its files were kept.', + details: { state: 'cancelled', finishedAt } + } + }); return true; })(); } export function canResumeCleanup(id: string): boolean { - return !!db().prepare(`SELECT d.id FROM server_deployments d + return !!db() + .prepare( + `SELECT d.id FROM server_deployments d WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server)) AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ? - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)` + ) .get(id, REMOVING_SERVER); } @@ -192,16 +400,30 @@ export function resumeCleanup(id: string): boolean { return db().transaction(() => { if (!canResumeCleanup(id)) return false; preserveOutcome(id); - db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id); - recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null, - event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } }); + db() + .prepare( + "UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?" + ) + .run(id); + recordProgress(id, { + phase: 'cleanup', + blocker: null, + checks: [], + observedAt: null, + event: { + kind: 'resumed', + message: 'Resumed local cleanup after confirmed instance removal' + } + }); return true; })(); } export function canResumeVerification(id: string): boolean { ensureHostSchema(db()); - return !!db().prepare(`SELECT d.id FROM server_deployments d + return !!db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL LEFT JOIN native_provisioning n ON n.deployment_id=d.id @@ -215,14 +437,17 @@ export function canResumeVerification(id: string): boolean { OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1)))) OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS ( SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL)))) - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)` + ) .get(id); } /** A conclusively failed initial create can enter the normal rebuild flow. */ export function canRetryWithRebuild(id: string): boolean { ensureHostSchema(db()); - return !!db().prepare(`SELECT d.id FROM server_deployments d + return !!db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id=d.id JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL @@ -233,14 +458,18 @@ export function canRetryWithRebuild(id: string): boolean { WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed' AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id) - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)` + ) .get(id); } export function canResumeRebuild(id: string): boolean { - return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id + return !!db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)` + ) .get(id, REMOVING_SERVER); } @@ -248,9 +477,21 @@ export function resumeRebuild(id: string): boolean { return db().transaction(() => { if (!canResumeRebuild(id)) return false; preserveOutcome(id); - db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id); - recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null, - event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } }); + db() + .prepare( + "UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?" + ) + .run(id); + recordProgress(id, { + phase: 'cleanup', + blocker: null, + checks: [], + observedAt: null, + event: { + kind: 'resumed', + message: 'Resumed rebuild cleanup after confirmed instance removal' + } + }); return true; })(); } @@ -260,46 +501,118 @@ export function resumeVerification(id: string): boolean { return db().transaction(() => { if (!canResumeVerification(id)) return false; preserveOutcome(id); - db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id); - deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null, - event: { kind: 'resumed', message: 'Resumed verification of the existing server' } }); + db() + .prepare( + "UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?" + ) + .run(id); + deploymentStage(id, 'Waiting for host agent health', { + phase: 'verify-container', + blocker: null, + checks: [], + observedAt: null, + event: { kind: 'resumed', message: 'Resumed verification of the existing server' } + }); return true; })(); } -export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void { +export function finishDeployment( + id: string, + result: { ok: boolean; message?: string; warnings?: string[] } +): void { // Creation failure cleanup deletes its own registry row, whose history trigger // marks it cancelled. Preserve that real failure; explicit cancellation stays terminal. - if (provisioningCancelled(id) && !db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'").get(id)) return; + if ( + provisioningCancelled(id) && + !db() + .prepare( + "SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'" + ) + .get(id) + ) + return; const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done'; db().transaction(() => { const finishedAt = new Date().toISOString(); - db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`) - .run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id); - for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } }); - recordProgress(id, { blocker: null, event: { kind: 'outcome', - message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`, - details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } }); + db() + .prepare( + `UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?` + ) + .run( + state, + finishedAt, + result.message ?? null, + JSON.stringify(result.warnings ?? []), + id + ); + for (const warning of result.warnings ?? []) + recordProgress(id, { event: { kind: 'warning', message: warning } }); + recordProgress(id, { + blocker: null, + event: { + kind: 'outcome', + message: result.ok + ? state === 'warning' + ? 'Completed with warnings' + : 'Completed' + : `Failed: ${result.message ?? 'Unknown error'}`, + details: { + state, + finishedAt, + error: result.message ?? null, + warnings: result.warnings ?? [] + } + } + }); })(); } /** Legacy failures also need a durable structured result before same-ID resume clears it. */ function preserveOutcome(id: string): void { - const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as - { state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined; - if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return; - const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1") + const row = db() + .prepare( + 'SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?' + ) + .get(id) as + | { + state: string; + finished_at: string | null; + error: string | null; + warnings: string; + observation_json: string | null; + } + | undefined; + if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) + return; + const previous = db() + .prepare( + "SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1" + ) .get(id) as { details_json: string | null } | undefined; try { const details = JSON.parse(previous?.details_json ?? 'null'); if (details?.state === row.state && details?.finishedAt === row.finished_at) return; - } catch { /* Preserve the row's known outcome when old event metadata is missing. */ } - recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome', - message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`, - details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } }); + } catch { + /* Preserve the row's known outcome when old event metadata is missing. */ + } + recordProgress(id, { + observedAt: readProgress(row)?.observedAt ?? null, + event: { + kind: 'outcome', + message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`, + details: { + state: row.state, + finishedAt: row.finished_at, + error: row.error, + warnings: warningMessages(row.warnings) + } + } + }); } -const activeDeployment = "d.server_deleted_at IS NULL AND d.state IN ('queued', 'awaiting-health', 'running', 'waiting')"; +const activeDeployment = + "d.server_deleted_at IS NULL AND d.state IN ('queued', 'awaiting-health', 'running', 'waiting')"; const reviewDeployment = `d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS ( SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=d.id AND c.resolved_at >= COALESCE(d.finished_at,d.started_at))`; @@ -314,29 +627,53 @@ const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d. `; function deploymentListQuery(options: ProvisioningListOptions) { - const conditions = [options.view === 'active' ? `(${activeDeployment})` - : options.view === 'history' ? `NOT (${activeDeployment})` : '1']; + const conditions = [ + options.view === 'active' + ? `(${activeDeployment})` + : options.view === 'history' + ? `NOT (${activeDeployment})` + : '1' + ]; const parameters: string[] = []; - if (options.action) { conditions.push('d.action = ?'); parameters.push(options.action); } + if (options.action) { + conditions.push('d.action = ?'); + parameters.push(options.action); + } if (options.status === 'attention') conditions.push(reviewDeployment); else if (options.status === 'pending') conditions.push("d.state IN ('queued','waiting')"); - else if (options.status) { conditions.push('d.state=?'); parameters.push(options.status); } - const linked = (kinds: readonly string[]) => `(EXISTS (SELECT 1 FROM provisioning_sources src WHERE src.operation_id=d.id AND src.kind IN (${kinds.map(kind => `'${kind}'`).join(',')})) - OR ${kinds.map(kind => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`; + else if (options.status) { + conditions.push('d.state=?'); + parameters.push(options.status); + } + const linked = ( + kinds: readonly string[] + ) => `(EXISTS (SELECT 1 FROM provisioning_sources src WHERE src.operation_id=d.id AND src.kind IN (${kinds.map((kind) => `'${kind}'`).join(',')})) + OR ${kinds.map((kind) => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`; if (options.source === 'manual' || options.source === 'system') { conditions.push(`NOT ${linked(Object.values(provisioningSourceKinds).flat())}`); - conditions.push(options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'"); + conditions.push( + options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'" + ); } else if (options.source) conditions.push(linked(provisioningSourceKinds[options.source])); const search = (options.server ?? '').slice(0, 100).toLowerCase(); if (search) { // Read only distinct names to preserve friendly map/lobby searches, including deleted servers. - const names = (db().prepare('SELECT DISTINCT server FROM server_deployments').all() as { server: string }[]) - .map(row => row.server).filter(name => `${name} ${serverTitle(name)}`.toLowerCase().includes(search)); + const names = ( + db().prepare('SELECT DISTINCT server FROM server_deployments').all() as { + server: string; + }[] + ) + .map((row) => row.server) + .filter((name) => `${name} ${serverTitle(name)}`.toLowerCase().includes(search)); conditions.push('d.server IN (SELECT value FROM json_each(?))'); parameters.push(JSON.stringify(names)); } - return { where: conditions.join(' AND '), parameters, - limit: provisioningPageSize(options.limit ?? 25), offset: provisioningPageOffset(options.offset ?? 0) }; + return { + where: conditions.join(' AND '), + parameters, + limit: provisioningPageSize(options.limit ?? 25), + offset: provisioningPageOffset(options.offset ?? 0) + }; } function activeDeploymentRank(row: ProvisioningListRow): number { @@ -346,32 +683,72 @@ function activeDeploymentRank(row: ProvisioningListRow): number { // Launch preparation keeps its queued/waiting state until command submission. if (row.phase === 'prepare' && !row.waitCode) return 1; // Remote file staging currently records its progress only in the stage message. - if (row.waitCode === 'native-host' && ['Staging and verifying files on the host.', - 'Waiting for verified remote file staging.'].includes(row.stage)) return 1; + if ( + row.waitCode === 'native-host' && + [ + 'Staging and verifying files on the host.', + 'Waiting for verified remote file staging.' + ].includes(row.stage) + ) + return 1; if (group === 'queued') return 3; // A submitted command remains in progress while its agent result is pending. - if (row.state === 'running' && row.phase === 'container-action' && row.waitCode === 'native-host') return 1; + if ( + row.state === 'running' && + row.phase === 'container-action' && + row.waitCode === 'native-host' + ) + return 1; return 2; } -function deploymentListRows(query: ReturnType, active: boolean): ProvisioningListRow[] { - const rows = db().prepare(`SELECT ${deploymentListColumns} FROM server_deployments d WHERE ${query.where} - ORDER BY ${active ? 'd.started_at ASC, d.rowid ASC' : 'd.started_at DESC, d.rowid DESC LIMIT ? OFFSET ?'}`) - .all(REMOVING_SERVER, ...query.parameters, ...(active ? [] : [query.limit, query.offset])) as ProvisioningListRow[]; +function deploymentListRows( + query: ReturnType, + active: boolean +): ProvisioningListRow[] { + const rows = db() + .prepare( + `SELECT ${deploymentListColumns} FROM server_deployments d WHERE ${query.where} + ORDER BY ${active ? 'd.started_at ASC, d.rowid ASC' : 'd.started_at DESC, d.rowid DESC LIMIT ? OFFSET ?'}` + ) + .all( + REMOVING_SERVER, + ...query.parameters, + ...(active ? [] : [query.limit, query.offset]) + ) as ProvisioningListRow[]; if (!rows.length) return rows; // Stable sorting keeps request/insertion order within each progress group. - if (active) rows.sort((left, right) => activeDeploymentRank(left) - activeDeploymentRank(right)); - const links = db().prepare(`SELECT operation_id,kind,source_id FROM provisioning_sources - WHERE operation_id IN (SELECT value FROM json_each(?)) ORDER BY kind,source_id`).all(JSON.stringify(rows.map(row => row.id))) as - (Omit & { operation_id: string })[]; + if (active) + rows.sort((left, right) => activeDeploymentRank(left) - activeDeploymentRank(right)); + const links = db() + .prepare( + `SELECT operation_id,kind,source_id FROM provisioning_sources + WHERE operation_id IN (SELECT value FROM json_each(?)) ORDER BY kind,source_id` + ) + .all(JSON.stringify(rows.map((row) => row.id))) as (Omit< + ProvisioningSource, + 'label' | 'href' + > & { operation_id: string })[]; const byOperation = new Map[]>(); - for (const { operation_id, ...link } of links) byOperation.set(operation_id, [...(byOperation.get(operation_id) ?? []), link]); + for (const { operation_id, ...link } of links) + byOperation.set(operation_id, [...(byOperation.get(operation_id) ?? []), link]); const cache = new Map(); for (const row of rows) { const sources = byOperation.get(row.id) ?? []; - const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/); - if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] }); - row.sources = sources.map(source => { + const fallback = row.triggered_by.match( + /^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/ + ); + if ( + fallback && + !sources.some( + (source) => source.kind === fallback[1] && source.source_id === fallback[2] + ) + ) + sources.unshift({ + kind: fallback[1] as ProvisioningSource['kind'], + source_id: fallback[2] + }); + row.sources = sources.map((source) => { const key = `${source.kind}:${source.source_id}`; if (!cache.has(key)) cache.set(key, sourceDetails(source)); return cache.get(key)!; @@ -389,10 +766,20 @@ export function deploymentListPage(options: ProvisioningListOptions = {}): Provi return db().transaction(() => { const query = deploymentListQuery(options); const deployments = deploymentListRows(query, options.view === 'active'); - const total = options.view === 'active' ? deployments.length : (db().prepare( - `SELECT COUNT(*) AS total FROM server_deployments d WHERE ${query.where}` - ).get(...query.parameters) as { total: number }).total; - const { needsReview } = db().prepare(`SELECT COUNT(*) AS needsReview FROM server_deployments d WHERE ${reviewDeployment}`) + const total = + options.view === 'active' + ? deployments.length + : ( + db() + .prepare( + `SELECT COUNT(*) AS total FROM server_deployments d WHERE ${query.where}` + ) + .get(...query.parameters) as { total: number } + ).total; + const { needsReview } = db() + .prepare( + `SELECT COUNT(*) AS needsReview FROM server_deployments d WHERE ${reviewDeployment}` + ) .get() as { needsReview: number }; return { deployments, total, needsReview, limit: query.limit, offset: query.offset }; })(); @@ -400,12 +787,30 @@ export function deploymentListPage(options: ProvisioningListOptions = {}): Provi export function interruptDeployments(): void { db().transaction(() => { - const native = 'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents'; - const running = db().prepare(`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`).all() as { id: string }[]; + const native = + 'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents'; + const running = db() + .prepare( + `SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})` + ) + .all() as { id: string }[]; const finishedAt = new Date().toISOString(); - const error = 'The application restarted before this operation finished. Check the server before retrying.'; - db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`).run(finishedAt, error); - for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome', - message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } }); + const error = + 'The application restarted before this operation finished. Check the server before retrying.'; + db() + .prepare( + `UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})` + ) + .run(finishedAt, error); + for (const { id } of running) + recordProgress(id, { + blocker: null, + event: { + kind: 'outcome', + message: + 'Interrupted by application restart. Check the server before retrying.', + details: { state: 'interrupted', finishedAt, error } + } + }); })(); } diff --git a/src/lib/server/provisioning/host-drain.ts b/src/lib/server/provisioning/host-drain.ts index d20d550..fe77c8b 100644 --- a/src/lib/server/provisioning/host-drain.ts +++ b/src/lib/server/provisioning/host-drain.ts @@ -7,50 +7,107 @@ import { beginDeployment, deploymentStage } from './history.js'; import type { NativePlacement } from '../../hosts.js'; export { captureMoveRecipe } from './move-recipes.js'; -export { hostMigration, isRebalanceMigration, processMigrationRemoval, requestInstanceMove } from './migrations.js'; +export { + hostMigration, + isRebalanceMigration, + processMigrationRemoval, + requestInstanceMove +} from './migrations.js'; /** Drain requests survive controller restarts and never make a source slot available early. */ export function reconcileHostDrains(): void { - for (const host of listHosts().filter(host => host.mode === 'draining' && !host.decommission)) { + for (const host of listHosts().filter( + (host) => host.mode === 'draining' && !host.decommission + )) { db().transaction(() => { // Accepted launches that have no instance yet can be placed elsewhere. - const queued = db().prepare(`SELECT q.deployment_id,q.selection FROM native_creation_intents q + const queued = db() + .prepare( + `SELECT q.deployment_id,q.selection FROM native_creation_intents q JOIN server_deployments d ON d.id=q.deployment_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.state='queued' AND d.server_deleted_at IS NULL AND d.submitted_at IS NULL AND n.operation_id IS NULL AND n.stage_operation_id IS NULL - AND json_extract(q.selection,'$.hostId')=?`).all(host.id) as { deployment_id: string; selection: string }[]; + AND json_extract(q.selection,'$.hostId')=?` + ) + .all(host.id) as { deployment_id: string; selection: string }[]; for (const row of queued) { const selection: NativePlacement = JSON.parse(row.selection); - db().prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?') - .run(JSON.stringify({ libraryId: selection.libraryId, image: selection.image, automatic: true, network: {} }), row.deployment_id); + db() + .prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?') + .run( + JSON.stringify({ + libraryId: selection.libraryId, + image: selection.image, + automatic: true, + network: {} + }), + row.deployment_id + ); } - const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[]; + const instances = db() + .prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL') + .all(host.id) as NativeInstance[]; for (const instance of instances) { const server = getServer(instance.server); if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue; // An explicit cancellation withdraws this instance's current drain attempt. - const latest = db().prepare(`SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id + const latest = db() + .prepare( + `SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.instance_id=? AND m.source_host_id=? AND d.triggered_by='host-drain' - AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id, host.id) as { state: string } | undefined; + AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1` + ) + .get(instance.id, host.id) as { state: string } | undefined; if (latest?.state === 'cancelled') continue; - if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue; - const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image, - native: { libraryId: instance.library_id, image: instance.image } }; + if ( + db() + .prepare( + `SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted')` + ) + .get(instance.server) + ) + continue; + const recipe = { + ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), + image: instance.image, + native: { libraryId: instance.library_id, image: instance.image } + }; const id = beginDeployment(instance.server, 'rebuild', 'host-drain'); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); - db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)') + db() + .prepare( + 'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)' + ) .run(id, instance.server, 'zombie', recipe.mapSlug, recipe.expectedMap); const stored = recipeStorage(recipe); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary); - db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)').run(id, host.id, instance.id); - deploymentStage(id, 'Host draining; waiting for an empty lobby and destination capacity'); + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)' + ) + .run(id, stored.recipe, stored.recipeSummary); + db() + .prepare( + 'INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)' + ) + .run(id, host.id, instance.id); + deploymentStage( + id, + 'Host draining; waiting for an empty lobby and destination capacity' + ); } - const unfinished = db().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id - WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1`).get(host.id); - const ports = db().prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1').get(host.id); - if (!instances.length && !unfinished && !ports) updateHost(host.id, { mode: 'disabled' }); + const unfinished = db() + .prepare( + `SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id + WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1` + ) + .get(host.id); + const ports = db() + .prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1') + .get(host.id); + if (!instances.length && !unfinished && !ports) + updateHost(host.id, { mode: 'disabled' }); })(); } } diff --git a/src/lib/server/provisioning/index.ts b/src/lib/server/provisioning/index.ts index 1b11247..67a4a8e 100644 --- a/src/lib/server/provisioning/index.ts +++ b/src/lib/server/provisioning/index.ts @@ -1,8 +1,30 @@ // Request/observation boundary used by APIs, campaigns, beta tests, and schedules. // Execution is driven exclusively by the provisioning worker. -export { requestLifecycle, requestRetryWithRebuild, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js'; +export { + requestLifecycle, + requestRetryWithRebuild, + requestRestart, + requestWatchdogLifecycle, + escalateWatchdogRestart, + provisioningOperation, + cancelProvisioning, + releaseRestartRequest +} from './requests.js'; export type { LifecycleOperation, RestartSource } from './requests.js'; -export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js'; -export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js'; +export { + queueCreateServer, + destroyServer, + previewCreation, + creationQueue, + nextLobbyNumber +} from './instances.js'; +export { + getDeployment, + listDeployments, + resumeVerification, + resumeCleanup, + resumeRebuild, + cancelRemoval +} from './history.js'; export { deploymentActivity } from './activity.js'; export { restartSettling, restartRecoveryPending } from './restarts.js'; diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index a6281fd..a93bc73 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -5,17 +5,22 @@ import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import type { GameMode, ZombieGame } from '../fleet/map-aliases.js'; import { nativeInstance } from '../backends/native.js'; import { beginDeployment, deploymentStage, resumeCleanup } from './history.js'; -import { previewNativeCreation, queueNativeCreation, queueNativeRebuild, cancelQueuedNativeCreation } from './native.js'; +import { + previewNativeCreation, + queueNativeCreation, + queueNativeRebuild, + cancelQueuedNativeCreation +} from './native.js'; export type CreateServerFailure = | 'bad-request' | 'conflict' - | 'unconfigured' // native host configuration missing - | 'unknown-map' // slug isn't in the catalog - | 'no-slot' // all lobby slots for this map are taken - | 'no-port' // port range exhausted - | 'fs' // couldn't create host dir - | 'db'; // shouldn't happen, but wraps INSERT failures + | 'unconfigured' // native host configuration missing + | 'unknown-map' // slug isn't in the catalog + | 'no-slot' // all lobby slots for this map are taken + | 'no-port' // port range exhausted + | 'fs' // couldn't create host dir + | 'db'; // shouldn't happen, but wraps INSERT failures export type CreateServerResult = | { ok: true; name: string; port: number | null; deploymentId: string } @@ -23,7 +28,12 @@ export type CreateServerResult = export type DestroyServerResult = | { ok: true; warnings: string[] } - | { ok: false; kind: 'not-found' | 'conflict' | 'fs' | 'db'; message: string; warnings: string[] }; + | { + ok: false; + kind: 'not-found' | 'conflict' | 'fs' | 'db'; + message: string; + warnings: string[]; + }; export interface CreateServerInput { hostId?: string; @@ -32,26 +42,28 @@ export interface CreateServerInput { bindAddress?: string; publicAddress?: string; publicPort?: number; - game?: ZombieGame; // omitted by older clients: T5 - gameMode: GameMode; // 'zombie' — only value supported today - mapSlug: string; // 'cosmodrome' - plutoKey?: string; // added to the game pool and reserved for this launch when supplied + game?: ZombieGame; // omitted by older clients: T5 + gameMode: GameMode; // 'zombie' — only value supported today + mapSlug: string; // 'cosmodrome' + plutoKey?: string; // added to the game pool and reserved for this launch when supplied expectedName?: string; // optional preview guard; never selects a lobby previewId?: string; - createdBy?: string; // 'ui' by default + createdBy?: string; // 'ui' by default } export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null { const rows = db() - .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION + .prepare( + `SELECT name FROM servers WHERE name LIKE ? UNION SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION SELECT server_id AS name FROM server_key_leases UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`) + WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))` + ) .all(`${baseName}_%`) as Array<{ name: string }>; const taken = new Set(); const prefix = `${baseName}_`; - for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) { + for (const r of [...rows, ...additionalNames.map((name) => ({ name }))]) { if (!r.name.startsWith(prefix)) continue; const tail = r.name.slice(prefix.length); if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10)); @@ -62,41 +74,94 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = [] return null; } -export function previewCreation(input: Parameters[0]) { return previewNativeCreation(input); } -export function queueCreateServer(input: CreateServerInput): CreateServerResult { return queueNativeCreation(input); } -export const queueRebuildServer = (name: string, triggeredBy = 'ui') => queueNativeRebuild(name, triggeredBy); +export function previewCreation(input: Parameters[0]) { + return previewNativeCreation(input); +} +export function queueCreateServer(input: CreateServerInput): CreateServerResult { + return queueNativeCreation(input); +} +export const queueRebuildServer = (name: string, triggeredBy = 'ui') => + queueNativeRebuild(name, triggeredBy); // Persist the request before returning so navigation and client disconnects cannot lose it. function hasActiveCreation(name: string): boolean { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name); + return !!db() + .prepare( + "SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')" + ) + .get(name); } -export function destroyServer(name: string, triggeredBy = 'ui', autoscalingDecisionId?: string): DestroyServerResult & { deploymentId?: string } { +export function destroyServer( + name: string, + triggeredBy = 'ui', + autoscalingDecisionId?: string +): DestroyServerResult & { deploymentId?: string } { const owned = scalingOwnershipBlocker(name); if (autoscalingDecisionId) { const identity = mapIdentity(getServer(name) ?? { name }); - if (!identity) return { ok: false, kind: 'conflict', message: 'Map identity unavailable.', warnings: [] }; + if (!identity) + return { + ok: false, + kind: 'conflict', + message: 'Map identity unavailable.', + warnings: [] + }; assertScalingOwnership(identity.game, identity.mapSlug, autoscalingDecisionId); } else if (owned) return { ok: false, kind: 'conflict', message: owned, warnings: [] }; const cancelled = cancelQueuedNativeCreation(name, triggeredBy); if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled }; const blocker = activeBetaRemovalBlocker(name); if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; - if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') - AND state IN ('waiting','queued','running')`).get(name)) { - return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] }; + if ( + db() + .prepare( + `SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') + AND state IN ('waiting','queued','running')` + ) + .get(name) + ) { + return { + ok: false, + kind: 'conflict', + message: 'Wait for the current provisioning operation before destroying this server.', + warnings: [] + }; } - const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined; + const existing = db() + .prepare( + "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')" + ) + .get(name) as { id: string } | undefined; if (existing) return { ok: true, warnings: [], deploymentId: existing.id }; - const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined; - if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id }; - if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; - if (!nativeInstance(name)) return { ok: false, kind: 'conflict', message: 'Server has no native host instance.', warnings: [] }; + const previous = db() + .prepare( + 'SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1' + ) + .get(name) as { id: string } | undefined; + if (previous && resumeCleanup(previous.id)) + return { ok: true, warnings: [], deploymentId: previous.id }; + if (!getServer(name) && !hasActiveCreation(name)) + return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; + if (!nativeInstance(name)) + return { + ok: false, + kind: 'conflict', + message: 'Server has no native host instance.', + warnings: [] + }; const operation = db().transaction(() => { const id = beginDeployment(name, 'destroy', triggeredBy); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, - blocker: { code: 'eligibility', message: 'Waiting for safe removal eligibility checks.', automatic: true } }); + deploymentStage(id, 'Waiting to verify the server is empty', { + phase: 'eligibility', + observedAt: null, + blocker: { + code: 'eligibility', + message: 'Waiting for safe removal eligibility checks.', + automatic: true + } + }); return id; })(); return { ok: true, warnings: [], deploymentId: operation }; @@ -105,8 +170,15 @@ export function destroyServer(name: string, triggeredBy = 'ui', autoscalingDecis /** One host-wide FIFO, shared by collections and individual launches. */ export function creationQueue() { type Entry = { id: string; server: string; state: string; stage: string }; - const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments + const jobs = db() + .prepare( + `SELECT id, server, state, stage FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health') - ORDER BY started_at, rowid`).all() as Entry[]; - return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') }; + ORDER BY started_at, rowid` + ) + .all() as Entry[]; + return { + active: jobs.filter((job) => job.state !== 'queued'), + queued: jobs.filter((job) => job.state === 'queued') + }; } diff --git a/src/lib/server/provisioning/launch-preparation.ts b/src/lib/server/provisioning/launch-preparation.ts index 2a2ea6c..a9f9f6a 100644 --- a/src/lib/server/provisioning/launch-preparation.ts +++ b/src/lib/server/provisioning/launch-preparation.ts @@ -30,50 +30,83 @@ export interface PreparedNativeLaunch { let worker: Worker | null = null; let sequence = 0; -const pending = new Map void; - reject: (error: Error) => void; - timer: NodeJS.Timeout; -}>(); +const pending = new Map< + number, + { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + } +>(); function stop(error: Error, current = worker): Promise | undefined { if (current !== worker) return; worker = null; - for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); } + for (const request of pending.values()) { + clearTimeout(request.timer); + request.reject(error); + } pending.clear(); return current?.terminate(); } /** The reader and JSON encoder run off the controller's request thread. */ -function run(kind: 'launch' | 'placement', input: NativeLaunchInput | PlacementOptions): Promise { +function run( + kind: 'launch' | 'placement', + input: NativeLaunchInput | PlacementOptions +): Promise { db(); // Migrations must finish before the independent read-only connection opens. if (!worker) { - const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production' - ? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), { - workerData: { dbPath: resolve(config.dbPath) } + const current = (worker = new Worker( + resolve( + process.env.NODE_ENV === 'production' + ? 'build/launch/worker.mjs' + : '.svelte-kit/launch/worker.mjs' + ), + { + workerData: { dbPath: resolve(config.dbPath) } + } + )); + current.on( + 'message', + ({ id, value, error }: { id: number; value: unknown; error?: string }) => { + if (current !== worker) return; + const request = pending.get(id); + if (!request) return; + pending.delete(id); + clearTimeout(request.timer); + if (error) request.reject(new Error(error)); + else request.resolve(value); + if (!pending.size) current.unref(); + } + ); + current.on('error', (error) => { + void stop(error, current); }); - current.on('message', ({ id, value, error }: { id: number; value: unknown; error?: string }) => { - if (current !== worker) return; - const request = pending.get(id); - if (!request) return; - pending.delete(id); clearTimeout(request.timer); - if (error) request.reject(new Error(error)); else request.resolve(value); - if (!pending.size) current.unref(); + current.on('exit', () => { + void stop( + new Error('Launch preparation worker stopped; preparation will retry.'), + current + ); }); - current.on('error', error => { void stop(error, current); }); - current.on('exit', () => { void stop(new Error('Launch preparation worker stopped; preparation will retry.'), current); }); } worker.ref(); const id = ++sequence; return new Promise((resolve, reject) => { - const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000); - pending.set(id, { resolve: value => resolve(value as T), reject, timer }); + const timer = setTimeout(() => { + void stop(new Error('Launch preparation timed out; preparation will retry.')); + }, 60_000); + pending.set(id, { resolve: (value) => resolve(value as T), reject, timer }); worker!.postMessage({ id, kind, input }); }); } -export function prepareNativeLaunch(input: NativeLaunchInput): Promise { return run('launch', input); } -export function evaluateNativePlacement(input: PlacementOptions): Promise<{ hostId: string; lastHeartbeat: string | null }> { +export function prepareNativeLaunch(input: NativeLaunchInput): Promise { + return run('launch', input); +} +export function evaluateNativePlacement( + input: PlacementOptions +): Promise<{ hostId: string; lastHeartbeat: string | null }> { return run('placement', input); } diff --git a/src/lib/server/provisioning/launch-worker.ts b/src/lib/server/provisioning/launch-worker.ts index b2e4b58..ede2052 100644 --- a/src/lib/server/provisioning/launch-worker.ts +++ b/src/lib/server/provisioning/launch-worker.ts @@ -2,64 +2,143 @@ import { parentPort } from 'node:worker_threads'; import { posix } from 'node:path'; import { db } from '../db/index.js'; import { relDestForKind } from '../config.js'; -import { captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; +import { + captureRecipe, + recipeForDeployment, + summarizeRecipe, + type LaunchRecipe +} from '../fleet/launch-recipes.js'; import { sha256Buffer } from '../fleet/files.js'; import type { NativeLaunchInput } from './launch-preparation.js'; import { choosePlacement, type PlacementOptions } from '../hosts/capacity.js'; -parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeLaunchInput } | { id: number; kind: 'placement'; input: PlacementOptions }) => { - const { id } = message; - try { - if (message.kind === 'placement') { - const decision = db().transaction(() => choosePlacement(message.input))(); - parentPort!.postMessage({ id, value: { hostId: decision.host.id, lastHeartbeat: decision.host.lastHeartbeat } }); - return; +parentPort!.on( + 'message', + ( + message: + | { id: number; kind: 'launch'; input: NativeLaunchInput } + | { id: number; kind: 'placement'; input: PlacementOptions } + ) => { + const { id } = message; + try { + if (message.kind === 'placement') { + const decision = db().transaction(() => choosePlacement(message.input))(); + parentPort!.postMessage({ + id, + value: { hostId: decision.host.id, lastHeartbeat: decision.host.lastHeartbeat } + }); + return; + } + const input = message.input; + // All file versions and composed configuration belong to one snapshot. + const { recipe, password } = db().transaction(() => { + if (!input.frozenRecipeId) + return { + recipe: { + ...captureRecipe(input.game, input.mapSlug, input.rconPassword), + native: input.native, + image: input.instance.image + }, + password: input.rconPassword + }; + const stored = recipeForDeployment(input.frozenRecipeId)?.recipe; + const migration = db() + .prepare( + 'SELECT runtime_password FROM ec_host_migrations WHERE deployment_id=?' + ) + .get(input.frozenRecipeId) as { runtime_password: string | null } | undefined; + if ( + !stored || + !migration?.runtime_password || + stored.files.some((file) => file.content === undefined) + ) { + throw new Error( + 'The preserved deployed recipe or runtime credentials are unavailable.' + ); + } + for (const file of stored.files) { + const bytes = Buffer.from(file.content!, 'base64'); + if (bytes.length !== file.size || sha256Buffer(bytes) !== file.sha256) + throw new Error('Preserved deployed file verification failed.'); + } + return { + recipe: { + ...stored, + native: input.native, + image: input.instance.image + } as LaunchRecipe, + password: migration.runtime_password + }; + })(); + const instance = input.instance; + const payload = { + name: instance.server, + game: recipe.game, + libraryId: instance.library_id, + image: instance.image, + port: instance.port, + bindAddress: instance.bind_address, + rconPassword: password, + env: { + PLUTAINER_SERVER_NAME: instance.server, + PLUTAINER_PORT: String(instance.port), + PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', + PLUTAINER_CONFIG_FILE: recipe.configFile, + PLUTAINER_MAP_ROTATE: 'false', + PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, + PLUTO_SERVER_KEY: input.serverKey + }, + files: recipe.files.map((file) => ({ + path: posix.join(relDestForKind(file.kind, recipe.game), file.path), + contentSha256: file.sha256, + sha256: file.sha256 + })), + ...(input.rebuild + ? { + nextGeneration: instance.generation + 1, + ...(input.safetyOverride ? { safetyOverride: true } : {}), + ...(input.unhealthyRecovery ? { unhealthyRecovery: true } : {}) + } + : {}) + }; + const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)); + const blobs = [ + ...new Map( + recipe.files.map((file) => [ + file.sha256, + { + sha256: file.sha256, + content: new Uint8Array(Buffer.from(file.content, 'base64')) + } + ]) + ).values() + ]; + const storedRecipe = { + ...recipe, + files: recipe.files.map(({ content, ...file }) => ({ + ...file, + contentHash: file.sha256 + })) + }; + const recipeJson = encode(storedRecipe); + const payloadJson = encode(payload); + const plan = { + ...recipe, + files: recipe.files.map(({ content: _content, ...file }) => file) + }; + const recipeSummary = JSON.stringify( + summarizeRecipe(plan, sha256Buffer(Buffer.from(JSON.stringify(recipe)))) + ); + // Transfer ownership of the encoded bytes; do not clone giant strings or objects. + parentPort!.postMessage( + { id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson, blobs } }, + [recipeJson.buffer, payloadJson.buffer, ...blobs.map((blob) => blob.content.buffer)] + ); + } catch (error) { + parentPort!.postMessage({ + id, + error: error instanceof Error ? error.message : 'Launch preparation failed.' + }); } - const input = message.input; - // All file versions and composed configuration belong to one snapshot. - const { recipe, password } = db().transaction(() => { - if (!input.frozenRecipeId) return { recipe: { ...captureRecipe(input.game, input.mapSlug, input.rconPassword), - native: input.native, image: input.instance.image }, password: input.rconPassword }; - const stored = recipeForDeployment(input.frozenRecipeId)?.recipe; - const migration = db().prepare('SELECT runtime_password FROM ec_host_migrations WHERE deployment_id=?') - .get(input.frozenRecipeId) as { runtime_password: string | null } | undefined; - if (!stored || !migration?.runtime_password || stored.files.some(file => file.content === undefined)) { - throw new Error('The preserved deployed recipe or runtime credentials are unavailable.'); - } - for (const file of stored.files) { - const bytes = Buffer.from(file.content!, 'base64'); - if (bytes.length !== file.size || sha256Buffer(bytes) !== file.sha256) throw new Error('Preserved deployed file verification failed.'); - } - return { recipe: { ...stored, native: input.native, image: input.instance.image } as LaunchRecipe, - password: migration.runtime_password }; - })(); - const instance = input.instance; - const payload = { - name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, - port: instance.port, bindAddress: instance.bind_address, rconPassword: password, - env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), - PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile, - PLUTAINER_MAP_ROTATE: 'false', - PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, - PLUTO_SERVER_KEY: input.serverKey }, - files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), - contentSha256: file.sha256, sha256: file.sha256 })), - ...(input.rebuild ? { nextGeneration: instance.generation + 1, - ...(input.safetyOverride ? { safetyOverride: true } : {}), - ...(input.unhealthyRecovery ? { unhealthyRecovery: true } : {}) } : {}) - }; - const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)); - const blobs = [...new Map(recipe.files.map(file => [file.sha256, - { sha256: file.sha256, content: new Uint8Array(Buffer.from(file.content, 'base64')) }])).values()]; - const storedRecipe = { ...recipe, files: recipe.files.map(({ content, ...file }) => ({ ...file, contentHash: file.sha256 })) }; - const recipeJson = encode(storedRecipe); - const payloadJson = encode(payload); - const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) }; - const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(JSON.stringify(recipe))))); - // Transfer ownership of the encoded bytes; do not clone giant strings or objects. - parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson, blobs } }, - [recipeJson.buffer, payloadJson.buffer, ...blobs.map(blob => blob.content.buffer)]); - } catch (error) { - parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' }); } -}); +); diff --git a/src/lib/server/provisioning/migrations.ts b/src/lib/server/provisioning/migrations.ts index 8398a5b..4754aa6 100644 --- a/src/lib/server/provisioning/migrations.ts +++ b/src/lib/server/provisioning/migrations.ts @@ -1,13 +1,28 @@ import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js'; import { db } from '../db/index.js'; import { provisioningCancelled } from './cancellation.js'; -import { nativeInstance, nativeRequest, runtimePasswordForServer, type NativeInstance } from '../backends/native.js'; -import { getHost, getHostOperationStatus, queueHostOperation, reserveHostPort } from '../hosts/index.js'; +import { + nativeInstance, + nativeRequest, + runtimePasswordForServer, + type NativeInstance +} from '../backends/native.js'; +import { + getHost, + getHostOperationStatus, + queueHostOperation, + reserveHostPort +} from '../hosts/index.js'; import { choosePlacement } from '../hosts/capacity.js'; import { scheduleAvailabilityRefresh } from '../hosts/availability.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { getServerKey } from '../fleet/server-keys.js'; -import { recipeForDeployment, recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; +import { + recipeForDeployment, + recipeStorage, + recipeSummaryForDeployment, + type LaunchRecipePlan +} from '../fleet/launch-recipes.js'; import { captureMoveRecipe, moveRecipeMatches, moveRecipeStorage } from './move-recipes.js'; import { rebalanceAdmission } from '../rebalancing/admission.js'; import { probePlayerEvidence } from '../fleet/status-evidence.js'; @@ -16,168 +31,394 @@ import { fleetSettings } from '../settings.js'; import type { NativePlacement } from '../../hosts.js'; interface Migration { - deployment_id: string; source_host_id: string; instance_id: string; - target_host_id: string | null; target_port: number | null; target_bind: string | null; - remove_operation_id: string | null; removed_at: string | null; + deployment_id: string; + source_host_id: string; + instance_id: string; + target_host_id: string | null; + target_port: number | null; + target_bind: string | null; + remove_operation_id: string | null; + removed_at: string | null; +} +interface Job { + id: string; + server: string; + action: string; + state: string; + stage: string; } -interface Job { id: string; server: string; action: string; state: string; stage: string } type Recipe = LaunchRecipePlan & { native: NativePlacement }; export { captureMoveRecipe } from './move-recipes.js'; export function isRebalanceMigration(id: string): boolean { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id); + return !!db() + .prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'") + .get(id); } /** Queue a move to one explicit host; its source remains active throughout. */ -export function requestInstanceMove(serverName: string, targetHostId: string, - options: { triggeredBy: string; recipe?: LaunchRecipePlan }): string { +export function requestInstanceMove( + serverName: string, + targetHostId: string, + options: { triggeredBy: string; recipe?: LaunchRecipePlan } +): string { return db().transaction(() => { const instance = nativeInstance(serverName); const server = getServer(serverName); if (!instance || !server?.map_slug) throw new Error('The native instance is unavailable.'); - if (instance.host_id === targetHostId) throw new Error('Choose a different destination host.'); - if (getHost(instance.host_id)?.mode !== 'active') throw new Error('The source host must be active.'); - if (!options.triggeredBy.startsWith('rebalance:')) throw new Error('An instance move requires a rebalance run.'); + if (instance.host_id === targetHostId) + throw new Error('Choose a different destination host.'); + if (getHost(instance.host_id)?.mode !== 'active') + throw new Error('The source host must be active.'); + if (!options.triggeredBy.startsWith('rebalance:')) + throw new Error('An instance move requires a rebalance run.'); const blocker = activeBetaRemovalBlocker(serverName); if (blocker) throw new Error(blocker); - if (!getServerKey(serverName)) throw new Error('Save a server key before moving this instance.'); - if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(serverName)) { + if (!getServerKey(serverName)) + throw new Error('Save a server key before moving this instance.'); + if ( + db() + .prepare( + `SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted')` + ) + .get(serverName) + ) { throw new Error('Another provisioning operation is active for this instance.'); } const recipe = options.recipe ?? captureMoveRecipe(serverName); - if (recipe.game !== server.game || recipe.mapSlug !== server.map_slug || recipe.image !== instance.image - || !moveRecipeMatches(serverName, recipe)) throw new Error('The deployed files changed; generate a fresh rebalance preview.'); + if ( + recipe.game !== server.game || + recipe.mapSlug !== server.map_slug || + recipe.image !== instance.image || + !moveRecipeMatches(serverName, recipe) + ) + throw new Error('The deployed files changed; generate a fresh rebalance preview.'); const password = runtimePasswordForServer(serverName); if (!password) throw new Error('The deployed runtime credentials are unavailable.'); - choosePlacement({ hostId: targetHostId, excludeHostIds: [instance.host_id], libraryId: instance.library_id, - image: instance.image, thorough: true, dispatch: true }); + choosePlacement({ + hostId: targetHostId, + excludeHostIds: [instance.host_id], + libraryId: instance.library_id, + image: instance.image, + thorough: true, + dispatch: true + }); const id = beginDeployment(serverName, 'rebuild', options.triggeredBy); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); - db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)') + db() + .prepare( + 'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)' + ) .run(id, serverName, 'zombie', recipe.mapSlug, recipe.expectedMap); - const stored = moveRecipeStorage({ ...recipe, native: { libraryId: instance.library_id, image: instance.image, hostId: targetHostId } }); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary); - db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,runtime_password) VALUES(?,?,?,?)') + const stored = moveRecipeStorage({ + ...recipe, + native: { libraryId: instance.library_id, image: instance.image, hostId: targetHostId } + }); + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)' + ) + .run(id, stored.recipe, stored.recipeSummary); + db() + .prepare( + 'INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,runtime_password) VALUES(?,?,?,?)' + ) .run(id, instance.host_id, instance.id, password); - deploymentStage(id, 'Rebalance queued; waiting to confirm an empty lobby and destination capacity', { phase: 'eligibility', recipeStatus: 'queued' }); + deploymentStage( + id, + 'Rebalance queued; waiting to confirm an empty lobby and destination capacity', + { phase: 'eligibility', recipeStatus: 'queued' } + ); return id; })(); } export function hostMigration(id: string): Migration | undefined { - return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined; + return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as + Migration | undefined; } /** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */ -export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration, - eligibility: () => string | null): Promise { +export async function processMigrationRemoval( + job: Job, + instance: NativeInstance, + migration: Migration, + eligibility: () => string | null +): Promise { if (provisioningCancelled(job.id)) return; - const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }; + const hold = (message: string) => { + if (!provisioningCancelled(job.id)) + deploymentStage(job.id, message, { + blocker: { code: 'native-host', message, automatic: true } + }); + }; if (migration.remove_operation_id) { const operation = getHostOperationStatus(migration.remove_operation_id); - if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) { + if ( + operation?.state === 'failed' && + Date.now() - Date.parse(operation.updatedAt) >= 60_000 + ) { // A failed receipt certifies no mutation began. Unknown results keep // their reservation and original operation ID until reconciled. db().transaction(() => { - db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id); - db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL, - remove_operation_id=NULL WHERE deployment_id=?`).run(job.id); - db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); + db() + .prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?') + .run(migration.target_host_id, instance.id); + db() + .prepare( + `UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL, + remove_operation_id=NULL WHERE deployment_id=?` + ) + .run(job.id); + db() + .prepare( + "UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?" + ) + .run(job.id); scheduleAvailabilityRefresh(); })(); - hold('Source removal failed before mutation; checking destination capacity again before retrying.'); + hold( + 'Source removal failed before mutation; checking destination capacity again before retrying.' + ); return; } if (operation?.state !== 'completed') { - hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.'); + hold( + operation?.error ?? + 'Waiting for confirmed source removal. The original agent operation will reconcile.' + ); return; } db().transaction(() => { if (hostMigration(job.id)?.removed_at) return; const current = nativeInstance(job.server); - if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.'); + if ( + current?.id !== instance.id || + current.generation !== instance.generation || + current.host_id !== migration.source_host_id + ) + throw new Error('Migration source changed.'); const target = getHost(migration.target_host_id!); if (!target) throw new Error('Migration destination is unavailable.'); const rebalance = isRebalanceMigration(job.id); - const recipe = (rebalance ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe } - : recipeForDeployment(job.id)!.recipe) as Recipe; - recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id, - bindAddress: migration.target_bind!, publicAddress: target.publicAddress }; + const recipe = ( + rebalance + ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe } + : recipeForDeployment(job.id)!.recipe + ) as Recipe; + recipe.native = { + libraryId: instance.library_id, + image: instance.image, + hostId: target.id, + bindAddress: migration.target_bind!, + publicAddress: target.publicAddress + }; const stored = rebalance ? moveRecipeStorage(recipe) : recipeStorage(recipe); - db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id); - db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id); - db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`) - .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id); - db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id); - db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id); - db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL, + db() + .prepare( + 'UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?' + ) + .run(stored.recipe, stored.recipeSummary, job.id); + db() + .prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?') + .run(instance.host_id, instance.id); + db() + .prepare( + `UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?` + ) + .run( + target.id, + migration.target_bind, + migration.target_port, + target.publicAddress, + instance.id + ); + db() + .prepare( + 'UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?' + ) + .run(job.id); + db() + .prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?') + .run(migration.target_port, job.id); + db() + .prepare( + `UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL, last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL, - game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server); - db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id); - db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); - deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null }); + game_started_at=NULL,game_ended_at=NULL WHERE name=?` + ) + .run(migration.target_port, job.server); + db() + .prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?') + .run(operation.updatedAt, job.id); + db() + .prepare( + "UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?" + ) + .run(job.id); + deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { + phase: 'startup-queue', + blocker: null + }); scheduleAvailabilityRefresh(); })(); return; } - if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; } - const reason = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server); - if (reason) { hold(reason); return; } - if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; } + if (!getHost(instance.host_id)?.connected) { + hold('Waiting for the source host to reconnect.'); + return; + } + const reason = + rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server); + if (reason) { + hold(reason); + return; + } + if (!getServerKey(job.server)) { + hold('Save a server key before migrating this instance.'); + return; + } const rebalance = isRebalanceMigration(job.id); const recipe = rebalance ? recipeSummaryForDeployment(job.id)!.recipe : null; if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) { - hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return; + hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); + return; } - const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true, - excludeHostIds: [instance.host_id], ...(rebalance ? { hostId: recipe!.native!.hostId } : {}) }; + const options = { + libraryId: instance.library_id, + image: instance.image, + thorough: true, + dispatch: true, + excludeHostIds: [instance.host_id], + ...(rebalance ? { hostId: recipe!.native!.hostId } : {}) + }; // No source mutation is submitted unless another host can take this instance. choosePlacement(options); - const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect'); + const health = await nativeRequest<{ state: string; healthy?: boolean }>( + instance, + 'instance.inspect' + ); if (provisioningCancelled(job.id)) return; if (rebalance && (health.state !== 'running' || health.healthy !== true)) { - hold('The source lobby is no longer healthy and running. Waiting for current health evidence.'); return; + hold( + 'The source lobby is no longer healthy and running. Waiting for current health evidence.' + ); + return; } - if (!['stopped','exited','created','dead','missing'].includes(health.state)) { + if (!['stopped', 'exited', 'created', 'dead', 'missing'].includes(health.state)) { const occupancy = await probePlayerEvidence(job.server); if (provisioningCancelled(job.id)) return; - if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; } + if (occupancy.players !== 0) { + hold( + occupancy.players === null + ? 'Waiting for a fresh local player count.' + : `Waiting for ${occupancy.players} player(s) to leave.` + ); + return; + } } db().transaction(() => { - const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; - if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return; + const state = db() + .prepare( + 'SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL' + ) + .get(job.id) as { state: string } | undefined; + if ( + !state || + !['waiting', 'queued', 'interrupted'].includes(state.state) || + hostMigration(job.id)?.remove_operation_id + ) + return; const current = nativeInstance(job.server); - if (current?.id !== instance.id || current.generation !== instance.generation - || getHost(instance.host_id)?.mode !== (rebalance ? 'active' : 'draining')) return; - const blocker = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server); - if (blocker) { hold(blocker); return; } + if ( + current?.id !== instance.id || + current.generation !== instance.generation || + getHost(instance.host_id)?.mode !== (rebalance ? 'active' : 'draining') + ) + return; + const blocker = + rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server); + if (blocker) { + hold(blocker); + return; + } if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) { - hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return; + hold( + 'The deployed files changed; stop this run and generate a fresh rebalance preview.' + ); + return; } if (rebalance) { - const startups = db().prepare(`SELECT COUNT(*) AS n FROM ( + const startups = db() + .prepare( + `SELECT COUNT(*) AS n FROM ( SELECT id FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup}) - UNION SELECT value FROM json_each(?) WHERE value!=?)`).get(job.id, JSON.stringify([...preparingJobs]), job.id) as { n: number }; - if (startups.n >= fleetSettings().createConcurrency) { hold('Waiting for startup capacity before moving this lobby.'); return; } + UNION SELECT value FROM json_each(?) WHERE value!=?)` + ) + .get(job.id, JSON.stringify([...preparingJobs]), job.id) as { n: number }; + if (startups.n >= fleetSettings().createConcurrency) { + hold('Waiting for startup capacity before moving this lobby.'); + return; + } + } + const removing = db() + .prepare( + "SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')" + ) + .get() as { n: number }; + if (removing.n >= fleetSettings().destroyConcurrency) { + hold('Waiting for destruction capacity.'); + return; } - const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number }; - if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; } const { host, port } = choosePlacement(options); - const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id); - if (capacityBlocker) { hold(capacityBlocker); return; } - reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' }); - reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' }); - const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations - WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`) - .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n; - const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, { - instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}` + const capacityBlocker = + concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? + concurrencyBlocker(job.id, 'rebuild', host.id); + if (capacityBlocker) { + hold(capacityBlocker); + return; + } + reserveHostPort(host.id, instance.id, { + port, + bindAddress: host.bindAddress, + protocol: 'udp' }); - db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?') + reserveHostPort(host.id, instance.id, { + port, + bindAddress: host.bindAddress, + protocol: 'tcp' + }); + const attempt = ( + db() + .prepare( + `SELECT COUNT(*) AS n FROM ec_host_operations + WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?` + ) + .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number } + ).n; + const operation = queueHostOperation( + instance.host_id, + 'instance.remove', + {}, + { + instanceId: instance.id, + generation: instance.generation, + idempotencyKey: `${job.id}:migrate-remove:${attempt}` + } + ); + db() + .prepare( + 'UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?' + ) .run(host.id, port, host.bindAddress, operation.id, job.id); - db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); - deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null }); + db() + .prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?") + .run(new Date().toISOString(), job.id); + deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { + phase: 'container-action', + blocker: null + }); })(); } diff --git a/src/lib/server/provisioning/move-recipes.ts b/src/lib/server/provisioning/move-recipes.ts index 093b98a..ecde12a 100644 --- a/src/lib/server/provisioning/move-recipes.ts +++ b/src/lib/server/provisioning/move-recipes.ts @@ -2,58 +2,112 @@ import { db } from '../db/index.js'; import { nativeInstance, runtimePasswordForServer } from '../backends/native.js'; import { getServer } from '../fleet/servers.js'; import { listServerScripts } from '../fleet/inventory.js'; -import { recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; +import { + recipeStorage, + recipeSummaryForDeployment, + type LaunchRecipePlan +} from '../fleet/launch-recipes.js'; /** Preserve the recorded deployed inventory, including changes made after launch. */ export function captureMoveRecipe(server: string): LaunchRecipePlan { const instance = nativeInstance(server); const row = getServer(server); - if (!instance || !row?.map_slug) throw new Error('A live native instance and saved map are required before moving.'); - if (!runtimePasswordForServer(server)) throw new Error('The deployed runtime credentials are unavailable.'); - const launch = db().prepare(`SELECT d.id FROM server_deployments d + if (!instance || !row?.map_slug) + throw new Error('A live native instance and saved map are required before moving.'); + if (!runtimePasswordForServer(server)) + throw new Error('The deployed runtime credentials are unavailable.'); + const launch = db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id JOIN server_launch_recipes r ON r.deployment_id=d.id WHERE n.instance_id=? AND d.state IN ('done','warning') AND d.server_deleted_at IS NULL - ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { id: string } | undefined; + ORDER BY d.rowid DESC LIMIT 1` + ) + .get(instance.id) as { id: string } | undefined; const previous = launch && recipeSummaryForDeployment(launch.id)?.recipe; if (!previous || previous.game !== row.game || previous.mapSlug !== row.map_slug) { throw new Error('The deployed launch recipe is unavailable or differs from the saved map.'); } const inventory = listServerScripts(server); if (!inventory.length) throw new Error('The deployed file inventory is unavailable.'); - const files = inventory.map(file => { - const before = previous.files.find(item => item.path === file.rel_path); - const version = file.script_version_id === null ? undefined : db().prepare(`SELECT v.version,v.kind,s.name - FROM script_versions v JOIN scripts s ON s.id=v.script_id WHERE v.id=?`).get(file.script_version_id) as - { version: number; kind: string; name: string } | undefined; - const kind = version?.kind === 'baseline' ? 'config' : version?.kind ?? before?.kind; - if (!['script', 'plugin', 'config'].includes(kind ?? '')) throw new Error(`The deployed file type is unavailable for ${file.rel_path}.`); - if (!db().prepare('SELECT 1 FROM file_blobs WHERE sha256=? AND size=?').get(file.sha256, file.size)) { + const files = inventory.map((file) => { + const before = previous.files.find((item) => item.path === file.rel_path); + const version = + file.script_version_id === null + ? undefined + : (db() + .prepare( + `SELECT v.version,v.kind,s.name + FROM script_versions v JOIN scripts s ON s.id=v.script_id WHERE v.id=?` + ) + .get(file.script_version_id) as + { version: number; kind: string; name: string } | undefined); + const kind = version?.kind === 'baseline' ? 'config' : (version?.kind ?? before?.kind); + if (!['script', 'plugin', 'config'].includes(kind ?? '')) + throw new Error(`The deployed file type is unavailable for ${file.rel_path}.`); + if ( + !db() + .prepare('SELECT 1 FROM file_blobs WHERE sha256=? AND size=?') + .get(file.sha256, file.size) + ) { throw new Error(`The deployed bytes are unavailable for ${file.rel_path}.`); } - return { name: version?.name ?? before?.name ?? file.rel_path, kind: kind as 'script' | 'plugin' | 'config', - path: file.rel_path, versionId: file.script_version_id, version: version?.version ?? null, - sha256: file.sha256, size: file.size }; + return { + name: version?.name ?? before?.name ?? file.rel_path, + kind: kind as 'script' | 'plugin' | 'config', + path: file.rel_path, + versionId: file.script_version_id, + version: version?.version ?? null, + sha256: file.sha256, + size: file.size + }; }); - if (!files.some(file => file.path === 'electric_cherry_map.cfg' && file.kind === 'config')) { + if (!files.some((file) => file.path === 'electric_cherry_map.cfg' && file.kind === 'config')) { throw new Error('The deployed map configuration is unavailable.'); } - return { format: 1, game: previous.game, mapSlug: previous.mapSlug, mapName: previous.mapName, - expectedMap: previous.expectedMap, configFile: previous.configFile, - configSource: 'Preserved deployed files', capturedAt: new Date().toISOString(), image: instance.image, - native: { libraryId: instance.library_id, image: instance.image }, files }; + return { + format: 1, + game: previous.game, + mapSlug: previous.mapSlug, + mapName: previous.mapName, + expectedMap: previous.expectedMap, + configFile: previous.configFile, + configSource: 'Preserved deployed files', + capturedAt: new Date().toISOString(), + image: instance.image, + native: { libraryId: instance.library_id, image: instance.image }, + files + }; } /** Immutable hash references keep deployed bytes alive without decoding plugins on the controller. */ export function moveRecipeStorage(recipe: LaunchRecipePlan): ReturnType { - const metadata = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) }; + const metadata = { + ...recipe, + files: recipe.files.map(({ content: _content, ...file }) => file) + }; const stored = recipeStorage(metadata); - return { recipeSummary: stored.recipeSummary, - recipe: JSON.stringify({ ...metadata, files: metadata.files.map(file => ({ ...file, contentHash: file.sha256 })) }) }; + return { + recipeSummary: stored.recipeSummary, + recipe: JSON.stringify({ + ...metadata, + files: metadata.files.map((file) => ({ ...file, contentHash: file.sha256 })) + }) + }; } export function moveRecipeMatches(server: string, recipe: LaunchRecipePlan): boolean { const inventory = listServerScripts(server); - return inventory.length === recipe.files.length && inventory.every(file => recipe.files.some(saved => - saved.path === file.rel_path && saved.sha256 === file.sha256 && saved.size === file.size)); + return ( + inventory.length === recipe.files.length && + inventory.every((file) => + recipe.files.some( + (saved) => + saved.path === file.rel_path && + saved.sha256 === file.sha256 && + saved.size === file.size + ) + ) + ); } diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index f309f83..63e7e13 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -2,7 +2,12 @@ import { assertScalingOwnership } from '../autoscaling/ownership.js'; import { removalAuthorization } from '../autoscaling/authorization.js'; import { scalingIntent } from '../autoscaling/admission.js'; import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js'; -import { concurrencyBlocker, preparingJobs, committedRebalanceStartup, lifecycleHolders } from './concurrency.js'; +import { + concurrencyBlocker, + preparingJobs, + committedRebalanceStartup, + lifecycleHolders +} from './concurrency.js'; import { isIP } from 'node:net'; import { setImmediate as yieldToController } from 'node:timers/promises'; import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; @@ -12,15 +17,52 @@ import { db } from '../db/index.js'; import { ensureHostSchema } from '../hosts/schema.js'; import { unsettledCancelledCommand } from './cancellation.js'; import { config, relDestForKind, serverDir } from '../config.js'; -import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js'; -import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort, type HostOperationStatus } from '../hosts/index.js'; +import { + nativeInstance, + nativeRequest, + unplacedNativeCreation, + type NativeInstance +} from '../backends/native.js'; +import { + getHost, + listHosts, + getHostOperationStatus, + queueHostOperation, + queueHostOperationSerialized, + releaseHostPorts, + reserveHostPort, + type HostOperationStatus +} from '../hosts/index.js'; import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js'; -import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeSummaryForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; -import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js'; -import { getServerKey, reserveServerKey, releaseServerKey, availableServerKeyCount } from '../fleet/server-keys.js'; +import { + blueprintRecipe, + captureRecipePlan, + recipeForDeployment, + recipeSummaryForDeployment, + recipeStorage, + summarizeRecipe, + type LaunchRecipePlan +} from '../fleet/launch-recipes.js'; +import { + prepareNativeLaunch, + evaluateNativePlacement, + type PreparedNativeLaunch +} from './launch-preparation.js'; +import { + getServerKey, + reserveServerKey, + releaseServerKey, + availableServerKeyCount +} from '../fleet/server-keys.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js'; -import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, cancelRemoval } from './history.js'; +import { + beginDeployment, + deploymentStage, + finishDeployment, + REMOVING_SERVER, + cancelRemoval +} from './history.js'; import { recordProgress, type CheckInput } from './progress.js'; import { trackActivity } from '../pipeline/activity.js'; import { fleetSettings } from '../settings.js'; @@ -43,54 +85,126 @@ import { runtimeLaunchSql } from '../fleet/runtime-config.js'; export type NativeSelection = import('../../hosts.js').NativePlacement; type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection }; -interface Job { selection?: string | null; id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; - instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; applied_at: string | null; - recovery_step: 'restart' | 'rebuild' | null } -interface FilePlan { id: number; path: string; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } +interface Job { + selection?: string | null; + id: string; + server: string; + action: ProvisioningAction; + state: string; + submitted_at: string | null; + stage: string; + instance_id: string; + generation: number; + stage_operation_id: string | null; + operation_id: string | null; + applied_at: string | null; + recovery_step: 'restart' | 'rebuild' | null; +} +interface FilePlan { + id: number; + path: string; + sha256: string | null; + checkBefore: boolean; + expectedBeforeSha: string | null; +} const unhealthyRecoveryCapability = 'instance.unhealthy-recovery'; /** Validate a launch request without selecting a host or consuming capacity. */ -export function nativeLaunchSelection(input: Record, game: string): NativeSelection { - if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.'); +export function nativeLaunchSelection( + input: Record, + game: string +): NativeSelection { + if (input.hostId !== undefined && typeof input.hostId !== 'string') + throw new Error('Invalid host.'); const libraryId = newestReadyLibraryId(game); - if (!libraryId) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); + if (!libraryId) + throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) { - throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`); + throw new Error( + `The current ${game.toUpperCase()} game files changed. Refresh the launch preview.` + ); } - const image = input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image; - if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); + const image = + input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image; + if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) + throw new Error('Invalid runtime image.'); const automatic = input.automatic === true || !input.hostId; if (!automatic) { const host = getHost(input.hostId as string); if (!host) throw new Error('The selected host was not found.'); - if (host.decommission) throw new Error('The selected host is being removed. Choose another host.'); + if (host.decommission) + throw new Error('The selected host is being removed. Choose another host.'); } // Only the controller-configured default may be prepared automatically. // Custom runtimes must already be present in trusted host inventory; a // launch request must never authorize a registry download. - if (image !== config.defaultRuntimeImage && !listHosts().some(host => - !host.decommission && (automatic ? host.mode === 'active' : host.id === input.hostId) - && host.inventory.runtimeImages?.includes(image))) { - throw new Error('Custom runtime images must be prepared by an admin from the host page before launching.'); + if ( + image !== config.defaultRuntimeImage && + !listHosts().some( + (host) => + !host.decommission && + (automatic ? host.mode === 'active' : host.id === input.hostId) && + host.inventory.runtimeImages?.includes(image) + ) + ) { + throw new Error( + 'Custom runtime images must be prepared by an admin from the host page before launching.' + ); } - const rawNetwork = input.automatic === true ? input.network ?? {} : input; - if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.'); + const rawNetwork = input.automatic === true ? (input.network ?? {}) : input; + if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) + throw new Error('Invalid network overrides.'); const network = rawNetwork as NativeSelection; const bindAddress = network.bindAddress || undefined; - if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.'); - if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.'); - if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.'); - return { hostId: automatic ? undefined : input.hostId as string, libraryId, image, automatic, - bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort, - network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } }; + if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) + throw new Error('Invalid bind address.'); + if ( + network.publicAddress !== undefined && + (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253) + ) + throw new Error('Invalid public address.'); + if ( + network.publicPort !== undefined && + (!Number.isInteger(network.publicPort) || + Number(network.publicPort) < 1 || + Number(network.publicPort) > 65535) + ) + throw new Error('Invalid public port.'); + return { + hostId: automatic ? undefined : (input.hostId as string), + libraryId, + image, + automatic, + bindAddress, + publicAddress: network.publicAddress, + publicPort: network.publicPort, + network: { + bindAddress, + publicAddress: network.publicAddress, + publicPort: network.publicPort + } + }; } -export function nativeSelection(input: Record, game: string, options: Pick = {}): NativeSelection { +export function nativeSelection( + input: Record, + game: string, + options: Pick = {} +): NativeSelection { const selection = nativeLaunchSelection(input, game); - const { host } = choosePlacement({ hostId: selection.hostId, libraryId: selection.libraryId, - image: selection.image!, bindAddress: selection.bindAddress, ...options }); - return { ...selection, hostId: host.id, bindAddress: selection.bindAddress ?? host.bindAddress, - publicAddress: selection.publicAddress ?? host.publicAddress }; + const { host } = choosePlacement({ + hostId: selection.hostId, + libraryId: selection.libraryId, + image: selection.image!, + bindAddress: selection.bindAddress, + ...options + }); + return { + ...selection, + hostId: host.id, + bindAddress: selection.bindAddress ?? host.bindAddress, + publicAddress: selection.publicAddress ?? host.publicAddress + }; } export function previewNativeCreation(input: Record): LaunchPreview { @@ -101,92 +215,219 @@ export function previewNativeCreation(input: Record): LaunchPre if (!previous) throw new Error('The recorded deployment is unavailable.'); recipe = captureRecipePlan(previous.recipe.game, previous.recipe.mapSlug); blueprintName = previous.blueprintName; - } else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); + } else if (typeof input.blueprintId === 'string') + ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); else { - if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); + if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') + throw new Error('Choose a supported game and map.'); recipe = captureRecipePlan(input.game, input.mapSlug); } const selection = nativeSelection(input, recipe.game); - recipe.native = selection; recipe.image = selection.image!; + recipe.native = selection; + recipe.image = selection.image!; const base = `${recipe.game}_zombie_${recipe.mapSlug}`; const lobby = nextLobbyNumber(base); const name = lobby === null ? null : `${base}_${lobby}`; const port = nextNativePort(selection); - const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); - db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)') + const id = randomUUID(); + const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); + db() + .prepare( + 'INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)' + ) .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); const checks: LaunchPreview['checks'] = [ - { label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId!)!.name}: capacity available.` }, - { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` }, - { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' }, - { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, + { + label: 'Host capacity', + state: 'pass', + detail: selection.automatic + ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' + : `${getHost(selection.hostId!)!.name}: capacity available.` + }, + { + label: 'Game files', + state: 'pass', + detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` + }, + { + label: 'Lobby', + state: name ? 'pass' : 'blocked', + detail: name ?? 'All lobby IDs for this map are reserved.' + }, + { + label: 'Host port', + state: port ? 'pass' : 'blocked', + detail: port + ? selection.automatic + ? 'UDP/TCP capacity available; port assigned at placement.' + : `${selection.bindAddress}:${port} / UDP and TCP` + : 'No available UDP/TCP port on this host.' + }, { label: 'Configuration', state: 'pass', detail: recipe.configSource } ]; - return { id, expiresAt, name, port, hasSavedKey: !!name && availableServerKeyCount(recipe.game) > 0, recipe: summarizeRecipe(recipe), blueprintName, checks, - ready: checks.every(check => check.state !== 'blocked') }; + return { + id, + expiresAt, + name, + port, + hasSavedKey: !!name && availableServerKeyCount(recipe.game) > 0, + recipe: summarizeRecipe(recipe), + blueprintName, + checks, + ready: checks.every((check) => check.state !== 'blocked') + }; } export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null { const host = selection.hostId ? getHost(selection.hostId) : null; - return host ? availableHostPort(host, { bindAddress: selection.bindAddress, - planned: taken.map(port => ({ hostId: host.id, port })) }) : null; + return host + ? availableHostPort(host, { + bindAddress: selection.bindAddress, + planned: taken.map((port) => ({ hostId: host.id, port })) + }) + : null; } export function queueNativeCreation(input: CreateServerInput): CreateServerResult { try { return db().transaction(() => { - const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as - { id: string; recipe: string; server: string; port: number | null; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined; + const preview = input.previewId + ? (db() + .prepare('SELECT * FROM server_launch_previews WHERE id=?') + .get(input.previewId) as + | { + id: string; + recipe: string; + server: string; + port: number | null; + expires_at: string; + deployment_id: string | null; + blueprint_name: string | null; + } + | undefined) + : undefined; if (input.previewId && !preview) throw new Error('Review a fresh launch preview.'); - if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id }; - if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.'); - const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined; + if (preview?.deployment_id) + return { + ok: true as const, + name: preview.server, + port: preview.port, + deploymentId: preview.deployment_id + }; + if (preview && preview.expires_at <= new Date().toISOString()) + throw new Error('Launch preview expired.'); + const saved = preview ? (JSON.parse(preview.recipe) as NativeRecipe) : undefined; const game = input.game ?? 't5'; - if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.'); - const selection = nativeLaunchSelection((saved?.native ?? input) as unknown as Record, game); + if ( + input.gameMode !== 'zombie' || + (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug)) + ) + throw new Error('Selection changed. Review a fresh launch preview.'); + const selection = nativeLaunchSelection( + (saved?.native ?? input) as unknown as Record, + game + ); const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`); if (lobby === null) throw new Error('All lobby IDs are reserved.'); const name = `${game}_zombie_${input.mapSlug}_${lobby}`; - if ((input.expectedName && input.expectedName !== name) || (preview && preview.server !== name)) throw new Error('Available lobby changed. Review a fresh preview.'); - const result = queueNativeCreationIntent({ server: name, game, mapSlug: input.mapSlug, selection, - plutoKey: input.plutoKey, createdBy: input.createdBy, blueprintName: preview?.blueprint_name, - recipe: saved ?? captureRecipePlan(game, input.mapSlug) }); - if (result.ok && preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=NULL WHERE id=?').run(result.deploymentId, preview.id); + if ( + (input.expectedName && input.expectedName !== name) || + (preview && preview.server !== name) + ) + throw new Error('Available lobby changed. Review a fresh preview.'); + const result = queueNativeCreationIntent({ + server: name, + game, + mapSlug: input.mapSlug, + selection, + plutoKey: input.plutoKey, + createdBy: input.createdBy, + blueprintName: preview?.blueprint_name, + recipe: saved ?? captureRecipePlan(game, input.mapSlug) + }); + if (result.ok && preview) + db() + .prepare( + 'UPDATE server_launch_previews SET deployment_id=?,port=NULL WHERE id=?' + ) + .run(result.deploymentId, preview.id); return result; })(); - } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } + } catch (error) { + return { ok: false, kind: 'bad-request', message: (error as Error).message }; + } } /** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */ -export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection; - plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan; autoscalingDecisionId?: string }): CreateServerResult { +export function queueNativeCreationIntent(input: { + server: string; + game: 't5' | 't6'; + mapSlug: string; + selection: NativeSelection; + plutoKey?: string; + createdBy?: string; + blueprintName?: string | null; + recipe?: LaunchRecipePlan; + autoscalingDecisionId?: string; +}): CreateServerResult { try { return db().transaction(() => { assertScalingOwnership(input.game, input.mapSlug, input.autoscalingDecisionId); const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`; - const reserved = db().prepare(`SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=? - AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))`).get(input.server); - if ((!input.autoscalingDecisionId && input.server !== expected) || nativeInstance(input.server) || getServer(input.server) || reserved) throw new Error('Available lobby changed. Review a fresh preview.'); + const reserved = db() + .prepare( + `SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=? + AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))` + ) + .get(input.server); + if ( + (!input.autoscalingDecisionId && input.server !== expected) || + nativeInstance(input.server) || + getServer(input.server) || + reserved + ) + throw new Error('Available lobby changed. Review a fresh preview.'); reserveServerKey(input.server, input.game, input.plutoKey); - const recipe: NativeRecipe = { ...(input.recipe ?? captureRecipePlan(input.game, input.mapSlug)), - native: input.selection, image: input.selection.image! }; + const recipe: NativeRecipe = { + ...(input.recipe ?? captureRecipePlan(input.game, input.mapSlug)), + native: input.selection, + image: input.selection.image! + }; const id = beginDeployment(input.server, 'create', input.createdBy ?? 'ui'); db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id); - db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)') + db() + .prepare( + 'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)' + ) .run(id, input.server, 'zombie', input.mapSlug, recipe.expectedMap); - db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run(id, JSON.stringify(input.selection)); + db() + .prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)') + .run(id, JSON.stringify(input.selection)); const stored = recipeStorage(recipe); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)') + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)' + ) .run(id, stored.recipe, stored.recipeSummary, input.blueprintName ?? null); - deploymentStage(id, 'Awaiting placement', { phase: 'eligibility', recipeStatus: 'queued' }); + deploymentStage(id, 'Awaiting placement', { + phase: 'eligibility', + recipeStatus: 'queued' + }); return { ok: true as const, name: input.server, port: null, deploymentId: id }; })(); - } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } + } catch (error) { + return { ok: false, kind: 'bad-request', message: (error as Error).message }; + } } export function bindNativeOperation(id: string, server: string): void { const instance = nativeInstance(server); - if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation); + if (instance) + db() + .prepare( + 'INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)' + ) + .run(id, instance.id, instance.generation); } /** Cancelling an unsubmitted launch releases only its local reservation. */ @@ -197,8 +438,12 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string): const instance = nativeInstance(server); if (!instance) return null; return db().transaction(() => { - const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id - WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined; + const creation = db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id + WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL` + ) + .get(instance.id) as { id: string } | undefined; if (!creation) return null; return cancelCreation(creation.id, triggeredBy); })(); @@ -206,40 +451,85 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string): export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult { try { - const instance = nativeInstance(name)!; const server = getServer(name); + const instance = nativeInstance(name)!; + const server = getServer(name); const blocker = activeBetaRemovalBlocker(name); if (blocker) throw new Error(blocker); - if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.'); + if (!server?.map_slug || !getServerKey(name)) + throw new Error('A saved map and server key are required before rebuilding.'); assertReadyLibrary(instance.host_id, instance.library_id); - const recipe: NativeRecipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image, - native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } }; + const recipe: NativeRecipe = { + ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), + image: instance.image, + native: { + hostId: instance.host_id, + libraryId: instance.library_id, + image: instance.image, + bindAddress: instance.bind_address + } + }; return db().transaction(() => { const id = beginDeployment(name, 'rebuild', triggeredBy); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); - db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') + db() + .prepare( + 'INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)' + ) .run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap); const stored = recipeStorage(recipe); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary); - deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' }); + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)' + ) + .run(id, stored.recipe, stored.recipeSummary); + deploymentStage(id, 'Waiting to rebuild when empty', { + phase: 'eligibility', + recipeStatus: 'queued' + }); return { ok: true as const, name, port: instance.port, deploymentId: id }; })(); - } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } + } catch (error) { + return { ok: false, kind: 'bad-request', message: (error as Error).message }; + } } function filesFor(job: Job): FilePlan[] { - const rows = db().prepare(`SELECT f.id,f.kind,f.rel_path,f.content_hash,f.check_before,f.expected_before_sha + const rows = db() + .prepare( + `SELECT f.id,f.kind,f.rel_path,f.content_hash,f.check_before,f.expected_before_sha FROM staged_file_changes f JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server - WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as Array< - Pick & { content_hash: string | null }>; - return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path), - sha256: row.content_hash, checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha })); + WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id` + ) + .all(job.id, job.server) as Array< + Pick< + StagedFileChange, + 'id' | 'kind' | 'rel_path' | 'check_before' | 'expected_before_sha' + > & { content_hash: string | null } + >; + return rows.map((row) => ({ + id: row.id, + path: posix.join( + relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), + row.rel_path + ), + sha256: row.content_hash, + checkBefore: !!row.check_before, + expectedBeforeSha: row.expected_before_sha + })); } -function fileMetadata({ contentBase64, ...file }: FilePlan & { contentBase64?: string | null }): FilePlan { return file; } +function fileMetadata({ + contentBase64, + ...file +}: FilePlan & { contentBase64?: string | null }): FilePlan { + return file; +} function storedFiles(job: Job): FilePlan[] { - const row = db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(job.id) as { files_json: string | null } | undefined; + const row = db() + .prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?') + .get(job.id) as { files_json: string | null } | undefined; return (JSON.parse(row?.files_json ?? '[]') as FilePlan[]).map(fileMetadata); } @@ -248,97 +538,230 @@ function applyReceipt(job: Job, result: Record) { if (job.applied_at || job.recovery_step) return; const plans = storedFiles(job); if (!plans.length) return; - const before = result.before as { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] | undefined; + const before = result.before as + | { id?: number; path: string; contentBase64: string | null; contentSha256?: string }[] + | undefined; db().transaction(() => { for (const plan of plans) { - const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path); + const snapshot = before?.find((item) => item.id === plan.id || item.path === plan.path); if (!snapshot) throw new Error('Agent activation receipt is missing a before image.'); - const row = db().prepare(`SELECT *, ${blobSize('content_hash')} AS size FROM staged_file_changes WHERE id=?`).get(plan.id) as - Omit & { content_hash: string | null; size: number }; + const row = db() + .prepare( + `SELECT *, ${blobSize('content_hash')} AS size FROM staged_file_changes WHERE id=?` + ) + .get(plan.id) as Omit & { + content_hash: string | null; + size: number; + }; if (!row || row.state === 'applied') continue; - const bytes = snapshot.contentSha256 ? getFileBlob(snapshot.contentSha256) - : snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); + const bytes = snapshot.contentSha256 + ? getFileBlob(snapshot.contentSha256) + : snapshot.contentBase64 === null + ? null + : Buffer.from(snapshot.contentBase64, 'base64'); if (row.source_kind === 'beta') { captureBetaBeforeImage(row, bytes); - db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); + db() + .prepare( + 'UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?' + ) + .run( + new Date().toISOString(), + new Date().toISOString(), + row.source_id, + job.server + ); } if (row.deployment_file_id && !row.preserve_before_image) { - db().prepare('UPDATE deployment_files SET verb=?,content_before_hash=?,sha256_before=?,snapshot_pending=0 WHERE id=?') - .run(row.content_hash === null ? 'removed' : bytes === null ? 'added' : 'updated', putFileBlob(bytes), bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); - if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') - .run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`); + db() + .prepare( + 'UPDATE deployment_files SET verb=?,content_before_hash=?,sha256_before=?,snapshot_pending=0 WHERE id=?' + ) + .run( + row.content_hash === null + ? 'removed' + : bytes === null + ? 'added' + : 'updated', + putFileBlob(bytes), + bytes === null ? null : sha256Buffer(bytes), + row.deployment_file_id + ); + if (row.source_kind === 'campaign') + db() + .prepare( + 'INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)' + ) + .run( + row.source_id, + `${job.server}: captured rollback snapshot for ${row.rel_path}` + ); } if (row.content_hash === null) removeScript(job.server, row.rel_path); - else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null, - row.content_hash, row.size, row.inventory_source); - db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id); + else + recordScript( + job.server, + row.rel_path, + row.script_version_id && + db() + .prepare('SELECT 1 FROM script_versions WHERE id=?') + .get(row.script_version_id) + ? row.script_version_id + : null, + row.content_hash, + row.size, + row.inventory_source + ); + db() + .prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?") + .run(new Date().toISOString(), row.id); } - db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); + db() + .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?') + .run(new Date().toISOString(), job.id); })(); } /** Consume a conclusive partial failure without treating removal as a healthy rebuild. */ function applyIncompleteRebuildReceipt(job: Job, operation: HostOperationStatus): void { - const result = operation.result as { reconciled?: unknown; rebuildFailure?: { - phase?: unknown; generation?: unknown; nextGeneration?: unknown; - oldContainerMissing?: unknown; replacementContainerMissing?: unknown; replacementFilesRemoved?: unknown; - } | null } | null; + const result = operation.result as { + reconciled?: unknown; + rebuildFailure?: { + phase?: unknown; + generation?: unknown; + nextGeneration?: unknown; + oldContainerMissing?: unknown; + replacementContainerMissing?: unknown; + replacementFilesRemoved?: unknown; + } | null; + } | null; if (!result || !Object.prototype.hasOwnProperty.call(result, 'rebuildFailure')) return; const failure = result.rebuildFailure; - if (operation.state !== 'failed' || operation.kind !== 'instance.rebuild' || result.reconciled !== true - || !failure || failure.phase !== 'disrupting' || failure.generation !== job.generation - || failure.nextGeneration !== job.generation + 1 || failure.oldContainerMissing !== true - || failure.replacementContainerMissing !== true || failure.replacementFilesRemoved !== true) { - throw new Error('The interrupted rebuild receipt does not confirm safe replacement cleanup. Inspect the original host command.'); + if ( + operation.state !== 'failed' || + operation.kind !== 'instance.rebuild' || + result.reconciled !== true || + !failure || + failure.phase !== 'disrupting' || + failure.generation !== job.generation || + failure.nextGeneration !== job.generation + 1 || + failure.oldContainerMissing !== true || + failure.replacementContainerMissing !== true || + failure.replacementFilesRemoved !== true + ) { + throw new Error( + 'The interrupted rebuild receipt does not confirm safe replacement cleanup. Inspect the original host command.' + ); } db().transaction(() => { - const binding = db().prepare('SELECT operation_id,applied_at FROM native_provisioning WHERE deployment_id=?') + const binding = db() + .prepare( + 'SELECT operation_id,applied_at FROM native_provisioning WHERE deployment_id=?' + ) .get(job.id) as { operation_id: string | null; applied_at: string | null } | undefined; - if (binding?.operation_id !== operation.id) throw new Error('The interrupted rebuild command no longer belongs to this operation.'); + if (binding?.operation_id !== operation.id) + throw new Error('The interrupted rebuild command no longer belongs to this operation.'); if (binding.applied_at) return; - const requested = db().prepare(`SELECT COALESCE(json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'), + const requested = db() + .prepare( + `SELECT COALESCE(json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'), json_extract(COALESCE(p.payload,o.payload),'$.newGeneration')) AS nextGeneration FROM ec_host_operations o - LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(operation.id) as - { nextGeneration: number | null } | undefined; + LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?` + ) + .get(operation.id) as { nextGeneration: number | null } | undefined; if (requested?.nextGeneration !== failure.nextGeneration) { - throw new Error('The interrupted rebuild receipt does not match its requested replacement generation.'); + throw new Error( + 'The interrupted rebuild receipt does not match its requested replacement generation.' + ); } const instance = nativeInstance(job.server); - if (!instance || instance.id !== job.instance_id || instance.generation !== job.generation - || operation.instanceId !== instance.id || operation.hostId !== instance.host_id || operation.generation !== instance.generation) { - throw new Error('The interrupted rebuild belongs to a different instance or generation. Inspect its host command.'); + if ( + !instance || + instance.id !== job.instance_id || + instance.generation !== job.generation || + operation.instanceId !== instance.id || + operation.hostId !== instance.host_id || + operation.generation !== instance.generation + ) { + throw new Error( + 'The interrupted rebuild belongs to a different instance or generation. Inspect its host command.' + ); } - db().prepare("UPDATE servers SET container_state='missing',container_checked_at=?,last_players=NULL,last_roster=NULL WHERE name=?") + db() + .prepare( + "UPDATE servers SET container_state='missing',container_checked_at=?,last_players=NULL,last_roster=NULL WHERE name=?" + ) .run(operation.updatedAt, job.server); - db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id); - recordProgress(job.id, { checks: [{ key: 'container', label: 'Container state after interrupted rebuild', source: 'agent', - result: 'fail', actual: 'missing', checkedAt: operation.updatedAt, - message: 'The old container is absent and replacement creation never started. Partial replacement files were removed.' }], - event: { kind: 'reconciled', message: 'Interrupted rebuild settled: old container removed; replacement creation failed.', - details: { operationId: operation.id, generation: job.generation, nextGeneration: failure.nextGeneration } } }); + db() + .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?') + .run(operation.updatedAt, job.id); + recordProgress(job.id, { + checks: [ + { + key: 'container', + label: 'Container state after interrupted rebuild', + source: 'agent', + result: 'fail', + actual: 'missing', + checkedAt: operation.updatedAt, + message: + 'The old container is absent and replacement creation never started. Partial replacement files were removed.' + } + ], + event: { + kind: 'reconciled', + message: + 'Interrupted rebuild settled: old container removed; replacement creation failed.', + details: { + operationId: operation.id, + generation: job.generation, + nextGeneration: failure.nextGeneration + } + } + }); })(); } /** Retain confirmed file snapshots and instance identity even after observation is cancelled. */ export function reconcileNativeProvisioningReceipt(id: string): void { - const job = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.operation_id,n.applied_at,n.recovery_step + const job = db() + .prepare( + `SELECT d.*,n.instance_id,n.generation,n.operation_id,n.applied_at,n.recovery_step FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id - WHERE d.id=? AND d.server_deleted_at IS NULL`).get(id) as Job | undefined; + WHERE d.id=? AND d.server_deleted_at IS NULL` + ) + .get(id) as Job | undefined; if (!job) return; const migration = hostMigration(id); if (job.state === 'cancelled' && migration?.remove_operation_id && !migration.removed_at) { const removal = getHostOperationStatus(migration.remove_operation_id); if (removal?.state === 'failed') { - db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, migration.instance_id); + db() + .prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?') + .run(migration.target_host_id, migration.instance_id); } else if (removal?.state === 'completed') { const original = nativeInstance(job.server); - if (original?.id === migration.instance_id && original.host_id === migration.source_host_id && original.generation === job.generation) { - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(removal.updatedAt, original.id); + if ( + original?.id === migration.instance_id && + original.host_id === migration.source_host_id && + original.generation === job.generation + ) { + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(removal.updatedAt, original.id); releaseHostPorts(original.id); db().prepare('DELETE FROM servers WHERE name=?').run(job.server); releaseServerKey(job.server); - db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(removal.updatedAt, id); - recordProgress(id, { event: { kind: 'cleanup', message: 'Cancelled migration: source removal completed; replacement creation was not started.' } }); + db() + .prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?') + .run(removal.updatedAt, id); + recordProgress(id, { + event: { + kind: 'cleanup', + message: + 'Cancelled migration: source removal completed; replacement creation was not started.' + } + }); } } } @@ -347,7 +770,8 @@ export function reconcileNativeProvisioningReceipt(id: string): void { if (operation?.state === 'failed') { applyIncompleteRebuildReceipt(job, operation); for (const file of storedFiles(job)) { - db().prepare("UPDATE staged_file_changes SET state=? WHERE id=? AND state='applying'") + db() + .prepare("UPDATE staged_file_changes SET state=? WHERE id=? AND state='applying'") .run(job.state === 'cancelled' ? 'cancelled' : 'pending', file.id); } return; @@ -355,16 +779,25 @@ export function reconcileNativeProvisioningReceipt(id: string): void { if (operation?.state !== 'completed') return; const instance = nativeInstance(job.server); if (!instance || instance.id !== job.instance_id) return; - if (job.state === 'cancelled' && job.action === 'create' - && db().prepare('SELECT 1 FROM native_creation_cancellations WHERE creation_id=?').get(id)) { + if ( + job.state === 'cancelled' && + job.action === 'create' && + db().prepare('SELECT 1 FROM native_creation_cancellations WHERE creation_id=?').get(id) + ) { // Cleanup owns both generations of a cancelled recovery rebuild. Do not // advance the registry underneath its already-bound removal command. - db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, id); + db() + .prepare( + 'UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?' + ) + .run(operation.updatedAt, id); return; } if (job.action === 'destroy') { if (instance.generation !== job.generation) return; - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(operation.updatedAt, instance.id); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(operation.updatedAt, instance.id); releaseHostPorts(instance.id); db().prepare('DELETE FROM servers WHERE name=?').run(job.server); releaseServerKey(job.server); @@ -372,31 +805,64 @@ export function reconcileNativeProvisioningReceipt(id: string): void { return; } if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) { - db().prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?').run(instance.id); + db() + .prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?') + .run(instance.id); // Replacements queued while the delivered rebuild was settling bound the // previous generation. Only untouched, newer jobs may follow this receipt. - if (job.state === 'cancelled') db().prepare(`UPDATE native_provisioning SET generation=? + if (job.state === 'cancelled') + db() + .prepare( + `UPDATE native_provisioning SET generation=? WHERE instance_id=? AND generation=? AND operation_id IS NULL AND stage_operation_id IS NULL AND deployment_id IN (SELECT newer.id FROM server_deployments newer WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?) - AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))`) - .run(job.generation + 1, instance.id, job.generation, id); - } else if (instance.generation !== job.generation && !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1)) return; + AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))` + ) + .run(job.generation + 1, instance.id, job.generation, id); + } else if ( + instance.generation !== job.generation && + !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1) + ) + return; applyReceipt(job, (operation.result ?? {}) as Record); - db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, id); + db() + .prepare( + 'UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?' + ) + .run(operation.updatedAt, id); } -function hold(job: Job, message: string) { if (!stillActive(job)) return; deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); } +function hold(job: Job, message: string) { + if (!stillActive(job)) return; + deploymentStage(job.id, message, { + blocker: { code: 'native-host', message, automatic: true } + }); +} function stillActive(job: Job): boolean { - return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(job.id); + return !!db() + .prepare( + `SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted')` + ) + .get(job.id); } function complete(job: Job) { db().transaction(() => { - db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); - db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id); - if (job.recovery_step) recordProgress(job.id, { recovery: { state: 'released', operationId: job.id, - message: 'Container and game health verified after provisioning recovery.' } }); + db() + .prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?') + .run(new Date().toISOString(), job.id); + db() + .prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?') + .run(new Date().toISOString(), job.id); + if (job.recovery_step) + recordProgress(job.id, { + recovery: { + state: 'released', + operationId: job.id, + message: 'Container and game health verified after provisioning recovery.' + } + }); finishDeployment(job.id, { ok: true }); })(); } @@ -408,69 +874,150 @@ function queueUnhealthyRecovery(job: Job, instance: NativeInstance): void { const live = nativeInstance(job.server); if (live?.id !== instance.id || live.generation !== instance.generation) return; if (job.recovery_step === 'rebuild') { - const message = 'Container is still unhealthy after recovery restart and rebuild. Inspect the server before retrying.'; - db().prepare('UPDATE restart_rollout SET error=? WHERE operation_id=?').run(message, job.id); + const message = + 'Container is still unhealthy after recovery restart and rebuild. Inspect the server before retrying.'; + db() + .prepare('UPDATE restart_rollout SET error=? WHERE operation_id=?') + .run(message, job.id); finishDeployment(job.id, { ok: false, message }); return; } const host = getHost(instance.host_id); - if (!host?.connected || host.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; } + if (!host?.connected || host.mode === 'disabled') { + hold(job, 'Host is unavailable; waiting for reconnection.'); + return; + } if (!host.inventory.capabilities?.includes(unhealthyRecoveryCapability)) { - hold(job, 'Update this host’s agent to recover unhealthy containers during provisioning.'); return; + hold( + job, + 'Update this host’s agent to recover unhealthy containers during provisioning.' + ); + return; } const step = job.recovery_step === 'restart' ? 'rebuild' : 'restart'; const settings = fleetSettings(); - const slots = db().prepare(`SELECT i.host_id FROM restart_rollout r LEFT JOIN native_instances i - ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND (r.operation_id IS NULL OR r.operation_id!=?)`) + const slots = db() + .prepare( + `SELECT i.host_id FROM restart_rollout r LEFT JOIN native_instances i + ON i.server=r.server AND i.retired_at IS NULL WHERE r.healthy_at IS NULL AND (r.operation_id IS NULL OR r.operation_id!=?)` + ) .all(job.id) as { host_id: string | null }[]; if (settings.restartConcurrency > 0 && slots.length >= settings.restartConcurrency) { - hold(job, 'Waiting for restart recovery capacity.'); return; + hold(job, 'Waiting for restart recovery capacity.'); + return; } - if (settings.restartConcurrencyPerHost > 0 && slots.filter(slot => slot.host_id === instance.host_id).length >= settings.restartConcurrencyPerHost) { - hold(job, 'Waiting for restart recovery capacity on this host.'); return; + if ( + settings.restartConcurrencyPerHost > 0 && + slots.filter((slot) => slot.host_id === instance.host_id).length >= + settings.restartConcurrencyPerHost + ) { + hold(job, 'Waiting for restart recovery capacity on this host.'); + return; } let payload: Record = { unhealthyRecovery: true }; if (step === 'rebuild') { const beta = activeBetaRemovalBlocker(job.server); - if (beta) { hold(job, beta); return; } + if (beta) { + hold(job, beta); + return; + } if (job.action !== 'rebuild' && settings.rebuildConcurrency > 0) { - if (lifecycleHolders().filter(holder => holder.id !== job.id && holder.action === 'rebuild').length >= settings.rebuildConcurrency) { - hold(job, 'Waiting for rebuild capacity.'); return; + if ( + lifecycleHolders().filter( + (holder) => holder.id !== job.id && holder.action === 'rebuild' + ).length >= settings.rebuildConcurrency + ) { + hold(job, 'Waiting for rebuild capacity.'); + return; } } assertReadyLibrary(instance.host_id, instance.library_id); // Recover the files and credentials selected for this launch, even // if new versions have been promoted while readiness was pending. - const launch = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o + const launch = db() + .prepare( + `SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o JOIN native_instances i ON ${runtimeLaunchSql} LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id - WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL ORDER BY o.rowid DESC LIMIT 1`) + WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL ORDER BY o.rowid DESC LIMIT 1` + ) .get(instance.id) as { payload: string } | undefined; - if (!launch) { hold(job, 'Waiting for the original launch payload before recovery rebuild.'); return; } - payload = { ...JSON.parse(launch.payload), nextGeneration: instance.generation + 1, unhealthyRecovery: true }; + if (!launch) { + hold(job, 'Waiting for the original launch payload before recovery rebuild.'); + return; + } + payload = { + ...JSON.parse(launch.payload), + nextGeneration: instance.generation + 1, + unhealthyRecovery: true + }; delete payload.safetyOverride; } - const operation = queueHostOperationSerialized(instance.host_id, `instance.${step}`, Buffer.from(JSON.stringify(payload)), - { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:unhealthy:${step}` }, - { name: instance.server, unhealthyRecovery: true }); + const operation = queueHostOperationSerialized( + instance.host_id, + `instance.${step}`, + Buffer.from(JSON.stringify(payload)), + { + instanceId: instance.id, + generation: instance.generation, + idempotencyKey: `${job.id}:unhealthy:${step}` + }, + { name: instance.server, unhealthyRecovery: true } + ); const at = new Date().toISOString(); - db().prepare('UPDATE native_provisioning SET operation_id=?,generation=?,recovery_step=?,applied_at=NULL WHERE deployment_id=?') + db() + .prepare( + 'UPDATE native_provisioning SET operation_id=?,generation=?,recovery_step=?,applied_at=NULL WHERE deployment_id=?' + ) .run(operation.id, instance.generation, step, job.id); - db().prepare("UPDATE server_deployments SET state='running',submitted_at=?,error=NULL,finished_at=NULL WHERE id=?").run(at, job.id); - db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) - DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, at, job.id); + db() + .prepare( + "UPDATE server_deployments SET state='running',submitted_at=?,error=NULL,finished_at=NULL WHERE id=?" + ) + .run(at, job.id); + db() + .prepare( + `INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) + DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL` + ) + .run(job.server, at, job.id); const message = `Container unhealthy; ${step === 'restart' ? 'forcing a restart' : 'forcing a rebuild after restart recovery failed'}.`; - deploymentStage(job.id, message, { phase: 'container-action', blocker: null, checks: [], - command: { state: 'attempted', message: `Recovery ${step} queued as agent operation ${operation.id}.` }, - recovery: { state: 'held', operationId: job.id, message: 'Recovery runs within this provisioning operation.' }, - event: { kind: 'unhealthy-recovery', message, details: { step, operationId: operation.id, generation: instance.generation } } }); + deploymentStage(job.id, message, { + phase: 'container-action', + blocker: null, + checks: [], + command: { + state: 'attempted', + message: `Recovery ${step} queued as agent operation ${operation.id}.` + }, + recovery: { + state: 'held', + operationId: job.id, + message: 'Recovery runs within this provisioning operation.' + }, + event: { + kind: 'unhealthy-recovery', + message, + details: { step, operationId: operation.id, generation: instance.generation } + } + }); })(); } -export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null, - options: { wait?: boolean } = {}): Promise { +export async function processNativeProvisioning( + eligibility: (operation: { + id: string; + server: string; + action: string; + state: string; + stage: string; + }) => string | null, + options: { wait?: boolean } = {} +): Promise { ensureHostSchema(db()); - const cancelled = db().prepare(`SELECT d.id FROM server_deployments d + const cancelled = db() + .prepare( + `SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN ec_host_operations o ON o.id=n.operation_id LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id @@ -483,31 +1030,52 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s WHERE s.operation_id=d.id AND f.state='applying')) OR (m.removed_at IS NULL AND (removal.state='completed' OR (removal.state='failed' AND EXISTS (SELECT 1 FROM ec_host_ports p WHERE p.host_id=m.target_host_id AND p.instance_id=m.instance_id)))) - )`).all() as { id: string }[]; + )` + ) + .all() as { id: string }[]; for (const row of cancelled) { - try { reconcileNativeProvisioningReceipt(row.id); } - catch (error) { - deploymentStage(row.id, 'Cancelled; waiting to reconcile the delivered command: ' + (error as Error).message); + try { + reconcileNativeProvisioningReceipt(row.id); + } catch (error) { + deploymentStage( + row.id, + 'Cancelled; waiting to reconcile the delivered command: ' + (error as Error).message + ); } } reconcileHostDrains(); - const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection + const jobs = db() + .prepare( + `SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN native_creation_intents q ON q.deployment_id=d.id - WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[]; + WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid` + ) + .all() as Job[]; // One host's observation must not prevent another job being admitted. // Per-job ownership also covers overlapping direct calls and worker passes. - const work = jobs.map(job => { + const work = jobs.map((job) => { const pending = activeJobs.get(job.id); if (pending) return pending; - const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`, - () => processJob(job, eligibility)) - .catch(error => { if (stillActive(job)) hold(job, (error as Error).message); }) - .finally(() => { activeJobs.delete(job.id); preparingJobs.delete(job.id); }); + const work = trackActivity( + `provisioning:${job.id}`, + `Evaluating ${job.action} for ${job.server}`, + () => processJob(job, eligibility) + ) + .catch((error) => { + if (stillActive(job)) hold(job, (error as Error).message); + }) + .finally(() => { + activeJobs.delete(job.id); + preparingJobs.delete(job.id); + }); activeJobs.set(job.id, work); return work; }); - if (options.wait !== false) { await Promise.all(work); reconcileHostDrains(); } + if (options.wait !== false) { + await Promise.all(work); + reconcileHostDrains(); + } } const activeJobs = new Map>(); @@ -515,69 +1083,171 @@ const activeJobs = new Map>(); function otherPreparations(id: string): number { if (!preparingJobs.size) return 0; - const reserved = new Set((db().prepare(`SELECT id FROM server_deployments WHERE id IN (SELECT value FROM json_each(?)) - AND ${committedRebalanceStartup}`).all(JSON.stringify([...preparingJobs])) as { id: string }[]).map(row => row.id)); - return [...preparingJobs].filter(other => other !== id && !reserved.has(other)).length; + const reserved = new Set( + ( + db() + .prepare( + `SELECT id FROM server_deployments WHERE id IN (SELECT value FROM json_each(?)) + AND ${committedRebalanceStartup}` + ) + .all(JSON.stringify([...preparingJobs])) as { id: string }[] + ).map((row) => row.id) + ); + return [...preparingJobs].filter((other) => other !== id && !reserved.has(other)).length; } let launchCommit: Promise = Promise.resolve(); function commitPreparedLaunch(commit: () => void): Promise { // Worker replies can arrive together. Persist one launch per event-loop turn // so a batch of large SQLite writes cannot hold off requests and health timers. - const next = launchCommit.then(async () => { await yieldToController(); commit(); }); + const next = launchCommit.then(async () => { + await yieldToController(); + commit(); + }); launchCommit = next.catch(() => {}); return next; } function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boolean { - const selected = (recipe: LaunchRecipePlan) => ({ game: recipe.game, mapSlug: recipe.mapSlug, - expectedMap: recipe.expectedMap, configFile: recipe.configFile, configSource: recipe.configSource, files: recipe.files }); + const selected = (recipe: LaunchRecipePlan) => ({ + game: recipe.game, + mapSlug: recipe.mapSlug, + expectedMap: recipe.expectedMap, + configFile: recipe.configFile, + configSource: recipe.configSource, + files: recipe.files + }); return JSON.stringify(selected(first)) === JSON.stringify(selected(second)); } /** Capacity history and host ranking run in the background reader, one job at a time. */ -async function placeQueuedCreation(job: Job, instance: NativeInstance | null, eligibility: (job: Job) => string | null): Promise { +async function placeQueuedCreation( + job: Job, + instance: NativeInstance | null, + eligibility: (job: Job) => string | null +): Promise { const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe; const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!; - const network = prior.automatic ? prior.network ?? {} : prior; - const options: PlacementOptions = { hostId: prior.automatic ? undefined : prior.hostId, - libraryId: prior.libraryId, image: prior.image ?? recipe.image, bindAddress: network.bindAddress, - thorough: true, dispatch: true, excludeInstanceId: instance?.id }; + const network = prior.automatic ? (prior.network ?? {}) : prior; + const options: PlacementOptions = { + hostId: prior.automatic ? undefined : prior.hostId, + libraryId: prior.libraryId, + image: prior.image ?? recipe.image, + bindAddress: network.bindAddress, + thorough: true, + dispatch: true, + excludeInstanceId: instance?.id + }; const decision = await evaluateNativePlacement(options); return db().transaction(() => { - const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; + const state = db() + .prepare( + 'SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL' + ) + .get(job.id) as { state: string } | undefined; if (state?.state !== 'queued') return null; if (eligibility(job)) return null; const live = nativeInstance(job.server); - if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null; - const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL - AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n; - if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; } + if ( + instance + ? live?.id !== instance.id || live.generation !== instance.generation + : !!live || !!getServer(job.server) + ) + return null; + const active = ( + db() + .prepare( + `SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL + AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')` + ) + .get(job.id) as { n: number } + ).n; + if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { + hold(job, 'Waiting for startup capacity.'); + return null; + } const capacityBlocker = concurrencyBlocker(job.id, job.action, decision.hostId); - if (capacityBlocker) { hold(job, capacityBlocker); return null; } - if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; } + if (capacityBlocker) { + hold(job, capacityBlocker); + return null; + } + if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { + hold(job, 'Host measurements changed; waiting for another placement check.'); + return null; + } // A short current-state check protects against reservations made while the // reader ranked hosts. It never scans resource history or contacts agents. - const { host, port } = choosePlacement({ ...options, hostId: decision.hostId, thorough: false }); + const { host, port } = choosePlacement({ + ...options, + hostId: decision.hostId, + thorough: false + }); const bind = network.bindAddress ?? host.bindAddress; const instanceId = instance?.id ?? randomUUID(); if (instance) releaseHostPorts(instanceId); reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'udp' }); reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'tcp' }); - if (instance) db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`) - .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instanceId); + if (instance) + db() + .prepare( + `UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?` + ) + .run( + host.id, + bind, + port, + network.publicAddress ?? host.publicAddress, + network.publicPort ?? null, + instanceId + ); else { - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at) - VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, job.server, host.id, prior.libraryId, options.image, bind, port, - network.publicAddress ?? host.publicAddress, network.publicPort ?? null, new Date().toISOString()); - db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(job.id, instanceId); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at) + VALUES(?,?,?,?,?,?,?,?,?,?)` + ) + .run( + instanceId, + job.server, + host.id, + prior.libraryId, + options.image, + bind, + port, + network.publicAddress ?? host.publicAddress, + network.publicPort ?? null, + new Date().toISOString() + ); + db() + .prepare( + 'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)' + ) + .run(job.id, instanceId); } - db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id); - recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort }; + db() + .prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?') + .run(port, job.id); + recipe.native = { + ...prior, + hostId: host.id, + bindAddress: bind, + publicAddress: network.publicAddress ?? host.publicAddress, + publicPort: network.publicPort + }; const stored = recipeStorage(recipe); - db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id); - db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id); - deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' }); + db() + .prepare( + 'UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?' + ) + .run(stored.recipe, stored.recipeSummary, job.id); + db() + .prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?') + .run(port, job.id); + deploymentStage( + job.id, + `Placed on ${host.name} after CPU, memory, history and port checks`, + { phase: 'eligibility' } + ); preparingJobs.add(job.id); return nativeInstance(job.server)!; })(); @@ -588,35 +1258,79 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const unsettled = unsettledCancelledCommand(job.server, job.id); if (unsettled) { const pendingReceipt = unsettled.state === 'completed' || unsettled.state === 'failed'; - const message = `Waiting for the host command from cancelled operation ${unsettled.deploymentId} ${pendingReceipt ? 'to be reconciled' : 'to settle'}. Agent state: ${unsettled.state}.` - + (unsettled.error ? ` Last error: ${unsettled.error.slice(0, 500)}` : ''); - deploymentStage(job.id, message, { blocker: { code: 'cancelled-command', message, automatic: true, - href: `/provisioning/${encodeURIComponent(unsettled.deploymentId)}`, label: 'View cancelled operation' } }); + const message = + `Waiting for the host command from cancelled operation ${unsettled.deploymentId} ${pendingReceipt ? 'to be reconciled' : 'to settle'}. Agent state: ${unsettled.state}.` + + (unsettled.error ? ` Last error: ${unsettled.error.slice(0, 500)}` : ''); + deploymentStage(job.id, message, { + blocker: { + code: 'cancelled-command', + message, + automatic: true, + href: `/provisioning/${encodeURIComponent(unsettled.deploymentId)}`, + label: 'View cancelled operation' + } + }); return; } let instance = nativeInstance(job.server); - if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) { + if ( + job.action === 'create' && + !job.operation_id && + !job.stage_operation_id && + !job.submitted_at + ) { // A resumed bound job owns its original identity. An unplaced intent // cannot adopt an instance or registered server that appeared later. - if (job.instance_id ? !instance || instance.id !== job.instance_id || instance.generation !== job.generation - : !!instance || !!getServer(job.server)) { - finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; + if ( + job.instance_id + ? !instance || + instance.id !== job.instance_id || + instance.generation !== job.generation + : !!instance || !!getServer(job.server) + ) { + finishDeployment(job.id, { + ok: false, + message: 'Original instance was retired or replaced; no command was sent.' + }); + return; } const settings = fleetSettings(); - const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL - AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id) as { n: number }; - if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } + const active = db() + .prepare( + `SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL + AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})` + ) + .get(job.id) as { n: number }; + if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { + hold(job, 'Waiting for startup capacity.'); + return; + } const capacityBlocker = concurrencyBlocker(job.id, job.action); - if (capacityBlocker) { hold(job, capacityBlocker); return; } + if (capacityBlocker) { + hold(job, capacityBlocker); + return; + } if (eligibility(job)) return; instance = await placeQueuedCreation(job, instance, eligibility); if (!instance) return; - job.instance_id = instance.id; job.generation = instance.generation; + job.instance_id = instance.id; + job.generation = instance.generation; } const migration = hostMigration(job.id); - const rebuilding = job.recovery_step === 'rebuild' || (!job.recovery_step && job.action === 'rebuild' && !migration); - if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(rebuilding && instance.generation === job.generation + 1))) { - finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; + const rebuilding = + job.recovery_step === 'rebuild' || + (!job.recovery_step && job.action === 'rebuild' && !migration); + if ( + !instance || + instance.id !== job.instance_id || + (instance.generation !== job.generation && + !(rebuilding && instance.generation === job.generation + 1)) + ) { + finishDeployment(job.id, { + ok: false, + message: 'Original instance was retired or replaced; no command was sent.' + }); + return; } if (migration && !migration.removed_at) { await processMigrationRemoval(job, instance, migration, () => eligibility(job)); @@ -624,39 +1338,97 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { } const host = getHost(instance.host_id); if (!host?.connected || host.mode === 'disabled') { - db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server); - hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return; + db() + .prepare( + "UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?" + ) + .run(job.server); + hold( + job, + 'Host is unavailable; existing server state is unknown. Waiting for reconnection.' + ); + return; } if (job.operation_id) { const operation = getHostOperationStatus(job.operation_id); - const recoveryCleanup = operation?.kind === 'instance.cancel-create' && !!db().prepare(`SELECT 1 FROM ec_host_operations - WHERE id=? AND json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL`).get(operation.id); - if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) - && (!host.inventory.capabilities?.includes('instance.cancel-create') || (recoveryCleanup && !host.inventory.capabilities?.includes(unhealthyRecoveryCapability)))) { - hold(job, hostAgentUpdatesEnabled(host.inventory) - ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.' - : 'Update the host agent to support creation cancellation: rerun the current installer on this host. Automatic updates are not enabled. New command delivery is paused; cleanup and port release await the updated agent.'); return; + const recoveryCleanup = + operation?.kind === 'instance.cancel-create' && + !!db() + .prepare( + `SELECT 1 FROM ec_host_operations + WHERE id=? AND json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL` + ) + .get(operation.id); + if ( + operation?.kind === 'instance.cancel-create' && + !['completed', 'failed'].includes(operation.state) && + (!host.inventory.capabilities?.includes('instance.cancel-create') || + (recoveryCleanup && + !host.inventory.capabilities?.includes(unhealthyRecoveryCapability))) + ) { + hold( + job, + hostAgentUpdatesEnabled(host.inventory) + ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.' + : 'Update the host agent to support creation cancellation: rerun the current installer on this host. Automatic updates are not enabled. New command delivery is paused; cleanup and port release await the updated agent.' + ); + return; + } + if (!operation || operation.state === 'unknown') { + hold( + job, + 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.' + ); + return; } - if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; } if (operation.state === 'failed') { applyIncompleteRebuildReceipt(job, operation); - if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) { + if ( + job.action === 'destroy' && + scalingIntent(job.id) && + (operation.result as { autoscalingInvalidated?: boolean } | null) + ?.autoscalingInvalidated === true + ) { db().transaction(() => { - db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?") + db() + .prepare( + "UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?" + ) .run(new Date().toISOString(), job.id); - recordProgress(job.id, { blocker: null, event: { kind: 'outcome', message: 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', details: { state: 'cancelled' } } }); + recordProgress(job.id, { + blocker: null, + event: { + kind: 'outcome', + message: + 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', + details: { state: 'cancelled' } + } + }); })(); return; } if (migration && !job.recovery_step) { if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) { db().transaction(() => { - db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id); - db().prepare('UPDATE native_provisioning SET operation_id=NULL WHERE deployment_id=?').run(job.id); - db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); + db() + .prepare('DELETE FROM restart_rollout WHERE operation_id=?') + .run(job.id); + db() + .prepare( + 'UPDATE native_provisioning SET operation_id=NULL WHERE deployment_id=?' + ) + .run(job.id); + db() + .prepare( + "UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?" + ) + .run(job.id); })(); } - hold(job, `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.`); + hold( + job, + `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.` + ); return; } // Failed commands either never mutated or have a conclusively @@ -664,282 +1436,756 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { db().transaction(() => { db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id); const files = storedFiles(job); - for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id); - finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' }); + for (const file of files) + db() + .prepare( + "UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'" + ) + .run(file.id); + finishDeployment(job.id, { + ok: false, + message: operation.error ?? 'Agent operation failed before mutation.' + }); })(); return; } if (operation.state !== 'completed') { const blocked = operation.result as { message?: string } | null; - hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return; + hold(job, blocked?.message ?? `Agent operation ${operation.state}`); + return; } if (job.action === 'destroy') { db().transaction(() => { - deploymentStage(job.id, REMOVING_SERVER, { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Agent confirmed instance and container removal', at: operation.updatedAt } }); - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id); + deploymentStage(job.id, REMOVING_SERVER, { + phase: 'cleanup', + checkpoint: { + key: 'container-action', + label: 'Agent confirmed instance and container removal', + at: operation.updatedAt + } + }); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), instance!.id); releaseHostPorts(instance!.id); db().prepare('DELETE FROM servers WHERE name=?').run(job.server); releaseServerKey(job.server); - recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } }); + recordProgress(job.id, { + checkpoint: { + key: 'cleanup', + label: 'Instance retired and port reservations released' + } + }); complete(job); finishCreationCancellation(job.id); })(); return; } if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) { - db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation); + db() + .prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?') + .run(job.generation + 1, instance.id, job.generation); instance = nativeInstance(job.server)!; } applyReceipt(job, (operation.result ?? {}) as Record); - db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, job.id); + db() + .prepare( + 'UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?' + ) + .run(operation.updatedAt, job.id); if (job.state !== 'awaiting-health') { - db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id); + db() + .prepare( + "UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?" + ) + .run(job.id); const action = job.recovery_step ? `recovery ${job.recovery_step}` : job.action; - recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${action}; verifying the resulting instance.` }, - checkpoint: { key: job.recovery_step ? `recovery-${job.recovery_step}` : 'container-action', label: `Agent confirmed ${action}`, at: operation.updatedAt } }); + recordProgress(job.id, { + command: { + state: 'accepted', + message: `Agent confirmed ${action}; verifying the resulting instance.` + }, + checkpoint: { + key: job.recovery_step ? `recovery-${job.recovery_step}` : 'container-action', + label: `Agent confirmed ${action}`, + at: operation.updatedAt + } + }); } const checks: CheckInput[] = []; - const check = (value: CheckInput, phase: import('../../provisioning-detail.js').ProvisioningPhase) => { + const check = ( + value: CheckInput, + phase: import('../../provisioning-detail.js').ProvisioningPhase + ) => { checks.push({ ...value, checkedAt: new Date().toISOString() }); - recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) }); + recordProgress(job.id, { + checks, + ...(value.result === 'pass' + ? { checkpoint: { key: phase, label: value.label } } + : {}) + }); }; - deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks }); + deploymentStage(job.id, 'Checking container state with host agent', { + phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', + blocker: null, + checks + }); const verificationIdentity = monitoringIdentity(job.server); - const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect'); + const health = await nativeRequest<{ healthy: boolean; state: string }>( + instance, + 'instance.inspect' + ); const containerObservedAt = new Date().toISOString(); if (!stillActive(job)) return; if (job.action === 'stop') { - const stopped = ['stopped','exited','created','dead'].includes(health.state); - check({ key: 'stopped', label: 'Stopped container verified', source: 'agent', result: stopped ? 'pass' : 'waiting', actual: health.state }, 'verify-stopped'); - if (!stopped) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; } - db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server); - complete(job); return; + const stopped = ['stopped', 'exited', 'created', 'dead'].includes(health.state); + check( + { + key: 'stopped', + label: 'Stopped container verified', + source: 'agent', + result: stopped ? 'pass' : 'waiting', + actual: health.state + }, + 'verify-stopped' + ); + if (!stopped) { + hold(job, 'Waiting for the agent to verify a stopped container.'); + return; + } + db() + .prepare( + 'UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?' + ) + .run(health.state, job.server); + complete(job); + return; } - check({ key: 'container', label: 'Container readiness verified', source: 'agent', result: health.healthy ? 'pass' : 'waiting', actual: health.state }, 'verify-container'); + check( + { + key: 'container', + label: 'Container readiness verified', + source: 'agent', + result: health.healthy ? 'pass' : 'waiting', + actual: health.state + }, + 'verify-container' + ); if (!health.healthy) { - if (health.state === 'unhealthy' && (job.action === 'create' || job.action === 'rebuild')) queueUnhealthyRecovery(job, instance); + if ( + health.state === 'unhealthy' && + (job.action === 'create' || job.action === 'rebuild') + ) + queueUnhealthyRecovery(job, instance); else hold(job, `Waiting for container readiness: ${health.state}`); return; } deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null }); const status = await probeServerStatus(job.server, operation.updatedAt); if (!stillActive(job)) return; - check({ key: 'game', label: 'Fresh game status verified', source: status.ok ? status.source : null, - result: status.ok ? 'pass' : 'unknown', actual: status.ok ? status.players : null, - message: status.ok ? null : status.reason }, 'verify-game'); - if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; } + check( + { + key: 'game', + label: 'Fresh game status verified', + source: status.ok ? status.source : null, + result: status.ok ? 'pass' : 'unknown', + actual: status.ok ? status.players : null, + message: status.ok ? null : status.reason + }, + 'verify-game' + ); + if (!status.ok) { + hold(job, `Waiting for game status: ${status.reason}`); + return; + } if (job.action === 'create' || job.action === 'rebuild') { - deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null }); - const recipe = db().prepare('SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?') + deploymentStage(job.id, 'Checking selected engine map', { + phase: 'verify-map', + blocker: null + }); + const recipe = db() + .prepare( + 'SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?' + ) .get(job.id) as { expectedMap: string }; - check({ key: 'map', label: 'Selected engine map verified', source: status.source, - result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map'); - if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; } - deploymentStage(job.id, 'Checking production Plutonium release', { phase: 'verify-plutonium', blocker: null }); + check( + { + key: 'map', + label: 'Selected engine map verified', + source: status.source, + result: status.map === recipe.expectedMap ? 'pass' : 'waiting', + expected: recipe.expectedMap, + actual: status.map + }, + 'verify-map' + ); + if (status.map !== recipe.expectedMap) { + hold( + job, + `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.` + ); + return; + } + deploymentStage(job.id, 'Checking production Plutonium release', { + phase: 'verify-plutonium', + blocker: null + }); await checkRelease(); if (!stillActive(job)) return; const target = freshRelease(); - checks.push({ key: 'plutonium-release', label: 'Production Plutonium release', source: 'manifest', - result: target === null ? 'unknown' : 'pass', actual: target }); + checks.push({ + key: 'plutonium-release', + label: 'Production Plutonium release', + source: 'manifest', + result: target === null ? 'unknown' : 'pass', + actual: target + }); recordProgress(job.id, { checks }); - deploymentStage(job.id, 'Checking running Plutonium revision', { phase: 'verify-plutonium', blocker: null }); + deploymentStage(job.id, 'Checking running Plutonium revision', { + phase: 'verify-plutonium', + blocker: null + }); const revision = await probeVersion(job.server, { operationId: job.id }); if (!stillActive(job)) return; - check({ key: 'plutonium-version', label: 'Plutonium revision verified', source: 'rcon', expected: target, actual: revision, - result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass' }, 'verify-plutonium'); - if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; } - if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; } + check( + { + key: 'plutonium-version', + label: 'Plutonium revision verified', + source: 'rcon', + expected: target, + actual: revision, + result: + revision === null || target === null + ? 'unknown' + : revision < target + ? 'fail' + : 'pass' + }, + 'verify-plutonium' + ); + if (revision === null || target === null) { + hold(job, 'Waiting to verify the Plutonium runtime revision.'); + return; + } + if (revision < target) { + finishDeployment(job.id, { + ok: false, + message: `Running Plutonium r${revision} is older than production r${target}.` + }); + return; + } } const completed = db().transaction(() => { - if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity, - { state: health.state, observedAt: containerObservedAt }, status)) return false; + if ( + !stillActive(job) || + !handoffVerifiedHealth( + job.server, + verificationIdentity, + { state: health.state, observedAt: containerObservedAt }, + status + ) + ) + return false; recordProgress(job.id, { blocker: null, checks }); complete(job); return true; })(); - if (!completed) hold(job, 'Waiting for current health evidence before completing verification.'); + if (!completed) + hold(job, 'Waiting for current health evidence before completing verification.'); return; } const reason = eligibility(job); - if (reason) { hold(job, reason); return; } + if (reason) { + hold(job, reason); + return; + } const overrideSafety = !!safetyOverride(job.id); if (overrideSafety && !host.inventory.capabilities?.includes(safetyOverrideCapability)) { - hold(job, 'Update this host’s agent before using admin override.'); return; + hold(job, 'Update this host’s agent before using admin override.'); + return; } if (job.action === 'destroy' || job.action === 'rebuild') { - const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; } + const beta = activeBetaRemovalBlocker(job.server); + if (beta) { + hold(job, beta); + return; + } } const files = job.stage_operation_id ? storedFiles(job) : filesFor(job); if (files.length && !job.stage_operation_id) { db().transaction(() => { - const stage = queueHostOperationSerialized(instance.host_id, 'files.stage', Buffer.from(JSON.stringify({ stageId: job.id, files: files.map(file => ({ ...file, ...(file.sha256 ? { contentSha256: file.sha256 } : { contentBase64: null }) })) })), - { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }, { stageId: job.id }); - db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?') + const stage = queueHostOperationSerialized( + instance.host_id, + 'files.stage', + Buffer.from( + JSON.stringify({ + stageId: job.id, + files: files.map((file) => ({ + ...file, + ...(file.sha256 + ? { contentSha256: file.sha256 } + : { contentBase64: null }) + })) + }) + ), + { + instanceId: instance.id, + generation: instance.generation, + idempotencyKey: `${job.id}:stage` + }, + { stageId: job.id } + ); + db() + .prepare( + 'UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?' + ) .run(stage.id, JSON.stringify(files.map(fileMetadata)), job.id); })(); - hold(job, 'Staging and verifying files on the host.'); return; + hold(job, 'Staging and verifying files on the host.'); + return; } if (job.stage_operation_id) { const stage = getHostOperationStatus(job.stage_operation_id); - if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; } - if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; } - if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) { - finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return; + if (stage?.state === 'failed') { + finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); + return; + } + if (stage?.state !== 'completed') { + hold(job, 'Waiting for verified remote file staging.'); + return; + } + if ( + files.some( + (file) => + ( + db() + .prepare('SELECT state FROM staged_file_changes WHERE id=?') + .get(file.id) as { state: string } | undefined + )?.state !== 'pending' + ) + ) { + finishDeployment(job.id, { + ok: false, + message: 'Staged file request was cancelled or changed before activation.' + }); + return; } } const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id); - if (capacityBlocker) { hold(job, capacityBlocker); return; } - const settings = fleetSettings(); - const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL - AND action IN (${actions.map(() => '?').join(',')}) AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id, ...actions) as { n: number }).n; - if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) { - deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', - blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; + if (capacityBlocker) { + hold(job, capacityBlocker); + return; + } + const settings = fleetSettings(); + const count = (actions: string[]) => + ( + db() + .prepare( + `SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL + AND action IN (${actions.map(() => '?').join(',')}) AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})` + ) + .get(job.id, ...actions) as { n: number } + ).n; + if ( + (job.action === 'create' || job.action === 'rebuild') && + count(['create', 'rebuild']) + otherPreparations(job.id) >= settings.createConcurrency + ) { + deploymentStage(job.id, 'Waiting for startup capacity.', { + phase: 'startup-queue', + blocker: { + code: 'startup-capacity', + message: 'Waiting for startup capacity.', + automatic: true + } + }); + return; + } + if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { + hold(job, 'Waiting for destruction capacity.'); + return; } - if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null }); - if (job.action === 'create' || migration) choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, - bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true }); - if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id); + if (job.action === 'create' || migration) + choosePlacement({ + hostId: instance.host_id, + libraryId: instance.library_id, + image: instance.image, + bindAddress: instance.bind_address, + excludeInstanceId: instance.id, + thorough: false, + dispatch: true + }); + if (job.action === 'create' || job.action === 'rebuild') + assertReadyLibrary(instance.host_id, instance.library_id); const checkOccupancy = async (): Promise => { if (!stillActive(job)) return false; if (overrideSafety) return true; // Fast controller check keeps occupied lobbies out of the operation slots. // The agent repeats this immediately before its local mutation. - deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null }); - const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect'); + deploymentStage(job.id, 'Checking container and player occupancy before submission', { + phase: 'eligibility', + blocker: null + }); + const health = await nativeRequest<{ state: string; healthy?: boolean }>( + instance, + 'instance.inspect' + ); if (!stillActive(job)) return false; if (job.action === 'rebuild' && health.state === 'unhealthy') { - if (!getHost(instance.host_id)?.inventory.capabilities?.includes(unhealthyRecoveryCapability)) { - hold(job, 'Update this host’s agent to recover unhealthy containers during provisioning.'); return false; + if ( + !getHost(instance.host_id)?.inventory.capabilities?.includes( + unhealthyRecoveryCapability + ) + ) { + hold( + job, + 'Update this host’s agent to recover unhealthy containers during provisioning.' + ); + return false; } return true; } - if (!['stopped','exited','created','dead','missing'].includes(health.state)) { + if (!['stopped', 'exited', 'created', 'dead', 'missing'].includes(health.state)) { const occupancy = await probePlayerEvidence(job.server); if (!stillActive(job)) return false; - const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s + const recoveredWatchdog = + health.healthy && + occupancy.players !== null && + db() + .prepare( + `SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id - WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id); - if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; } + WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'` + ) + .get(job.id); + if (recoveredWatchdog && occupancy.players !== null) { + resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); + return false; + } if (occupancy.players !== 0) { if (job.action === 'destroy' && scalingIntent(job.id)) cancelRemoval(job.id, false); - else hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); + else + hold( + job, + occupancy.players !== null + ? `Waiting for ${occupancy.players} player(s) to leave.` + : 'Waiting for a fresh local player count.' + ); return false; } } return true; }; - if (!migration && job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return; - const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; - if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return; + if ( + !migration && + job.action !== 'create' && + job.action !== 'start' && + !(await checkOccupancy()) + ) + return; + const current = db() + .prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL') + .get(job.id) as { state: string } | undefined; + if ( + !current || + !['queued', 'waiting', 'interrupted'].includes(current.state) || + eligibility(job) + ) + return; let prepared: PreparedNativeLaunch | undefined; let preparedKey: string | null = null; let preparedPassword: string | undefined; try { if (job.action === 'create' || job.action === 'rebuild') { - if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) { - hold(job, 'Waiting for startup capacity.'); return; + if ( + count(['create', 'rebuild']) + otherPreparations(job.id) >= + fleetSettings().createConcurrency + ) { + hold(job, 'Waiting for startup capacity.'); + return; } const capacityBlocker = concurrencyBlocker(job.id, job.action, instance.host_id); - if (capacityBlocker) { hold(job, capacityBlocker); return; } + if (capacityBlocker) { + hold(job, capacityBlocker); + return; + } preparingJobs.add(job.id); preparedKey = getServerKey(job.server) ?? null; if (!preparedKey) throw new Error('The saved server key was removed.'); const frozenRecipeId = migration && isRebalanceMigration(job.id) ? job.id : undefined; - const prior = (frozenRecipeId ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe } - : recipeForDeployment(job.id)!.recipe) as NativeRecipe; - deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', - checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); + const prior = ( + frozenRecipeId + ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe } + : recipeForDeployment(job.id)!.recipe + ) as NativeRecipe; + deploymentStage(job.id, 'Preparing launch files for the host agent', { + phase: 'prepare', + checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } + }); preparedPassword = config.rcon.password || randomBytes(32).toString('hex'); - prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native, - instance, serverKey: preparedKey, rconPassword: preparedPassword, - rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, - unhealthyRecovery: host.inventory.capabilities?.includes(unhealthyRecoveryCapability), frozenRecipeId }); - if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return; + prepared = await prepareNativeLaunch({ + game: prior.game, + mapSlug: prior.mapSlug, + native: prior.native, + instance, + serverKey: preparedKey, + rconPassword: preparedPassword, + rebuild: job.action === 'rebuild' && !migration, + safetyOverride: overrideSafety, + unhealthyRecovery: host.inventory.capabilities?.includes( + unhealthyRecoveryCapability + ), + frozenRecipeId + }); + if (job.action === 'rebuild' && !migration && !(await checkOccupancy())) return; } const commit = db().transaction(() => { - const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; - if (!state || !['queued','waiting','interrupted'].includes(state.state) || eligibility(job)) return; + const state = db() + .prepare( + 'SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL' + ) + .get(job.id) as { state: string } | undefined; + if ( + !state || + !['queued', 'waiting', 'interrupted'].includes(state.state) || + eligibility(job) + ) + return; const live = nativeInstance(job.server); - if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.'); + if (live?.id !== instance!.id || live.generation !== instance!.generation) + throw new Error('Instance changed during preflight.'); const liveHost = getHost(instance!.host_id); - if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; } + if (!liveHost?.connected || liveHost.mode === 'disabled') { + hold(job, 'Host is unavailable; waiting for reconnection.'); + return; + } // Confirmation can arrive while preflight or file preparation is awaiting I/O. if (!!safetyOverride(job.id) !== overrideSafety) return; - if (overrideSafety && !liveHost.inventory.capabilities?.includes(safetyOverrideCapability)) { - hold(job, 'Update this host’s agent before using admin override.'); return; + if ( + overrideSafety && + !liveHost.inventory.capabilities?.includes(safetyOverrideCapability) + ) { + hold(job, 'Update this host’s agent before using admin override.'); + return; } if (job.action === 'destroy' || job.action === 'rebuild') { const beta = activeBetaRemovalBlocker(job.server); - if (beta) { hold(job, beta); return; } + if (beta) { + hold(job, beta); + return; + } } - if (JSON.stringify(filesFor(job).map(fileMetadata)) !== JSON.stringify(files.map(fileMetadata))) { - finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return; + if ( + JSON.stringify(filesFor(job).map(fileMetadata)) !== + JSON.stringify(files.map(fileMetadata)) + ) { + finishDeployment(job.id, { + ok: false, + message: + 'File requests changed during preflight. Review and submit a fresh activation.' + }); + return; } const capacityBlocker = concurrencyBlocker(job.id, job.action, instance!.host_id); - if (capacityBlocker) { hold(job, capacityBlocker); return; } - // Async occupancy checks can overlap another job claiming the final slot. - if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) { - deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; + if (capacityBlocker) { + hold(job, capacityBlocker); + return; + } + // Async occupancy checks can overlap another job claiming the final slot. + if ( + (job.action === 'create' || job.action === 'rebuild') && + count(['create', 'rebuild']) + otherPreparations(job.id) >= + fleetSettings().createConcurrency + ) { + deploymentStage(job.id, 'Waiting for startup capacity.', { + phase: 'startup-queue', + blocker: { + code: 'startup-capacity', + message: 'Waiting for startup capacity.', + automatic: true + } + }); + return; + } + if ( + job.action === 'destroy' && + count(['destroy']) >= fleetSettings().destroyConcurrency + ) { + hold(job, 'Waiting for destruction capacity.'); + return; } - if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } if (prepared) { // Compare the same runtime config bytes while rechecking current promoted files. - const expected = migration && isRebalanceMigration(job.id) - ? { format: 1 as const, ...recipeSummaryForDeployment(job.id)!.recipe } - : captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword); - if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe, expected)) { - hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return; + const expected = + migration && isRebalanceMigration(job.id) + ? { format: 1 as const, ...recipeSummaryForDeployment(job.id)!.recipe } + : captureRecipePlan( + prepared.recipe.game, + prepared.recipe.mapSlug, + preparedPassword + ); + if ( + getServerKey(job.server) !== preparedKey || + !sameLaunchFiles(prepared.recipe, expected) + ) { + hold( + job, + 'Launch files or credentials changed during preparation; preparing the current selection next pass.' + ); + return; } assertReadyLibrary(instance!.host_id, instance!.library_id); - if (job.action === 'create' || migration) choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image, - bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true }); + if (job.action === 'create' || migration) + choosePlacement({ + hostId: instance!.host_id, + libraryId: instance!.library_id, + image: instance!.image, + bindAddress: instance!.bind_address, + excludeInstanceId: instance!.id, + thorough: false, + dispatch: true + }); } if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') { - db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) - DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id); + db() + .prepare( + `INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) + DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL` + ) + .run(job.server, new Date().toISOString(), job.id); } - recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: overrideSafety - ? 'Host eligibility checked; player safety checks bypassed by admin override' : 'Host eligibility and safety checks passed' } }); + recordProgress(job.id, { + blocker: null, + checkpoint: { + key: 'eligibility', + label: overrideSafety + ? 'Host eligibility checked; player safety checks bypassed by admin override' + : 'Host eligibility and safety checks passed' + } + }); const payload: Record = files.length ? { stageId: job.id } : {}; if (overrideSafety) payload.safetyOverride = true; if (job.action === 'destroy') { const authorization = removalAuthorization(job.id, instance!.host_id); if (authorization) { - if (overrideSafety) throw new Error('Autoscaling never bypasses player safety.'); + if (overrideSafety) + throw new Error('Autoscaling never bypasses player safety.'); payload.autoscaling = authorization; } } if (prepared) { - deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted', - checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); + deploymentStage(job.id, 'Preparing launch files for the host agent', { + phase: 'prepare', + recipeStatus: 'attempted', + checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } + }); for (const blob of prepared.blobs ?? []) putFileBlob(blob.content); const recipe = prepared.recipe; - db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?') - .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id); - if (job.action === 'create') db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) - VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), - recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port); - for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create'); - recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } }); + db() + .prepare( + 'UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?' + ) + .run( + Buffer.from( + prepared.recipeJson.buffer, + prepared.recipeJson.byteOffset, + prepared.recipeJson.byteLength + ), + prepared.recipeSummary, + job.id + ); + if (job.action === 'create') + db() + .prepare( + `INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) + VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')` + ) + .run( + job.server, + recipe.game, + recipe.expectedMap, + recipe.mapSlug, + serverDir(job.server), + recipe.configFile, + recipe.game === 't6' ? 't6zm' : 't5sp', + recipe.image, + instance!.port + ); + for (const file of recipe.files) + recordScript( + job.server, + file.path, + file.versionId, + file.sha256, + file.size, + 'native-create' + ); + recordProgress(job.id, { + checkpoint: { + key: 'prepare', + label: 'Launch files selected and command payload prepared' + } + }); } - const attempt = migration ? (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations - WHERE host_id=? AND instance_id=? AND generation=? AND kind='instance.create'`).get(instance!.host_id, instance!.id, instance!.generation) as { n: number }).n : 0; - const operationOptions = { instanceId: instance!.id, generation: instance!.generation, - idempotencyKey: migration ? `${job.id}:migration-create:${attempt}` : job.id }; + const attempt = migration + ? ( + db() + .prepare( + `SELECT COUNT(*) AS n FROM ec_host_operations + WHERE host_id=? AND instance_id=? AND generation=? AND kind='instance.create'` + ) + .get(instance!.host_id, instance!.id, instance!.generation) as { + n: number; + } + ).n + : 0; + const operationOptions = { + instanceId: instance!.id, + generation: instance!.generation, + idempotencyKey: migration ? `${job.id}:migration-create:${attempt}` : job.id + }; const kind = `instance.${migration ? 'create' : job.action === 'destroy' ? 'remove' : job.action}`; - const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind, - Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions, - { name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id }) + const operation = prepared + ? queueHostOperationSerialized( + instance!.host_id, + kind, + Buffer.from( + prepared.payloadJson.buffer, + prepared.payloadJson.byteOffset, + prepared.payloadJson.byteLength + ), + operationOptions, + { + name: instance!.server, + game: prepared.recipe.game, + libraryId: instance!.library_id + } + ) : queueHostOperation(instance!.host_id, kind, payload, operationOptions); - db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id); - for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id); - db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); - deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null, - command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } }); + db() + .prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?') + .run(operation.id, job.id); + for (const file of files) + db() + .prepare( + "UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'" + ) + .run(file.id); + db() + .prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?") + .run(new Date().toISOString(), job.id); + deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { + phase: 'container-action', + blocker: null, + command: { + state: 'attempted', + message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` + } + }); }); if (prepared) await commitPreparedLaunch(commit); else commit(); - } finally { preparingJobs.delete(job.id); } + } finally { + preparingJobs.delete(job.id); + } } diff --git a/src/lib/server/provisioning/progress.ts b/src/lib/server/provisioning/progress.ts index c476207..d99d37c 100644 --- a/src/lib/server/provisioning/progress.ts +++ b/src/lib/server/provisioning/progress.ts @@ -1,11 +1,26 @@ import { db } from '../db/index.js'; import { invalidateFleet } from '../pipeline/bus.js'; -import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck, - type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js'; +import { + provisioningPhases, + type ProvisioningPhase, + type ProvisioningProgress, + type ProvisioningCheck, + type ProvisioningCheckpoint, + type ProvisioningBlocker, + type RecoveryObservation +} from '../../provisioning-detail.js'; -export type CheckInput = Omit & - Partial>; -export type CheckpointInput = Omit & { at?: string; phase?: ProvisioningPhase | null }; +export type CheckInput = Omit< + ProvisioningCheck, + 'checkedAt' | 'receivedAt' | 'expected' | 'actual' | 'message' +> & + Partial< + Pick + >; +export type CheckpointInput = Omit & { + at?: string; + phase?: ProvisioningPhase | null; +}; export interface ProgressUpdate { phase?: ProvisioningPhase | null; blocker?: ProvisioningBlocker | null; @@ -20,35 +35,112 @@ export interface ProgressUpdate { event?: { kind: string; message: string; details?: Record }; } -const object = (value: unknown): value is Record => !!value && typeof value === 'object' && !Array.isArray(value); +const object = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); const nullableString = (value: unknown) => value === null || typeof value === 'string'; -const phase = (value: unknown): value is ProvisioningPhase => provisioningPhases.includes(value as ProvisioningPhase); -const scalar = (value: unknown) => nullableString(value) || typeof value === 'number' && Number.isFinite(value); +const phase = (value: unknown): value is ProvisioningPhase => + provisioningPhases.includes(value as ProvisioningPhase); +const scalar = (value: unknown) => + nullableString(value) || (typeof value === 'number' && Number.isFinite(value)); /** Invalid/old snapshots never manufacture evidence. Raw legacy stages remain available. */ -export function readProgress(row: { observation_json?: string | null }): ProvisioningProgress | null { +export function readProgress(row: { + observation_json?: string | null; +}): ProvisioningProgress | null { if (!row.observation_json) return null; try { const v: unknown = JSON.parse(row.observation_json); - if (!object(v) || v.version !== 1 || !(v.phase === null || phase(v.phase)) || !nullableString(v.phaseStartedAt) - || !nullableString(v.observedAt) || !Array.isArray(v.checks) || !Array.isArray(v.checkpoints)) return null; - if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string' - || typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href)) - || (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null; - if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key)) - && typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result)) - && (c.source === null || ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source))) - && typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null; - if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string' - && typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null; - if (v.command !== null && (!object(v.command) || !['attempted', 'accepted', 'rejected', 'uncertain'].includes(String(v.command.state)) - || typeof v.command.at !== 'string' || typeof v.command.message !== 'string')) return null; - if (v.recovery !== null && (!object(v.recovery) || !['held', 'released', 'transferred', 'unknown'].includes(String(v.recovery.state)) - || typeof v.recovery.at !== 'string' || typeof v.recovery.message !== 'string' - || (v.recovery.operationId !== undefined && !nullableString(v.recovery.operationId)))) return null; - if (v.recipeStatus !== null && v.recipeStatus !== 'queued' && v.recipeStatus !== 'attempted') return null; + if ( + !object(v) || + v.version !== 1 || + !(v.phase === null || phase(v.phase)) || + !nullableString(v.phaseStartedAt) || + !nullableString(v.observedAt) || + !Array.isArray(v.checks) || + !Array.isArray(v.checkpoints) + ) + return null; + if ( + v.blocker !== null && + (!object(v.blocker) || + typeof v.blocker.code !== 'string' || + typeof v.blocker.message !== 'string' || + typeof v.blocker.automatic !== 'boolean' || + (v.blocker.href !== undefined && !nullableString(v.blocker.href)) || + (v.blocker.label !== undefined && !nullableString(v.blocker.label))) + ) + return null; + if ( + v.checks.length > 10 || + !v.checks.every( + (c) => + object(c) && + [ + 'container', + 'game', + 'map', + 'plutonium-release', + 'plutonium-version', + 'occupancy', + 'stopped' + ].includes(String(c.key)) && + typeof c.label === 'string' && + ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result)) && + (c.source === null || + ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes( + String(c.source) + )) && + typeof c.checkedAt === 'string' && + nullableString(c.receivedAt) && + scalar(c.expected) && + scalar(c.actual) && + nullableString(c.message) + ) + ) + return null; + if ( + v.checkpoints.length > 100 || + !v.checkpoints.every( + (c) => + object(c) && + typeof c.key === 'string' && + typeof c.label === 'string' && + typeof c.at === 'string' && + (c.phase === null || phase(c.phase)) + ) + ) + return null; + if ( + v.command !== null && + (!object(v.command) || + !['attempted', 'accepted', 'rejected', 'uncertain'].includes( + String(v.command.state) + ) || + typeof v.command.at !== 'string' || + typeof v.command.message !== 'string') + ) + return null; + if ( + v.recovery !== null && + (!object(v.recovery) || + !['held', 'released', 'transferred', 'unknown'].includes( + String(v.recovery.state) + ) || + typeof v.recovery.at !== 'string' || + typeof v.recovery.message !== 'string' || + (v.recovery.operationId !== undefined && !nullableString(v.recovery.operationId))) + ) + return null; + if ( + v.recipeStatus !== null && + v.recipeStatus !== 'queued' && + v.recipeStatus !== 'attempted' + ) + return null; return v as unknown as ProvisioningProgress; - } catch { return null; } + } catch { + return null; + } } /** @@ -58,15 +150,26 @@ export function readProgress(row: { observation_json?: string | null }): Provisi export function recordProgress(id: string, update: ProgressUpdate): void { const handle = db(); handle.transaction(() => { - const row = handle.prepare('SELECT observation_json FROM server_deployments WHERE id = ?').get(id) as - { observation_json: string | null } | undefined; + const row = handle + .prepare('SELECT observation_json FROM server_deployments WHERE id = ?') + .get(id) as { observation_json: string | null } | undefined; if (!row) return; const now = new Date().toISOString(); const previous = readProgress(row); - const next: ProvisioningProgress = previous ? { ...previous, checkpoints: [...previous.checkpoints] } : { - version: 1, phase: null, phaseStartedAt: null, observedAt: null, blocker: null, checks: [], checkpoints: [], - command: null, recovery: null, recipeStatus: null - }; + const next: ProvisioningProgress = previous + ? { ...previous, checkpoints: [...previous.checkpoints] } + : { + version: 1, + phase: null, + phaseStartedAt: null, + observedAt: null, + blocker: null, + checks: [], + checkpoints: [], + command: null, + recovery: null, + recipeStatus: null + }; const events: { kind: string; message: string; details?: Record }[] = []; if (update.phase !== undefined && update.phase !== next.phase) { next.phase = update.phase; @@ -78,45 +181,99 @@ export function recordProgress(id: string, update: ProgressUpdate): void { const old = next.blocker; next.blocker = update.blocker; // Counts/error wording can change on every poll without a new kind of hold. - if (update.blocker && (old?.code !== update.blocker.code || old?.href !== update.blocker.href)) { - events.push({ kind: 'blocker', message: update.blocker.message, - details: { code: update.blocker.code, href: update.blocker.href ?? null } }); + if ( + update.blocker && + (old?.code !== update.blocker.code || old?.href !== update.blocker.href) + ) { + events.push({ + kind: 'blocker', + message: update.blocker.message, + details: { code: update.blocker.code, href: update.blocker.href ?? null } + }); } } if (update.checks !== undefined) { - next.checks = update.checks.map(check => ({ ...check, checkedAt: check.checkedAt ?? now, - receivedAt: check.receivedAt ?? null, expected: check.expected ?? null, actual: check.actual ?? null, - message: check.message ?? null })); + next.checks = update.checks.map((check) => ({ + ...check, + checkedAt: check.checkedAt ?? now, + receivedAt: check.receivedAt ?? null, + expected: check.expected ?? null, + actual: check.actual ?? null, + message: check.message ?? null + })); } - for (const input of [...(update.checkpoints ?? []), ...(update.checkpoint ? [update.checkpoint] : [])]) { - if (next.checkpoints.some(c => c.key === input.key)) continue; - const point: ProvisioningCheckpoint = { ...input, at: input.at ?? now, phase: input.phase ?? (phase(input.key) ? input.key : next.phase) }; + for (const input of [ + ...(update.checkpoints ?? []), + ...(update.checkpoint ? [update.checkpoint] : []) + ]) { + if (next.checkpoints.some((c) => c.key === input.key)) continue; + const point: ProvisioningCheckpoint = { + ...input, + at: input.at ?? now, + phase: input.phase ?? (phase(input.key) ? input.key : next.phase) + }; next.checkpoints.push(point); - events.push({ kind: 'checkpoint', message: point.label, details: { key: point.key, at: point.at } }); + events.push({ + kind: 'checkpoint', + message: point.label, + details: { key: point.key, at: point.at } + }); } if (update.command !== undefined) { if (!update.command) next.command = null; - else if (next.command?.state !== update.command.state || next.command?.message !== update.command.message) { + else if ( + next.command?.state !== update.command.state || + next.command?.message !== update.command.message + ) { next.command = { ...update.command, at: update.command.at ?? now }; - events.push({ kind: 'command', message: next.command.message, details: { state: next.command.state, at: next.command.at } }); + events.push({ + kind: 'command', + message: next.command.message, + details: { state: next.command.state, at: next.command.at } + }); } } if (update.recovery !== undefined) { if (!update.recovery) next.recovery = null; else { - const changed = next.recovery?.state !== update.recovery.state || next.recovery?.operationId !== update.recovery.operationId; - next.recovery = { ...update.recovery, at: changed ? update.recovery.at ?? now : next.recovery!.at }; - if (changed) events.push({ kind: 'recovery', message: next.recovery.message, - details: { state: next.recovery.state, at: next.recovery.at, operationId: next.recovery.operationId ?? null } }); + const changed = + next.recovery?.state !== update.recovery.state || + next.recovery?.operationId !== update.recovery.operationId; + next.recovery = { + ...update.recovery, + at: changed ? (update.recovery.at ?? now) : next.recovery!.at + }; + if (changed) + events.push({ + kind: 'recovery', + message: next.recovery.message, + details: { + state: next.recovery.state, + at: next.recovery.at, + operationId: next.recovery.operationId ?? null + } + }); } } if (update.recipeStatus !== undefined) next.recipeStatus = update.recipeStatus; if (update.event) events.push(update.event); - handle.prepare(`UPDATE server_deployments SET phase_key = ?, phase_started_at = ?, observed_at = ?, observation_json = ? WHERE id = ?`) + handle + .prepare( + `UPDATE server_deployments SET phase_key = ?, phase_started_at = ?, observed_at = ?, observation_json = ? WHERE id = ?` + ) .run(next.phase, next.phaseStartedAt, next.observedAt, JSON.stringify(next), id); - const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json) + const insert = + handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json) VALUES (?, ?, ?, ?, ?, ?)`); - for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null); + for (const event of events) + insert.run( + id, + now, + event.message, + event.kind, + next.phase, + event.details ? JSON.stringify(event.details) : null + ); if (events.length) invalidateFleet(); })(); } diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 460b796..0e56817 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -3,21 +3,38 @@ import { currentRequestUsername } from '../auth/request-context.js'; import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; -import { beginDeployment, canRetryWithRebuild, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; +import { + beginDeployment, + canRetryWithRebuild, + deploymentStage, + finishDeployment, + recordDeploymentEvent +} from './history.js'; import { queueRebuildServer } from './instances.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; import { recordProgress } from './progress.js'; import { nativeInstance, unplacedNativeCreation } from '../backends/native.js'; export interface LifecycleOperation { - id: string; server: string; action: string; state: string; stage: string; - submitted_at: string | null; finished_at: string | null; checked_at: string | null; - players: number | null; error: string | null; + id: string; + server: string; + action: string; + state: string; + stage: string; + submitted_at: string | null; + finished_at: string | null; + checked_at: string | null; + players: number | null; + error: string | null; started_at: string; } -export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string } +export interface RestartSource { + kind: 'campaign' | 'beta' | 'maintenance'; + id: string; +} export function provisioningOperation(id: string): LifecycleOperation | undefined { - return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined; + return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as + LifecycleOperation | undefined; } const sourceTables = { @@ -30,12 +47,17 @@ const sourceTables = { export function requestRestart(server: string, source: RestartSource): LifecycleOperation { return db().transaction(() => { const { table, key } = sourceTables[source.kind]; - const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`) - .get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined; + const row = db() + .prepare( + `SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL` + ) + .get(source.id, server) as + { provisioning_id: string | null; restarted_at: string | null } | undefined; if (!row) throw new Error('Restart source no longer exists'); if (row.provisioning_id) { const operation = provisioningOperation(row.provisioning_id); - if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying'); + if (!operation) + throw new Error('Provisioning job is missing; inspect the server before retrying'); if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') { adoptSubmittedRestart(operation.id, server, row.restarted_at); return provisioningOperation(operation.id)!; @@ -46,68 +68,167 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle if (row.restarted_at) { // The previous release already recorded some submissions in history. // Bind to that exact interval, never to a newer restart of the server. - id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id - WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`) - .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id; - if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id); + id = ( + db() + .prepare( + `SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id + WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?` + ) + .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined + )?.id; + if (id) + db() + .prepare( + 'UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?' + ) + .run(row.restarted_at, id); } if (!id) { id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, - blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); + deploymentStage(id, 'Waiting to verify the server is empty', { + phase: 'eligibility', + observedAt: null, + blocker: { + code: 'eligibility', + message: 'Waiting for eligibility and a fresh empty or stopped server.', + automatic: true + } + }); if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at); } - db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)') + db() + .prepare( + 'INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)' + ) .run(id, source.kind, source.id, server); - db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server); + db() + .prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`) + .run(id, source.id, server); return provisioningOperation(id)!; })(); } /** Upgrade observation only: existing restart intent is never resubmitted. */ function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void { - const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as - { operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined; - db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id); - recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } }); - if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) { + const slot = db() + .prepare( + 'SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?' + ) + .get(server) as + | { + operation_id: string | null; + requested_at: string; + checked_at: string | null; + healthy_at: string | null; + players: number | null; + } + | undefined; + db() + .prepare( + "UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?" + ) + .run(submittedAt, id); + recordProgress(id, { + command: { + state: 'attempted', + at: submittedAt, + message: 'Adopted a previously recorded submission; no command is replayed.' + } + }); + if ( + slot && + ((slot.operation_id && slot.requested_at >= submittedAt) || + (slot.requested_at < submittedAt && !slot.healthy_at)) + ) { const finishedAt = new Date().toISOString(); - const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.'; - db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?") + const error = + 'Existing restart cannot be correlated safely. Inspect the server before retrying.'; + db() + .prepare( + "UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?" + ) .run(finishedAt, error, id); - recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id }, - observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } }); + recordProgress(id, { + recovery: { + state: 'unknown', + message: 'Existing restart recovery could not be correlated with this operation.', + operationId: id + }, + observedAt: null, + event: { + kind: 'outcome', + message: error, + details: { state: 'interrupted', finishedAt, error } + } + }); return; } if (slot && slot.requested_at >= submittedAt) { - db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server); + db() + .prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?') + .run(id, server); } else { - db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) + db() + .prepare( + `INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id, - checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id); + checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL` + ) + .run(server, submittedAt, id); } if (slot?.healthy_at && slot.requested_at >= submittedAt) { - db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id); - recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.', - details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } }); + db() + .prepare( + "UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?" + ) + .run(slot.healthy_at, slot.players, id); + recordProgress(id, { + observedAt: slot.healthy_at, + event: { + kind: 'outcome', + message: 'Adopted recorded recovery completion.', + details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } + } + }); } deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', { - phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null, - recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held', + phase: 'verify-container', + checks: [], + blocker: null, + observedAt: slot?.checked_at ?? null, + recovery: { + state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held', at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined, - message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id } + message: + slot?.healthy_at && slot.requested_at >= submittedAt + ? 'Adopted recorded recovery completion.' + : 'Observing the previously submitted restart without replaying it.', + operationId: id + } }); } export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean { return db().transaction(() => { const finishedAt = new Date().toISOString(); - const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL + const result = db() + .prepare( + `UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL - AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)`) + AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)` + ) .run(finishedAt, reason, id); - if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } }); + if (result.changes) + recordProgress(id, { + blocker: null, + checks: [], + event: { + kind: 'outcome', + message: reason, + details: { state: 'cancelled', finishedAt, error: null, warnings: [] } + } + }); return result.changes > 0; })(); } @@ -115,191 +236,420 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester' /** Explicit caller retry starts a new job and keeps the old job as history. */ export function releaseRestartRequest(server: string, source: RestartSource): void { const { table, key } = sourceTables[source.kind]; - const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined; - if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt'); - db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server); + const row = db() + .prepare( + `SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL` + ) + .get(source.id, server) as { provisioning_id: string | null } | undefined; + if (row?.provisioning_id) + cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt'); + db() + .prepare( + `UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL` + ) + .run(source.id, server); } export function requestLifecycle(server: string, action: RuntimeAction) { - if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; + if (!runtimeActions.includes(action)) + return { + ok: false as const, + status: 400, + error: 'Choose start, restart, rebuild, or stop.' + }; // Restart and rebuild preserve the lineup; their pending work holds scaling decisions. const owned = action === 'start' || action === 'stop' ? scalingOwnershipBlocker(server) : null; if (owned) return { ok: false as const, status: 409, error: owned }; - if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' }; + if (!nativeInstance(server) && !unplacedNativeCreation(server)) + return { ok: false as const, status: 503, error: 'Server has no native host instance.' }; return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; if (blocker) return { ok: false as const, status: 409, error: blocker }; - const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ? + const existing = db() + .prepare( + `SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as - { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined; + AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1` + ) + .get(server) as + | { + id: string; + action: string; + state: string; + submitted_at: string | null; + triggered_by: string; + } + | undefined; if (existing) { - if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; - db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); - if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:') - && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id - WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) { + if (existing.action !== action) + return { + ok: false as const, + status: 409, + error: 'Another provisioning operation is already pending for this server.' + }; + db() + .prepare('UPDATE servers SET desired_state = ? WHERE name = ?') + .run(action === 'stop' ? 'stopped' : 'running', server); + if ( + existing.state === 'waiting' && + !existing.submitted_at && + existing.triggered_by.startsWith('watchdog:') && + db() + .prepare( + `SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id + WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'` + ) + .get(existing.id) + ) { // An operator explicitly requesting this queued action keeps // that intent even if the original health condition clears. - db().prepare("UPDATE server_deployments SET triggered_by = 'ui', requested_by = ? WHERE id = ?") + db() + .prepare( + "UPDATE server_deployments SET triggered_by = 'ui', requested_by = ? WHERE id = ?" + ) .run(currentRequestUsername(), existing.id); - recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action'); + recordDeploymentEvent( + existing.id, + 'Operator explicitly requested the queued health recovery action' + ); } return { ok: true as const, deploymentId: existing.id }; } const row = getServer(server); - if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' }; + if (!row || row.destroyed_at) + return { ok: false as const, status: 404, error: 'Server not found.' }; if (action === 'rebuild') { const result = queueRebuildServer(server); - if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server); - return result.ok ? { ok: true as const, deploymentId: result.deploymentId } - : { ok: false as const, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; + if (result.ok) + db() + .prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?") + .run(server); + return result.ok + ? { ok: true as const, deploymentId: result.deploymentId } + : { + ok: false as const, + status: result.kind === 'conflict' ? 409 : 400, + error: result.message + }; } const id = beginDeployment(server, action); - db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); + db() + .prepare('UPDATE servers SET desired_state = ? WHERE name = ?') + .run(action === 'stop' ? 'stopped' : 'running', server); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, - blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); + deploymentStage( + id, + action === 'start' + ? 'Waiting to start the container' + : 'Waiting to verify the server is empty', + { + phase: 'eligibility', + observedAt: null, + blocker: { + code: 'eligibility', + message: + action === 'start' + ? 'Waiting for start eligibility and recovery capacity.' + : 'Waiting for eligibility and a fresh empty or stopped server.', + automatic: true + } + } + ); return { ok: true as const, deploymentId: id }; })(); } -type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; +type LifecycleRequestResult = + { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; /** Keep the failed launch immutable and let normal rebuild admission own recovery. */ export function requestRetryWithRebuild(id: string): LifecycleRequestResult { return db().transaction(() => { - if (!canRetryWithRebuild(id)) return { ok: false as const, status: 409, - error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' }; + if (!canRetryWithRebuild(id)) + return { + ok: false as const, + status: 409, + error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' + }; const original = provisioningOperation(id)!; const result = requestLifecycle(original.server, 'rebuild'); if (!result.ok) return result; - recordProgress(id, { recovery: { state: 'transferred', operationId: result.deploymentId, - message: 'A new rebuild was requested for this server. The failed creation and its error are preserved.' }, - event: { kind: 'retry-requested', message: 'Retry with rebuild requested', details: { operationId: result.deploymentId } } }); - recordProgress(result.deploymentId, { event: { kind: 'retry-requested', - message: 'Rebuild requested after failed creation', details: { previousOperationId: id } } }); + recordProgress(id, { + recovery: { + state: 'transferred', + operationId: result.deploymentId, + message: + 'A new rebuild was requested for this server. The failed creation and its error are preserved.' + }, + event: { + kind: 'retry-requested', + message: 'Retry with rebuild requested', + details: { operationId: result.deploymentId } + } + }); + recordProgress(result.deploymentId, { + event: { + kind: 'retry-requested', + message: 'Rebuild requested after failed creation', + details: { previousOperationId: id } + } + }); return result; })(); } -function watchdogRequestGuard(server: string, incidentId: string): Extract | null { - const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; - if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; +function watchdogRequestGuard( + server: string, + incidentId: string +): Extract | null { + const incident = db() + .prepare( + "SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')" + ) + .get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; + if (!incident) + return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; if (!maintenanceAllowed(incident.kind)) { return { ok: false, status: 409, error: 'This maintenance policy is paused.' }; } const row = getServer(server); if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' }; - if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' }; - if (!nativeInstance(server)) return { ok: false, status: 503, error: 'Configure a host before provisioning.' }; + if (row.desired_state === 'stopped') + return { ok: false, status: 409, error: 'This server is intentionally stopped.' }; + if (!nativeInstance(server)) + return { ok: false, status: 503, error: 'Configure a host before provisioning.' }; return null; } -function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined { - return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id - WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id; +function existingWatchdogAction( + incidentId: string, + action: 'restart' | 'rebuild' +): string | undefined { + return ( + db() + .prepare( + `SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id + WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle` + ) + .get(incidentId, action) as { operation_id: string } | undefined + )?.operation_id; } -function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { +function queueWatchdogAction( + server: string, + action: 'restart' | 'rebuild', + incidentId: string +): LifecycleRequestResult { let id: string; if (action === 'rebuild') { const result = queueRebuildServer(server, `watchdog:${incidentId}`); - if (!result.ok) return { ok: false, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; + if (!result.ok) + return { + ok: false, + status: result.kind === 'conflict' ? 409 : 400, + error: result.message + }; id = result.deploymentId; } else { id = beginDeployment(server, action, `watchdog:${incidentId}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, - blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); + deploymentStage(id, 'Waiting to verify the server is empty', { + phase: 'eligibility', + observedAt: null, + blocker: { + code: 'eligibility', + message: 'Waiting for eligibility and a fresh empty or stopped server.', + automatic: true + } + }); } - db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server); + db() + .prepare( + "INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)" + ) + .run(id, incidentId, server); return { ok: true, deploymentId: id }; } /** Watchdog submits durable work; Provisioning remains the only execution owner. */ -export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { +export function requestWatchdogLifecycle( + server: string, + action: 'restart' | 'rebuild', + incidentId: string +): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, action); if (previous) return { ok: true as const, deploymentId: previous }; const error = watchdogRequestGuard(server, incidentId); if (error) return error; - const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d + const pending = db() + .prepare( + `SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' - WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`) + WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1` + ) .get(server) as { id: string; action: string; source_id: string | null } | undefined; - if (pending) return pending.source_id === incidentId && pending.action === action - ? { ok: true as const, deploymentId: pending.id } - : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; + if (pending) + return pending.source_id === incidentId && pending.action === action + ? { ok: true as const, deploymentId: pending.id } + : { + ok: false as const, + status: 409, + error: 'Another provisioning operation is already pending for this server.' + }; const next = queueWatchdogAction(server, action, incidentId); if (!next.ok) return next; // An operator retry starts a new incident cycle. Keep the previous // failed attempt's capacity, but let that new attempt use it instead // of waiting forever behind its own incident's terminal operation. - const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r + const retained = db() + .prepare( + `SELECT r.operation_id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ? JOIN watchdog_incidents i ON i.id = a.incident_id WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled') - AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined; + AND a.cycle < i.cycle` + ) + .get(incidentId, server) as { operation_id: string } | undefined; if (retained) { - const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') + const transferred = db() + .prepare( + 'UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL' + ) .run(next.deploymentId, server, retained.operation_id); if (transferred.changes) { - recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId, - message: 'Recovery capacity transferred to the explicit new incident attempt.' } }); - recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId, - message: 'Recovery capacity retained from the prior incident attempt.' } }); + recordProgress(retained.operation_id, { + recovery: { + state: 'transferred', + operationId: next.deploymentId, + message: + 'Recovery capacity transferred to the explicit new incident attempt.' + } + }); + recordProgress(next.deploymentId, { + recovery: { + state: 'held', + operationId: next.deploymentId, + message: 'Recovery capacity retained from the prior incident attempt.' + } + }); } - recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId); - recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id); + recordDeploymentEvent( + retained.operation_id, + 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId + ); + recordDeploymentEvent( + next.deploymentId, + 'Retained recovery capacity from prior incident attempt ' + retained.operation_id + ); } return next; })(); } /** Atomically reserve a replacement before handing it the failed restart's capacity. */ -export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult { +export function escalateWatchdogRestart( + incidentId: string, + operationId: string, + recoveryTimeoutMs = 300_000 +): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, 'rebuild'); if (previous) return { ok: true as const, deploymentId: previous }; const operation = provisioningOperation(operationId); - if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state) - || !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) { - return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' }; + if ( + !operation || + operation.action !== 'restart' || + !['awaiting-health', 'failed'].includes(operation.state) || + !db() + .prepare( + "SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?" + ) + .get(operationId, incidentId) + ) { + return { + ok: false as const, + status: 409, + error: 'The restart is not eligible for Watchdog escalation.' + }; } const error = watchdogRequestGuard(operation.server, incidentId); if (error) return error; - const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d + const pending = db() + .prepare( + `SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' - WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`) - .get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined; - if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId - ? { ok: true as const, deploymentId: pending.id } - : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; - const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as - { operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined; - const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at); - if ((slot && (slot.operation_id !== operationId || slot.healthy_at)) - || !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) { - return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' }; + WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1` + ) + .get(operation.server, operationId) as + { id: string; action: string; source_id: string | null } | undefined; + if (pending) + return pending.action === 'rebuild' && pending.source_id === incidentId + ? { ok: true as const, deploymentId: pending.id } + : { + ok: false as const, + status: 409, + error: 'Another provisioning operation is already pending for this server.' + }; + const slot = db() + .prepare( + 'SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?' + ) + .get(operation.server) as + | { operation_id: string | null; requested_at: string; healthy_at: string | null } + | undefined; + const anchor = Date.parse( + slot?.requested_at ?? operation.submitted_at ?? operation.started_at + ); + if ( + (slot && (slot.operation_id !== operationId || slot.healthy_at)) || + !Number.isFinite(anchor) || + Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs) + ) { + return { + ok: false as const, + status: 409, + error: 'Waiting for the submitted restart to finish its recovery window.' + }; } const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId); if (!replacement.ok) return replacement; - const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId; - if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason }); + const reason = + 'Watchdog restart recovery timed out; handed recovery to rebuild ' + + replacement.deploymentId; + if (operation.state === 'awaiting-health') + finishDeployment(operationId, { ok: false, message: reason }); else recordDeploymentEvent(operationId, reason); if (slot) { - const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') + const transferred = db() + .prepare( + 'UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL' + ) .run(replacement.deploymentId, reason, operation.server, operationId); if (transferred.changes) { - recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId, - message: 'Recovery capacity transferred to the rebuild operation.' } }); - recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId, - message: 'Recovery capacity retained from the previous restart.' } }); + recordProgress(operationId, { + recovery: { + state: 'transferred', + operationId: replacement.deploymentId, + message: 'Recovery capacity transferred to the rebuild operation.' + } + }); + recordProgress(replacement.deploymentId, { + recovery: { + state: 'held', + operationId: replacement.deploymentId, + message: 'Recovery capacity retained from the previous restart.' + } + }); } } - recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server'); + recordDeploymentEvent( + replacement.deploymentId, + 'Recovery handed off from restart ' + + operationId + + '; waiting for a fresh empty or stopped server' + ); return replacement; })(); } diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index ef6d487..158851a 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -16,7 +16,8 @@ interface RestartSlot { } function slotFor(server: string): RestartSlot | undefined { - return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; + return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as + RestartSlot | undefined; } export function restartRecoveryPending(server: string): boolean { @@ -24,51 +25,107 @@ export function restartRecoveryPending(server: string): boolean { } type SettlingSlot = Pick; -interface SettlingOperation { id: string; action: string; state: string } +interface SettlingOperation { + id: string; + action: string; + state: string; +} /** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */ -export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean { +export function recoverySettling( + slot: SettlingSlot | undefined, + operation?: SettlingOperation | null +): boolean { if (!slot || slot.healthy_at) return false; - if (slot.operation_id) return operation?.id === slot.operation_id - && ['start', 'restart', 'rebuild'].includes(operation.action) - && ['running', 'awaiting-health'].includes(operation.state); + if (slot.operation_id) + return ( + operation?.id === slot.operation_id && + ['start', 'restart', 'rebuild'].includes(operation.action) && + ['running', 'awaiting-health'].includes(operation.state) + ); // Legacy recovery slots have no operation whose verification can be followed. return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS; } export function restartSettling(server: string): boolean { const slot = slotFor(server); - const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments - WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined; + const operation = slot?.operation_id + ? (db() + .prepare( + `SELECT id, action, state FROM server_deployments + WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1` + ) + .get(server) as SettlingOperation | undefined) + : undefined; return recoverySettling(slot, operation); } -export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null { +export function rolloutWaitReason( + server: string, + operationId = '', + action: RuntimeAction = 'restart' +): string | null { return rolloutBlocker(server, operationId, action)?.message ?? null; } -export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null { - const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL +export function rolloutBlocker( + server: string, + operationId = '', + action: RuntimeAction = 'restart' +): ProvisioningBlocker | null { + const conflict = db() + .prepare( + `SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) - OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined; + OR (action = 'destroy' AND state IN ('waiting','running')))` + ) + .get(server, operationId) as { id: string } | undefined; if (conflict) { - return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true, - href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' }; + return { + code: 'operation-conflict', + message: 'Waiting for the current provisioning operation', + automatic: true, + href: `/provisioning/${encodeURIComponent(conflict.id)}`, + label: 'Blocking operation' + }; } const slot = slotFor(server); - const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d + const ownHandoff = + slot?.operation_id === operationId && + !!db() + .prepare( + `SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS ( - SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId); + SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')` + ) + .get(operationId); if (slot?.healthy_at === null && !ownHandoff) { - return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true, - href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' }; + return { + code: 'previous-recovery', + message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, + automatic: true, + href: slot.operation_id + ? `/provisioning/${encodeURIComponent(slot.operation_id)}` + : null, + label: 'Previous operation' + }; } - const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; - return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency - ? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null; + const { n } = db() + .prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL') + .get() as { n: number }; + return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= + fleetSettings().restartConcurrency + ? { + code: 'restart-capacity', + message: `Queued to ${action}; waiting for another server to recover`, + automatic: true + } + : null; } -export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { +export function rolloutHealth( + server: string +): { ok: true; players: number } | { ok: false; reason: string } { const slot = slotFor(server); if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' }; diff --git a/src/lib/server/provisioning/safety-override.ts b/src/lib/server/provisioning/safety-override.ts index 63e8db6..a889bf7 100644 --- a/src/lib/server/provisioning/safety-override.ts +++ b/src/lib/server/provisioning/safety-override.ts @@ -6,13 +6,24 @@ import type { ProvisioningDetail } from '../../provisioning-detail.js'; export const safetyOverrideCapability = 'instance.safety-override'; export function safetyOverride(id: string): ProvisioningDetail['safetyOverride'] { - return db().prepare(`SELECT actor_id AS actorId, actor_name AS actorName, confirmed_at AS confirmedAt - FROM provisioning_safety_overrides WHERE deployment_id=?`).get(id) as ProvisioningDetail['safetyOverride'] ?? null; + return ( + (db() + .prepare( + `SELECT actor_id AS actorId, actor_name AS actorName, confirmed_at AS confirmedAt + FROM provisioning_safety_overrides WHERE deployment_id=?` + ) + .get(id) as ProvisioningDetail['safetyOverride']) ?? null + ); } /** Approval belongs to one unsubmitted operation and its original live instance. */ -export function safetyOverrideAvailability(id: string): { applicable: boolean; unavailableReason: string | null } { - const target = db().prepare(`SELECT i.host_id FROM server_deployments d +export function safetyOverrideAvailability(id: string): { + applicable: boolean; + unavailableReason: string | null; +} { + const target = db() + .prepare( + `SELECT i.host_id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.generation=n.generation AND i.retired_at IS NULL WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action IN ('restart','stop','rebuild','destroy') @@ -20,26 +31,53 @@ export function safetyOverrideAvailability(id: string): { applicable: boolean; u AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id) AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.cleanup_id=d.id) AND NOT EXISTS (SELECT 1 FROM autoscaling_decisions a WHERE a.operation_id=d.id) - AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)`) + AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)` + ) .get(id) as { host_id: string } | undefined; if (!target) return { applicable: false, unavailableReason: null }; - return { applicable: true, unavailableReason: getHost(target.host_id)?.inventory.capabilities?.includes(safetyOverrideCapability) - ? null : 'Update this host’s agent before using admin override.' }; + return { + applicable: true, + unavailableReason: getHost(target.host_id)?.inventory.capabilities?.includes( + safetyOverrideCapability + ) + ? null + : 'Update this host’s agent before using admin override.' + }; } -export function enableSafetyOverride(id: string, actor: { id: string; username: string }): { ok: true } | { ok: false; error: string } { +export function enableSafetyOverride( + id: string, + actor: { id: string; username: string } +): { ok: true } | { ok: false; error: string } { return db().transaction(() => { // A retried confirmation never sends or repeats a command. if (safetyOverride(id)) return { ok: true as const }; const availability = safetyOverrideAvailability(id); - if (!availability.applicable) return { ok: false as const, error: 'This operation can no longer be overridden. Refresh to see its current state.' }; - if (availability.unavailableReason) return { ok: false as const, error: availability.unavailableReason }; + if (!availability.applicable) + return { + ok: false as const, + error: 'This operation can no longer be overridden. Refresh to see its current state.' + }; + if (availability.unavailableReason) + return { ok: false as const, error: availability.unavailableReason }; const confirmedAt = new Date().toISOString(); - db().prepare('INSERT INTO provisioning_safety_overrides(deployment_id,actor_id,actor_name,confirmed_at) VALUES(?,?,?,?)') + db() + .prepare( + 'INSERT INTO provisioning_safety_overrides(deployment_id,actor_id,actor_name,confirmed_at) VALUES(?,?,?,?)' + ) .run(id, actor.id, actor.username, confirmedAt); - recordProgress(id, { event: { kind: 'safety-override', - message: `Admin override confirmed by ${actor.username}: bypass player safety checks for this operation.`, - details: { actorId: actor.id, actorName: actor.username, confirmedAt, checks: ['player-occupancy'] } } }); + recordProgress(id, { + event: { + kind: 'safety-override', + message: `Admin override confirmed by ${actor.username}: bypass player safety checks for this operation.`, + details: { + actorId: actor.id, + actorName: actor.username, + confirmedAt, + checks: ['player-occupancy'] + } + } + }); return { ok: true as const }; })(); } diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index 0d96024..5bdc72f 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -8,43 +8,110 @@ import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; import { processNativeProvisioning } from './native.js'; -interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string } -const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[]; -const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2; +interface Source { + kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; + source_id: string; +} +const sourcesFor = (id: string) => + db() + .prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?') + .all(id) as Source[]; +const priority = (sources: Source[]) => + sources.some((s) => s.kind === 'watchdog') + ? 2 + : sources.some((s) => s.kind === 'beta') + ? 0 + : sources.some((s) => s.kind === 'campaign') + ? 1 + : sources.some((s) => s.kind === 'maintenance') + ? 3 + : 2; /** Source work may hold or withdraw a request, but never executes it. */ -function sourceEligibility(operation: Operation, onHold?: (source: Source) => void): 'ready' | 'held' | 'cancelled' { +function sourceEligibility( + operation: Operation, + onHold?: (source: Source) => void +): 'ready' | 'held' | 'cancelled' { const sources = sourcesFor(operation.id); if (!sources.length) return 'ready'; // Watchdog intent gates every linked source, including its drift campaign. - for (const source of sources.filter(source => source.kind === 'watchdog')) { - if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server) - || getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; - const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' }; - if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; } + for (const source of sources.filter((source) => source.kind === 'watchdog')) { + if ( + !db() + .prepare( + "SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')" + ) + .get(source.source_id, operation.server) || + getServer(operation.server)?.desired_state === 'stopped' + ) + return 'cancelled'; + const incident = db() + .prepare('SELECT kind FROM watchdog_incidents WHERE id = ?') + .get(source.source_id) as { kind: 'health' | 'drift' }; + if (!maintenanceAllowed(incident.kind)) { + onHold?.(source); + return 'held'; + } } let held = false; let ready = false; for (const source of sources) { if (source.kind === 'campaign') { - const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL') - .get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined; + const row = db() + .prepare( + 'SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL' + ) + .get(source.source_id, operation.server) as + | { restart_state: string; files_changed: number; provisioning_id: string } + | undefined; if (row?.provisioning_id !== operation.id || !row.files_changed) continue; - if (['pending','failed'].includes(row.restart_state)) ready = true; - if (['held','manual'].includes(row.restart_state)) { held = true; onHold?.(source); } + if (['pending', 'failed'].includes(row.restart_state)) ready = true; + if (['held', 'manual'].includes(row.restart_state)) { + held = true; + onHold?.(source); + } } else if (source.kind === 'beta') { - const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`) - .get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined; - if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue; - if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue; - if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true; - if (row.outcome === 'paused') { held = true; onHold?.(source); } + const row = db() + .prepare( + `SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL` + ) + .get(source.source_id, operation.server) as + | { state: string; stage: string; provisioning_id: string; outcome: string | null } + | undefined; + if ( + row?.provisioning_id !== operation.id || + !['running', 'ready-promote'].includes(row.state) + ) + continue; + if (!['awaiting-empty', 'restarting', 'awaiting-health'].includes(row.stage)) continue; + if (!row.outcome || ['active', 'passed'].includes(row.outcome)) ready = true; + if (row.outcome === 'paused') { + held = true; + onHold?.(source); + } } else if (source.kind === 'maintenance') { - const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined; - if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) { + const row = db() + .prepare( + 'SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL' + ) + .get(source.source_id) as + | { state: string; provisioning_id: string; kind: string; origin: string } + | undefined; + if ( + row?.provisioning_id === operation.id && + ['queued', 'restarting'].includes(row.state) + ) { if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; - if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) { onHold?.(source); return 'held'; } + if ( + !maintenanceAllowed() || + (row.kind === 'update' && + row.origin === 'automatic' && + !maintenanceAllowed('updates')) + ) { + onHold?.(source); + return 'held'; + } ready = true; } } else ready = true; @@ -55,56 +122,114 @@ function sourceEligibility(operation: Operation, onHold?: (source: Source) => vo function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null { const sources = sourcesFor(id); const rank = priority(sources); - const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[]; - const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number }; - const conflict = others.find(other => other.state !== 'waiting' || other.action === 'destroy' - || priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid)); + const others = db() + .prepare( + `SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health')` + ) + .all(server, id) as { id: string; state: string; action: string; rowid: number }[]; + const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { + rowid: number; + }; + const conflict = others.find( + (other) => + other.state !== 'waiting' || + other.action === 'destroy' || + priority(sourcesFor(other.id)) < rank || + (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid) + ); if (conflict) { - return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true, - href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' }; + return { + code: 'operation-conflict', + message: 'Waiting for another provisioning operation on this server', + automatic: true, + href: `/provisioning/${encodeURIComponent(conflict.id)}`, + label: 'Blocking operation' + }; } // Beta activation must not wait on the campaign rows that the beta itself // holds. Campaign/maintenance callers likewise never block on their own job. - const beta = rank > 0 ? db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) as { id: string } | undefined : undefined; - if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' }; - const campaign = rank > 1 ? db().prepare(`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL + const beta = + rank > 0 + ? (db() + .prepare( + `SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1` + ) + .get(server) as { id: string } | undefined) + : undefined; + if (beta) + return { + code: 'beta-work', + message: 'Waiting for beta work on this server', + automatic: true, + href: `/betas/${encodeURIComponent(beta.id)}`, + label: 'Blocking beta' + }; + const campaign = + rank > 1 + ? (db() + .prepare( + `SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS ( SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health'))) AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id) - LIMIT 1`).get(server, id) as { deployment_id: string } | undefined : undefined; - if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true, - href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' }; + LIMIT 1` + ) + .get(server, id) as { deployment_id: string } | undefined) + : undefined; + if (campaign) + return { + code: 'campaign-work', + message: 'Waiting for campaign work on this server', + automatic: true, + href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, + label: 'Blocking campaign' + }; return rolloutBlocker(server, id, action); } -interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string } +interface Operation { + id: string; + server: string; + action: RuntimeAction; + state: string; + stage: string; +} let awaitedPass: Promise | null = null; export function processProvisioning(options: { wait?: boolean } = {}): Promise { if (options.wait !== false && awaitedPass) return awaitedPass; // Each job owns its flight, so slow observations do not block new work. const pass = nativePass(options); if (options.wait === false) return pass; - awaitedPass = pass.finally(() => { awaitedPass = null; }); + awaitedPass = pass.finally(() => { + awaitedPass = null; + }); return awaitedPass; } export const processLifecycle = processProvisioning; function nativePass(options: { wait?: boolean }): Promise { - return processNativeProvisioning(operation => { + return processNativeProvisioning((operation) => { const scaling = scalingAdmission(operation.id); if (scaling) return scaling; if (operation.action === 'create') return null; if (operation.action === 'destroy') { - const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL - AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1`).get(operation.server, operation.id); + const other = db() + .prepare( + `SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL + AND (state='running' OR (state='awaiting-health' AND action NOT IN ('create','rebuild'))) LIMIT 1` + ) + .get(operation.server, operation.id); return other ? 'Waiting for the current server operation.' : null; } const typed = operation as Operation; const eligibility = sourceEligibility(typed); - if (eligibility === 'cancelled') { cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); return 'Requester cancelled this operation.'; } + if (eligibility === 'cancelled') { + cancelProvisioning(operation.id, 'Requester cancelled or replaced this operation'); + return 'Requester cancelled this operation.'; + } if (eligibility === 'held') return 'Waiting for the requester to resume.'; return blocker(operation.server, operation.id, typed.action)?.message ?? null; }, options); diff --git a/src/lib/server/rebalancing/admission.ts b/src/lib/server/rebalancing/admission.ts index 62ca32c..119cc1d 100644 --- a/src/lib/server/rebalancing/admission.ts +++ b/src/lib/server/rebalancing/admission.ts @@ -5,22 +5,37 @@ import { rebalanceSnapshot } from './snapshot.js'; /** A delivered migration must finish even after the operator stops its parent run. */ export function rebalanceAdmission(operationId: string): string | null { - const operation = db().prepare('SELECT triggered_by FROM server_deployments WHERE id=?').get(operationId) as { triggered_by: string } | undefined; + const operation = db() + .prepare('SELECT triggered_by FROM server_deployments WHERE id=?') + .get(operationId) as { triggered_by: string } | undefined; if (!operation?.triggered_by.startsWith('rebalance:')) return null; - const migration = db().prepare('SELECT remove_operation_id,removed_at FROM ec_host_migrations WHERE deployment_id=?') - .get(operationId) as { remove_operation_id: string | null; removed_at: string | null } | undefined; + const migration = db() + .prepare( + 'SELECT remove_operation_id,removed_at FROM ec_host_migrations WHERE deployment_id=?' + ) + .get(operationId) as + { remove_operation_id: string | null; removed_at: string | null } | undefined; if (migration?.remove_operation_id || migration?.removed_at) return null; - const row = db().prepare(`SELECT m.candidate,r.settings,r.state FROM rebalancing_moves m - JOIN rebalancing_runs r ON r.id=m.run_id WHERE m.operation_id=? AND m.state='moving'`).get(operationId) as - { candidate: string; settings: string; state: string } | undefined; + const row = db() + .prepare( + `SELECT m.candidate,r.settings,r.state FROM rebalancing_moves m + JOIN rebalancing_runs r ON r.id=m.run_id WHERE m.operation_id=? AND m.state='moving'` + ) + .get(operationId) as { candidate: string; settings: string; state: string } | undefined; if (!row || row.state !== 'running') return 'Rebalance run is no longer requesting this move.'; const settings = JSON.parse(row.settings) as RebalanceSettings; - return rebalanceMoveBlocker(rebalanceSnapshot(settings, operationId), JSON.parse(row.candidate) as RebalanceCandidate); + return rebalanceMoveBlocker( + rebalanceSnapshot(settings, operationId), + JSON.parse(row.candidate) as RebalanceCandidate + ); } /** Cancelling an individual uncommitted move also withdraws the rest of that run. */ export function requestRebalanceStopForOperation(operationId: string): void { - db().prepare(`UPDATE rebalancing_runs SET state='stopping',updated_at=?,message='Stopping after the current move.' - WHERE state='running' AND id IN (SELECT run_id FROM rebalancing_moves WHERE operation_id=?)`) + db() + .prepare( + `UPDATE rebalancing_runs SET state='stopping',updated_at=?,message='Stopping after the current move.' + WHERE state='running' AND id IN (SELECT run_id FROM rebalancing_moves WHERE operation_id=?)` + ) .run(new Date().toISOString(), operationId); } diff --git a/src/lib/server/rebalancing/index.ts b/src/lib/server/rebalancing/index.ts index 629bbd2..db59f07 100644 --- a/src/lib/server/rebalancing/index.ts +++ b/src/lib/server/rebalancing/index.ts @@ -3,92 +3,211 @@ import { db } from '../db/index.js'; import { getHost } from '../hosts/index.js'; import { requestInstanceMove } from '../provisioning/migrations.js'; import { cancelProvisioningJob } from '../provisioning/cancellation.js'; -import { rebalanceSettings, type RebalanceCandidate, type RebalanceMove, type RebalancePreview, - type RebalanceReport, type RebalanceRun } from '../../rebalancing.js'; +import { + rebalanceSettings, + type RebalanceCandidate, + type RebalanceMove, + type RebalancePreview, + type RebalanceReport, + type RebalanceRun +} from '../../rebalancing.js'; import { planRebalance, rebalanceMoveBlocker } from './planner.js'; import { rebalanceHosts, rebalanceSnapshot } from './snapshot.js'; import { rebalanceAdmission } from './admission.js'; -export class RebalanceError extends Error { constructor(message: string, public status = 400) { super(message); } } -interface RunRow { id: string; settings: string; state: RebalanceRun['state']; created_at: string; updated_at: string; message: string } -interface MoveRow { id: string; candidate: string; state: RebalanceMove['state']; operation_id: string | null; message: string; updated_at: string } +export class RebalanceError extends Error { + constructor( + message: string, + public status = 400 + ) { + super(message); + } +} +interface RunRow { + id: string; + settings: string; + state: RebalanceRun['state']; + created_at: string; + updated_at: string; + message: string; +} +interface MoveRow { + id: string; + candidate: string; + state: RebalanceMove['state']; + operation_id: string | null; + message: string; + updated_at: string; +} const atNow = () => new Date().toISOString(); -const movesFor = (id: string): RebalanceMove[] => (db().prepare('SELECT * FROM rebalancing_moves WHERE run_id=? ORDER BY ordinal').all(id) as MoveRow[]) - .map(row => ({ ...JSON.parse(row.candidate), id: row.id, state: row.state, operationId: row.operation_id, message: row.message, updatedAt: row.updated_at })); +const movesFor = (id: string): RebalanceMove[] => + ( + db() + .prepare('SELECT * FROM rebalancing_moves WHERE run_id=? ORDER BY ordinal') + .all(id) as MoveRow[] + ).map((row) => ({ + ...JSON.parse(row.candidate), + id: row.id, + state: row.state, + operationId: row.operation_id, + message: row.message, + updatedAt: row.updated_at + })); function present(row: RunRow): RebalanceRun { - return { id: row.id, settings: JSON.parse(row.settings), state: row.state, createdAt: row.created_at, - updatedAt: row.updated_at, message: row.message, moves: movesFor(row.id) }; + return { + id: row.id, + settings: JSON.parse(row.settings), + state: row.state, + createdAt: row.created_at, + updatedAt: row.updated_at, + message: row.message, + moves: movesFor(row.id) + }; } export function getRebalanceRun(id: string): RebalanceRun | null { - const row = db().prepare('SELECT * FROM rebalancing_runs WHERE id=?').get(id) as RunRow | undefined; + const row = db().prepare('SELECT * FROM rebalancing_runs WHERE id=?').get(id) as + RunRow | undefined; return row ? present(row) : null; } export function rebalanceReport(): RebalanceReport { - return { hosts: rebalanceHosts(), runs: (db().prepare(`SELECT * FROM rebalancing_runs - ORDER BY state IN ('running','stopping') DESC,created_at DESC,rowid DESC LIMIT 20`).all() as RunRow[]).map(present) }; + return { + hosts: rebalanceHosts(), + runs: ( + db() + .prepare( + `SELECT * FROM rebalancing_runs + ORDER BY state IN ('running','stopping') DESC,created_at DESC,rowid DESC LIMIT 20` + ) + .all() as RunRow[] + ).map(present) + }; } /** Preview has no resource reservations or host side effects. */ export function previewRebalance(input: unknown): RebalancePreview { - const settings = rebalanceSettings(input), now = Date.now(); + const settings = rebalanceSettings(input), + now = Date.now(); const plan = planRebalance(rebalanceSnapshot(settings), settings); - const preview: RebalancePreview = { ...plan, settings, id: randomUUID(), createdAt: new Date(now).toISOString(), - expiresAt: new Date(now + 5 * 60_000).toISOString() }; - db().prepare('INSERT INTO rebalancing_previews(id,preview,expires_at) VALUES(?,?,?)').run(preview.id, JSON.stringify(preview), preview.expiresAt); + const preview: RebalancePreview = { + ...plan, + settings, + id: randomUUID(), + createdAt: new Date(now).toISOString(), + expiresAt: new Date(now + 5 * 60_000).toISOString() + }; + db() + .prepare('INSERT INTO rebalancing_previews(id,preview,expires_at) VALUES(?,?,?)') + .run(preview.id, JSON.stringify(preview), preview.expiresAt); return preview; } function hasOutstandingRun(): boolean { - return !!db().prepare(`SELECT 1 FROM rebalancing_runs WHERE state IN ('running','stopping') + return !!db() + .prepare( + `SELECT 1 FROM rebalancing_runs WHERE state IN ('running','stopping') UNION ALL SELECT 1 FROM rebalancing_moves m JOIN server_deployments d ON d.id=m.operation_id WHERE d.state IN ('queued','waiting','running','awaiting-health','interrupted') UNION ALL SELECT 1 FROM rebalancing_moves m JOIN native_provisioning n ON n.deployment_id=m.operation_id LEFT JOIN ec_host_migrations migration ON migration.deployment_id=m.operation_id JOIN ec_host_operations o ON o.id IN (n.operation_id,n.stage_operation_id,migration.remove_operation_id) - WHERE o.state NOT IN ('completed','failed') LIMIT 1`).get(); + WHERE o.state NOT IN ('completed','failed') LIMIT 1` + ) + .get(); } -const moveSignature = (moves: RebalanceCandidate[]) => JSON.stringify(moves.map(move => - [move.instanceId, move.generation, move.sourceHostId, move.targetHostId])); +const moveSignature = (moves: RebalanceCandidate[]) => + JSON.stringify( + moves.map((move) => [ + move.instanceId, + move.generation, + move.sourceHostId, + move.targetHostId + ]) + ); export function startRebalance(previewId: string): RebalanceRun { return db().transaction(() => { - const row = db().prepare('SELECT preview,expires_at,run_id FROM rebalancing_previews WHERE id=?').get(previewId) as + const row = db() + .prepare('SELECT preview,expires_at,run_id FROM rebalancing_previews WHERE id=?') + .get(previewId) as { preview: string; expires_at: string; run_id: string | null } | undefined; - if (!row) throw new RebalanceError('Preview not found. Preview the distribution again.', 404); + if (!row) + throw new RebalanceError('Preview not found. Preview the distribution again.', 404); // A retried request returns the original bounded run, even after its preview expires. if (row.run_id) return getRebalanceRun(row.run_id)!; - if (Date.parse(row.expires_at) <= Date.now()) throw new RebalanceError('Preview expired. Preview the distribution again.', 409); - if (hasOutstandingRun()) throw new RebalanceError('Finish or stop the current rebalance and settle its operations before starting another.', 409); + if (Date.parse(row.expires_at) <= Date.now()) + throw new RebalanceError('Preview expired. Preview the distribution again.', 409); + if (hasOutstandingRun()) + throw new RebalanceError( + 'Finish or stop the current rebalance and settle its operations before starting another.', + 409 + ); const preview = JSON.parse(row.preview) as RebalancePreview; const fresh = planRebalance(rebalanceSnapshot(preview.settings), preview.settings); if (!fresh.moves.length || moveSignature(fresh.moves) !== moveSignature(preview.moves)) { - throw new RebalanceError('The fleet changed. Preview the distribution again before starting.', 409); + throw new RebalanceError( + 'The fleet changed. Preview the distribution again before starting.', + 409 + ); } - const id = randomUUID(), at = atNow(); - db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) - VALUES(?,?,'running',?,?,'Waiting to begin the first move.')`).run(id, JSON.stringify(preview.settings), at, at); - for (const [ordinal, move] of fresh.moves.entries()) db().prepare(`INSERT INTO rebalancing_moves + const id = randomUUID(), + at = atNow(); + db() + .prepare( + `INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) + VALUES(?,?,'running',?,?,'Waiting to begin the first move.')` + ) + .run(id, JSON.stringify(preview.settings), at, at); + for (const [ordinal, move] of fresh.moves.entries()) + db() + .prepare( + `INSERT INTO rebalancing_moves (id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at) - VALUES(?,?,?,?,?,?,?,?,?,'planned','Waiting for earlier moves.',?)`) - .run(randomUUID(), id, ordinal, move.server, move.instanceId, move.generation, move.sourceHostId, move.targetHostId, JSON.stringify(move), at); + VALUES(?,?,?,?,?,?,?,?,?,'planned','Waiting for earlier moves.',?)` + ) + .run( + randomUUID(), + id, + ordinal, + move.server, + move.instanceId, + move.generation, + move.sourceHostId, + move.targetHostId, + JSON.stringify(move), + at + ); db().prepare('UPDATE rebalancing_previews SET run_id=? WHERE id=?').run(id, previewId); return getRebalanceRun(id)!; })(); } function updateRun(id: string, state: RebalanceRun['state'], message: string): void { - db().prepare('UPDATE rebalancing_runs SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)') + db() + .prepare( + 'UPDATE rebalancing_runs SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)' + ) .run(state, message, atNow(), id, state, message); } function updateMove(id: string, state: RebalanceMove['state'], message: string): void { - db().prepare('UPDATE rebalancing_moves SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)') + db() + .prepare( + 'UPDATE rebalancing_moves SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)' + ) .run(state, message, atNow(), id, state, message); } function skipRemaining(id: string, reason: string): void { - db().prepare("UPDATE rebalancing_moves SET state='skipped',message=?,updated_at=? WHERE run_id=? AND state='planned'").run(reason, atNow(), id); + db() + .prepare( + "UPDATE rebalancing_moves SET state='skipped',message=?,updated_at=? WHERE run_id=? AND state='planned'" + ) + .run(reason, atNow(), id); } function committed(operationId: string): boolean { - return !!db().prepare('SELECT 1 FROM ec_host_migrations WHERE deployment_id=? AND (remove_operation_id IS NOT NULL OR removed_at IS NOT NULL)').get(operationId); + return !!db() + .prepare( + 'SELECT 1 FROM ec_host_migrations WHERE deployment_id=? AND (remove_operation_id IS NOT NULL OR removed_at IS NOT NULL)' + ) + .get(operationId); } export function stopRebalance(id: string): RebalanceRun { @@ -98,19 +217,23 @@ export function stopRebalance(id: string): RebalanceRun { if (!['running', 'stopping'].includes(run.state)) return run; updateRun(id, 'stopping', 'Stopping after the current move.'); skipRemaining(id, 'Stopped before this move began.'); - for (const move of run.moves.filter(move => move.state === 'moving')) { - if (move.operationId && !committed(move.operationId) - && cancelProvisioningJob(move.operationId, 'Rebalance stopped before source removal.')) { + for (const move of run.moves.filter((move) => move.state === 'moving')) { + if ( + move.operationId && + !committed(move.operationId) && + cancelProvisioningJob(move.operationId, 'Rebalance stopped before source removal.') + ) { updateMove(move.id, 'skipped', 'Stopped before source removal.'); } } - if (!movesFor(id).some(move => move.state === 'moving')) updateRun(id, 'stopped', 'Stopped. Completed moves are retained.'); + if (!movesFor(id).some((move) => move.state === 'moving')) + updateRun(id, 'stopped', 'Stopped. Completed moves are retained.'); return getRebalanceRun(id)!; })(); } function refreshedAfterMove(move: RebalanceMove): boolean { - return [move.sourceHostId, move.targetHostId].every(id => { + return [move.sourceHostId, move.targetHostId].every((id) => { const heartbeat = getHost(id)?.lastHeartbeat; return !!heartbeat && Date.parse(heartbeat) > Date.parse(move.updatedAt); }); @@ -118,70 +241,132 @@ function refreshedAfterMove(move: RebalanceMove): boolean { /** Only a single approved move is admitted; Provisioning owns every host command. */ function reconcileRun(run: RebalanceRun): boolean { - let active = run.moves.find(move => move.state === 'moving'); + const active = run.moves.find((move) => move.state === 'moving'); if (active) { - const operation = active.operationId && db().prepare(`SELECT d.state,d.stage,d.error,n.verified_at - FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(active.operationId) as - { state: string; stage: string; error: string | null; verified_at: string | null } | undefined; + const operation = + active.operationId && + (db() + .prepare( + `SELECT d.state,d.stage,d.error,n.verified_at + FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?` + ) + .get(active.operationId) as + | { state: string; stage: string; error: string | null; verified_at: string | null } + | undefined); if (!operation) { - updateMove(active.id, 'failed', 'Provisioning operation is unavailable. Inspect the instance before starting another run.'); + updateMove( + active.id, + 'failed', + 'Provisioning operation is unavailable. Inspect the instance before starting another run.' + ); skipRemaining(run.id, 'Stopped after a move could not be reconciled.'); - updateRun(run.id, 'failed', 'A move could not be reconciled.'); return false; + updateRun(run.id, 'failed', 'A move could not be reconciled.'); + return false; } if (operation.state === 'cancelled') { updateMove(active.id, 'skipped', operation.stage); - if (run.state === 'running') updateRun(run.id, 'stopping', 'An operation was cancelled; stopping the remaining moves.'); + if (run.state === 'running') + updateRun( + run.id, + 'stopping', + 'An operation was cancelled; stopping the remaining moves.' + ); } else if (['done', 'warning'].includes(operation.state) && operation.verified_at) { updateMove(active.id, 'done', 'Healthy on the destination host.'); } else if (['failed', 'warning', 'interrupted'].includes(operation.state)) { updateMove(active.id, 'failed', operation.error ?? operation.stage); skipRemaining(run.id, 'Stopped after a move needs attention.'); - updateRun(run.id, 'failed', 'A move needs attention. Use its Provisioning operation to recover.'); return false; + updateRun( + run.id, + 'failed', + 'A move needs attention. Use its Provisioning operation to recover.' + ); + return false; } else { if (active.operationId && !committed(active.operationId)) { - const reason = run.state === 'stopping' ? 'Rebalance stopped before source removal.' : rebalanceAdmission(active.operationId); - if (reason && cancelProvisioningJob(active.operationId, reason, { preserveRebalanceRun: true })) { + const reason = + run.state === 'stopping' + ? 'Rebalance stopped before source removal.' + : rebalanceAdmission(active.operationId); + if ( + reason && + cancelProvisioningJob(active.operationId, reason, { + preserveRebalanceRun: true + }) + ) { updateMove(active.id, 'skipped', reason); } else { updateMove(active.id, 'moving', operation.stage); - updateRun(run.id, run.state, operation.stage); return false; + updateRun(run.id, run.state, operation.stage); + return false; } } else { updateMove(active.id, 'moving', operation.stage); - updateRun(run.id, run.state, run.state === 'stopping' ? 'Finishing the current move before stopping.' : operation.stage); return false; + updateRun( + run.id, + run.state, + run.state === 'stopping' + ? 'Finishing the current move before stopping.' + : operation.stage + ); + return false; } } run = getRebalanceRun(run.id)!; - active = undefined; } if (run.state === 'stopping') { skipRemaining(run.id, 'Stopped before this move began.'); - updateRun(run.id, 'stopped', 'Stopped. Completed moves are retained.'); return false; + updateRun(run.id, 'stopped', 'Stopped. Completed moves are retained.'); + return false; } - if (!run.moves.some(move => move.state === 'planned')) { - const done = run.moves.filter(move => move.state === 'done').length; - const skipped = run.moves.filter(move => move.state === 'skipped').length; - updateRun(run.id, 'done', `Finished: ${done} moved${skipped ? `, ${skipped} skipped` : ''}.`); return false; + if (!run.moves.some((move) => move.state === 'planned')) { + const done = run.moves.filter((move) => move.state === 'done').length; + const skipped = run.moves.filter((move) => move.state === 'skipped').length; + updateRun( + run.id, + 'done', + `Finished: ${done} moved${skipped ? `, ${skipped} skipped` : ''}.` + ); + return false; } - const latest = run.moves.filter(move => move.state === 'done').sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0]; + const latest = run.moves + .filter((move) => move.state === 'done') + .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0]; if (latest && !refreshedAfterMove(latest)) { - updateRun(run.id, 'running', 'Waiting for fresh source and destination measurements after the last move.'); return false; + updateRun( + run.id, + 'running', + 'Waiting for fresh source and destination measurements after the last move.' + ); + return false; } const snapshot = rebalanceSnapshot(run.settings); - for (const move of run.moves.filter(move => move.state === 'planned')) { + for (const move of run.moves.filter((move) => move.state === 'planned')) { const reason = rebalanceMoveBlocker(snapshot, move); - if (reason) { updateMove(move.id, 'skipped', reason); continue; } + if (reason) { + updateMove(move.id, 'skipped', reason); + continue; + } // This transaction includes the durable move-to-operation link. A restarted // worker sees either both records or neither and cannot repeat admission. let operationId: string; - try { operationId = requestInstanceMove(move.server, move.targetHostId, { triggeredBy: `rebalance:${run.id}` }); } - catch (error) { updateMove(move.id, 'skipped', (error as Error).message); continue; } - db().prepare("UPDATE rebalancing_moves SET state='moving',operation_id=?,message='Waiting for safe migration admission.',updated_at=? WHERE id=?") + try { + operationId = requestInstanceMove(move.server, move.targetHostId, { + triggeredBy: `rebalance:${run.id}` + }); + } catch (error) { + updateMove(move.id, 'skipped', (error as Error).message); + continue; + } + db() + .prepare( + "UPDATE rebalancing_moves SET state='moving',operation_id=?,message='Waiting for safe migration admission.',updated_at=? WHERE id=?" + ) .run(operationId, atNow(), move.id); updateRun(run.id, 'running', `Moving ${move.server} to ${move.targetHostName}.`); return true; } - const done = run.moves.filter(move => move.state === 'done').length; + const done = run.moves.filter((move) => move.state === 'done').length; updateRun(run.id, 'done', `Finished: ${done} moved. Remaining moves no longer qualify.`); return false; } @@ -190,25 +375,45 @@ export async function runRebalancing(): Promise { return db().transaction(() => { // Recovery is explicit in Provisioning. Reflect a recovered move, but // never restart the other moves from the failed parent run. - const recovered = db().prepare(`SELECT m.id,m.run_id FROM rebalancing_moves m + const recovered = db() + .prepare( + `SELECT m.id,m.run_id FROM rebalancing_moves m JOIN rebalancing_runs r ON r.id=m.run_id JOIN server_deployments d ON d.id=m.operation_id JOIN native_provisioning n ON n.deployment_id=d.id - WHERE r.state='failed' AND m.state='failed' AND d.state IN ('done','warning') AND n.verified_at IS NOT NULL`) + WHERE r.state='failed' AND m.state='failed' AND d.state IN ('done','warning') AND n.verified_at IS NOT NULL` + ) .all() as { id: string; run_id: string }[]; for (const move of recovered) { - updateMove(move.id, 'done', 'Recovered through Provisioning; healthy on the destination host.'); - updateRun(move.run_id, 'stopped', 'The failed move was recovered. Remaining moves stay stopped.'); + updateMove( + move.id, + 'done', + 'Recovered through Provisioning; healthy on the destination host.' + ); + updateRun( + move.run_id, + 'stopped', + 'The failed move was recovered. Remaining moves stay stopped.' + ); } - const row = db().prepare("SELECT * FROM rebalancing_runs WHERE state IN ('running','stopping') ORDER BY created_at LIMIT 1").get() as RunRow | undefined; + const row = db() + .prepare( + "SELECT * FROM rebalancing_runs WHERE state IN ('running','stopping') ORDER BY created_at LIMIT 1" + ) + .get() as RunRow | undefined; if (!row) return false; return reconcileRun(present(row)); })(); } export function pruneRebalancing(): void { - const now = atNow(), cutoff = new Date(Date.now() - 31 * 86_400_000).toISOString(); + const now = atNow(), + cutoff = new Date(Date.now() - 31 * 86_400_000).toISOString(); db().prepare('DELETE FROM rebalancing_previews WHERE expires_at; } export interface RebalancingSnapshot { - hosts: RebalancingHost[]; instances: RebalancingInstance[]; - exclusions: RebalancePlan['exclusions']; warnings: string[]; + hosts: RebalancingHost[]; + instances: RebalancingInstance[]; + exclusions: RebalancePlan['exclusions']; + warnings: string[]; } -const pressure = (host: RebalancingHost, used = host.used) => Math.max(used.cpuCores / host.cpuCount, used.memoryBytes / host.memoryTotal); -const add = (a: Resources, b: Resources): Resources => ({ cpuCores: a.cpuCores + b.cpuCores, memoryBytes: a.memoryBytes + b.memoryBytes }); -const subtract = (a: Resources, b: Resources): Resources => ({ cpuCores: Math.max(0, a.cpuCores - b.cpuCores), memoryBytes: Math.max(0, a.memoryBytes - b.memoryBytes) }); -const maximum = (a: Resources, b: Resources): Resources => ({ cpuCores: Math.max(a.cpuCores, b.cpuCores), memoryBytes: Math.max(a.memoryBytes, b.memoryBytes) }); +const pressure = (host: RebalancingHost, used = host.used) => + Math.max(used.cpuCores / host.cpuCount, used.memoryBytes / host.memoryTotal); +const add = (a: Resources, b: Resources): Resources => ({ + cpuCores: a.cpuCores + b.cpuCores, + memoryBytes: a.memoryBytes + b.memoryBytes +}); +const subtract = (a: Resources, b: Resources): Resources => ({ + cpuCores: Math.max(0, a.cpuCores - b.cpuCores), + memoryBytes: Math.max(0, a.memoryBytes - b.memoryBytes) +}); +const maximum = (a: Resources, b: Resources): Resources => ({ + cpuCores: Math.max(a.cpuCores, b.cpuCores), + memoryBytes: Math.max(a.memoryBytes, b.memoryBytes) +}); -function evaluate(source: RebalancingHost, target: RebalancingHost, instance: RebalancingInstance): { reason: string | null; improvement: number; demand: Resources } { +function evaluate( + source: RebalancingHost, + target: RebalancingHost, + instance: RebalancingInstance +): { reason: string | null; improvement: number; demand: Resources } { const demand = maximum(instance.demand, target.running); const startup = add(target.used, maximum(demand, target.startup)); - const improvement = 100 * (Math.max(pressure(source), pressure(target)) - - Math.max(pressure(source, subtract(source.used, instance.relief)), pressure(target, add(target.used, demand)))); + const improvement = + 100 * + (Math.max(pressure(source), pressure(target)) - + Math.max( + pressure(source, subtract(source.used, instance.relief)), + pressure(target, add(target.used, demand)) + )); let reason: string | null = source.reason ?? target.reason; - if (!reason && source.id === target.id) reason = 'Source and destination must be different hosts.'; - if (!reason && !(target.id in instance.destinations)) reason = 'Destination compatibility has not been verified.'; + if (!reason && source.id === target.id) + reason = 'Source and destination must be different hosts.'; + if (!reason && !(target.id in instance.destinations)) + reason = 'Destination compatibility has not been verified.'; if (!reason) reason = instance.destinations[target.id] ?? null; if (!reason && target.ports < 1) reason = 'No available UDP/TCP game port.'; - if (!reason && startup.cpuCores > target.cpuCount * 0.95) reason = 'Destination has insufficient startup CPU capacity.'; - if (!reason && startup.memoryBytes > target.memoryTotal - 500_000_000) reason = 'Destination has insufficient startup memory capacity.'; - if (!reason && (!Number.isFinite(improvement) || improvement + 1e-8 < REBALANCE_MIN_IMPROVEMENT)) reason = `Move would not reduce the pair’s peak utilization by at least ${REBALANCE_MIN_IMPROVEMENT} percentage points.`; + if (!reason && startup.cpuCores > target.cpuCount * 0.95) + reason = 'Destination has insufficient startup CPU capacity.'; + if (!reason && startup.memoryBytes > target.memoryTotal - 500_000_000) + reason = 'Destination has insufficient startup memory capacity.'; + if ( + !reason && + (!Number.isFinite(improvement) || improvement + 1e-8 < REBALANCE_MIN_IMPROVEMENT) + ) + reason = `Move would not reduce the pair’s peak utilization by at least ${REBALANCE_MIN_IMPROVEMENT} percentage points.`; return { reason, improvement, demand }; } /** Validate the approved pair against fresh evidence without replacing it with a different move. */ -export function rebalanceMoveBlocker(snapshot: RebalancingSnapshot, move: Pick): string | null { - const instance = snapshot.instances.find(item => item.server === move.server && item.instanceId === move.instanceId - && item.generation === move.generation && item.sourceHostId === move.sourceHostId); - if (!instance) return snapshot.exclusions.find(item => item.server === move.server)?.reason ?? 'The instance or its generation has changed.'; - const source = snapshot.hosts.find(host => host.id === move.sourceHostId), target = snapshot.hosts.find(host => host.id === move.targetHostId); +export function rebalanceMoveBlocker( + snapshot: RebalancingSnapshot, + move: Pick< + RebalanceCandidate, + 'server' | 'instanceId' | 'generation' | 'sourceHostId' | 'targetHostId' + > +): string | null { + const instance = snapshot.instances.find( + (item) => + item.server === move.server && + item.instanceId === move.instanceId && + item.generation === move.generation && + item.sourceHostId === move.sourceHostId + ); + if (!instance) + return ( + snapshot.exclusions.find((item) => item.server === move.server)?.reason ?? + 'The instance or its generation has changed.' + ); + const source = snapshot.hosts.find((host) => host.id === move.sourceHostId), + target = snapshot.hosts.find((host) => host.id === move.targetHostId); if (!source || !target) return 'The selected source or destination is unavailable.'; return evaluate(source, target, instance).reason; } /** Greedy largest improvements minimize churn; simulated relief never changes live capacity. */ -export function planRebalance(snapshot: RebalancingSnapshot, settings: RebalanceSettings): RebalancePlan { - const hosts = snapshot.hosts.filter(host => settings.hostIds.includes(host.id)).map(host => ({ ...host, used: { ...host.used } })); - const before = new Map(hosts.map(host => [host.id, { ...host, used: { ...host.used } }])); - const instances = snapshot.instances.filter(instance => settings.hostIds.includes(instance.sourceHostId) - && (settings.game === 'all' || settings.game === instance.game)); - const moved = new Set(), moves: RebalanceCandidate[] = []; +export function planRebalance( + snapshot: RebalancingSnapshot, + settings: RebalanceSettings +): RebalancePlan { + const hosts = snapshot.hosts + .filter((host) => settings.hostIds.includes(host.id)) + .map((host) => ({ ...host, used: { ...host.used } })); + const before = new Map(hosts.map((host) => [host.id, { ...host, used: { ...host.used } }])); + const instances = snapshot.instances.filter( + (instance) => + settings.hostIds.includes(instance.sourceHostId) && + (settings.game === 'all' || settings.game === instance.game) + ); + const moved = new Set(), + moves: RebalanceCandidate[] = []; const limit = Math.max(1, Math.min(20, Math.floor(settings.maxMoves) || 1)); while (moves.length < limit) { - let best: { instance: RebalancingInstance; source: RebalancingHost; target: RebalancingHost; improvement: number; demand: Resources } | undefined; + let best: + | { + instance: RebalancingInstance; + source: RebalancingHost; + target: RebalancingHost; + improvement: number; + demand: Resources; + } + | undefined; for (const instance of [...instances].sort((a, b) => a.server.localeCompare(b.server))) { if (moved.has(instance.instanceId)) continue; - const source = hosts.find(host => host.id === instance.sourceHostId); + const source = hosts.find((host) => host.id === instance.sourceHostId); if (!source) continue; for (const target of [...hosts].sort((a, b) => a.id.localeCompare(b.id))) { const result = evaluate(source, target, instance); - if (!result.reason && (!best || result.improvement > best.improvement + 1e-8)) best = { instance, source, target, ...result }; + if (!result.reason && (!best || result.improvement > best.improvement + 1e-8)) + best = { instance, source, target, ...result }; } } if (!best) break; const { instance, source, target, improvement, demand } = best; - moves.push({ server: instance.server, instanceId: instance.instanceId, generation: instance.generation, - sourceHostId: source.id, sourceHostName: source.name, targetHostId: target.id, targetHostName: target.name, improvement }); - source.used = subtract(source.used, instance.relief); source.instances--; + moves.push({ + server: instance.server, + instanceId: instance.instanceId, + generation: instance.generation, + sourceHostId: source.id, + sourceHostName: source.name, + targetHostId: target.id, + targetHostName: target.name, + improvement + }); + source.used = subtract(source.used, instance.relief); + source.instances--; if (instance.releasesPort) source.ports++; - target.used = add(target.used, demand); target.instances++; target.ports--; + target.used = add(target.used, demand); + target.instances++; + target.ports--; moved.add(instance.instanceId); } const exclusions = [...snapshot.exclusions]; - for (const instance of instances.filter(item => !moved.has(item.instanceId))) { - const source = hosts.find(host => host.id === instance.sourceHostId)!; - const results = hosts.filter(host => host.id !== source.id).map(target => ({ target, ...evaluate(source, target, instance) })); - const fits = results.some(result => !result.reason); - const reason = fits && moves.length === limit ? 'Move limit reached.' : results.length - ? results.map(result => `${result.target.name}: ${result.reason ?? 'A different move provided more improvement.'}`).join(' ') - : 'No other participating host is available.'; + for (const instance of instances.filter((item) => !moved.has(item.instanceId))) { + const source = hosts.find((host) => host.id === instance.sourceHostId)!; + const results = hosts + .filter((host) => host.id !== source.id) + .map((target) => ({ target, ...evaluate(source, target, instance) })); + const fits = results.some((result) => !result.reason); + const reason = + fits && moves.length === limit + ? 'Move limit reached.' + : results.length + ? results + .map( + (result) => + `${result.target.name}: ${result.reason ?? 'A different move provided more improvement.'}` + ) + .join(' ') + : 'No other participating host is available.'; exclusions.push({ server: instance.server, reason }); } - const projection = (host: RebalancingHost) => ({ cpuPercent: host.cpuCount > 0 ? host.used.cpuCores / host.cpuCount * 100 : 0, - memoryPercent: host.memoryTotal > 0 ? host.used.memoryBytes / host.memoryTotal * 100 : 0, instances: host.instances }); - return { hosts: hosts.map(host => ({ id: host.id, name: host.name, before: projection(before.get(host.id)!), after: projection(host) })), - moves, exclusions, warnings: [...snapshot.warnings] }; + const projection = (host: RebalancingHost) => ({ + cpuPercent: host.cpuCount > 0 ? (host.used.cpuCores / host.cpuCount) * 100 : 0, + memoryPercent: host.memoryTotal > 0 ? (host.used.memoryBytes / host.memoryTotal) * 100 : 0, + instances: host.instances + }); + return { + hosts: hosts.map((host) => ({ + id: host.id, + name: host.name, + before: projection(before.get(host.id)!), + after: projection(host) + })), + moves, + exclusions, + warnings: [...snapshot.warnings] + }; } diff --git a/src/lib/server/rebalancing/snapshot.ts b/src/lib/server/rebalancing/snapshot.ts index e34c49b..271ff77 100644 --- a/src/lib/server/rebalancing/snapshot.ts +++ b/src/lib/server/rebalancing/snapshot.ts @@ -15,130 +15,364 @@ import type { RebalanceHostOption, RebalanceSettings } from '../../rebalancing.j import { REBALANCE_COOLDOWN_MS, type RebalancingSnapshot, type Resources } from './planner.js'; const HISTORY_MS = 5 * 60_000; -const nonnegative = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value) && value >= 0; +const nonnegative = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value) && value >= 0; function hostBlocker(host: Host, now: number): string | null { if (host.mode !== 'active' || host.decommission) return 'An active host is required.'; const stamp = Date.parse(host.lastHeartbeat ?? ''); - if (!host.connected || !Number.isFinite(stamp) || stamp > now || now - stamp > HEARTBEAT_MAX_AGE_MS) return 'Waiting for a fresh host report.'; - if (host.readiness !== 'ready') return host.readinessReasons.join(' ') || 'The host is not ready.'; + if ( + !host.connected || + !Number.isFinite(stamp) || + stamp > now || + now - stamp > HEARTBEAT_MAX_AGE_MS + ) + return 'Waiting for a fresh host report.'; + if (host.readiness !== 'ready') + return host.readinessReasons.join(' ') || 'The host is not ready.'; const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory; - if (!nonnegative(cpuCount) || !cpuCount || !nonnegative(cpuUsagePercent) || cpuUsagePercent > 100 - || !nonnegative(memoryTotal) || !memoryTotal || !nonnegative(memoryAvailable) || memoryAvailable > memoryTotal) return 'Waiting for valid CPU and memory measurements.'; + if ( + !nonnegative(cpuCount) || + !cpuCount || + !nonnegative(cpuUsagePercent) || + cpuUsagePercent > 100 || + !nonnegative(memoryTotal) || + !memoryTotal || + !nonnegative(memoryAvailable) || + memoryAvailable > memoryTotal + ) + return 'Waiting for valid CPU and memory measurements.'; return null; } export function rebalanceHosts(): RebalanceHostOption[] { const now = Date.now(); - return listHosts().map(host => { const reason = hostBlocker(host, now); return { id: host.id, name: host.name, eligible: !reason, reason }; }); + return listHosts().map((host) => { + const reason = hostBlocker(host, now); + return { id: host.id, name: host.name, eligible: !reason, reason }; + }); } /** Database evidence only: no probes, writes, reservations, or source capacity credits. */ -export function rebalanceSnapshot(settings: RebalanceSettings, exceptOperationId = ''): RebalancingSnapshot { - const handle = db(), now = Date.now(), at = new Date(now).toISOString(), since = new Date(now - HISTORY_MS).toISOString(); - const hosts = listHosts().filter(host => settings.hostIds.includes(host.id)); - const natives = handle.prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]; - const operations = handle.prepare(`SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?`).all(exceptOperationId) as { id: string; server: string; stage: string }[]; - const ownServer = exceptOperationId ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperationId) as { server: string } | undefined)?.server : undefined; +export function rebalanceSnapshot( + settings: RebalanceSettings, + exceptOperationId = '' +): RebalancingSnapshot { + const handle = db(), + now = Date.now(), + at = new Date(now).toISOString(), + since = new Date(now - HISTORY_MS).toISOString(); + const hosts = listHosts().filter((host) => settings.hostIds.includes(host.id)); + const natives = handle + .prepare('SELECT * FROM native_instances WHERE retired_at IS NULL') + .all() as NativeInstance[]; + const operations = handle + .prepare( + `SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL + AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?` + ) + .all(exceptOperationId) as { id: string; server: string; stage: string }[]; + const ownServer = exceptOperationId + ? ( + handle + .prepare('SELECT server FROM server_deployments WHERE id=?') + .get(exceptOperationId) as { server: string } | undefined + )?.server + : undefined; const observe = observationReader(); - const snapshot: RebalancingSnapshot = { hosts: [], instances: [], exclusions: [], warnings: [] }; - for (const id of settings.hostIds) if (!hosts.some(host => host.id === id)) snapshot.warnings.push(`Participating host ${id} is unavailable.`); + const snapshot: RebalancingSnapshot = { + hosts: [], + instances: [], + exclusions: [], + warnings: [] + }; + for (const id of settings.hostIds) + if (!hosts.some((host) => host.id === id)) + snapshot.warnings.push(`Participating host ${id} is unavailable.`); for (const host of hosts) { - const reason = hostBlocker(host, now), { running, startup } = capacityCalibration(host.id); + const reason = hostBlocker(host, now), + { running, startup } = capacityCalibration(host.id); if (reason) snapshot.warnings.push(`${host.name}: ${reason}`); const inventory = host.inventory; - const valid = !reason || nonnegative(inventory.cpuCount) && inventory.cpuCount > 0 && nonnegative(inventory.cpuUsagePercent) - && nonnegative(inventory.memoryTotal) && inventory.memoryTotal > 0 && nonnegative(inventory.memoryAvailable) && inventory.memoryAvailable <= inventory.memoryTotal; - const history = handle.prepare(`SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples - WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?`) - .get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as { cpu: number | null; memory: number | null }; - const used: Resources = { cpuCores: valid ? inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0) / 100 : 0, - memoryBytes: valid ? inventory.memoryTotal! - Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!) : 0 }; - const hostInstances = natives.filter(instance => instance.host_id === host.id), measured = new Map(inventory.instanceResources?.map(sample => [sample.instanceId, sample])); + const valid = + !reason || + (nonnegative(inventory.cpuCount) && + inventory.cpuCount > 0 && + nonnegative(inventory.cpuUsagePercent) && + nonnegative(inventory.memoryTotal) && + inventory.memoryTotal > 0 && + nonnegative(inventory.memoryAvailable) && + inventory.memoryAvailable <= inventory.memoryTotal); + const history = handle + .prepare( + `SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples + WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?` + ) + .get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as { + cpu: number | null; + memory: number | null; + }; + const used: Resources = { + cpuCores: valid + ? (inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0)) / + 100 + : 0, + memoryBytes: valid + ? inventory.memoryTotal! - + Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!) + : 0 + }; + const hostInstances = natives.filter((instance) => instance.host_id === host.id), + measured = new Map( + inventory.instanceResources?.map((sample) => [sample.instanceId, sample]) + ); const reserved = new Set(); const reserve = (id: string, allowance: Resources) => { if (reserved.has(id)) return; - reserved.add(id); used.cpuCores += allowance.cpuCores; used.memoryBytes += allowance.memoryBytes; + reserved.add(id); + used.cpuCores += allowance.cpuCores; + used.memoryBytes += allowance.memoryBytes; }; for (const instance of hostInstances) { const sample = measured.get(instance.id); if (sample?.generation === instance.generation) continue; // Already in the host measurement. - const latest = handle.prepare(`SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n + const latest = handle + .prepare( + `SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.id!=? - AND d.action IN ('create','rebuild') AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`) - .get(instance.id, instance.generation, exceptOperationId) as { state: string; submitted_at: string | null; operation_id: string | null; stage_operation_id: string | null } | undefined; - const accounted = handle.prepare(`SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id - WHERE n.instance_id=? AND n.generation=? AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at port.port >= host.portMin && port.port <= host.portMax - && (port.bind_address === host.bindAddress || ['0.0.0.0', '::'].includes(port.bind_address) || ['0.0.0.0', '::'].includes(host.bindAddress))).map(port => port.port)); - snapshot.hosts.push({ id: host.id, name: host.name, cpuCount: valid ? inventory.cpuCount! : 0, memoryTotal: valid ? inventory.memoryTotal! : 0, - used, running, startup, reason, instances: hostInstances.length + incoming.filter(row => !hostInstances.some(instance => instance.id === row.instance_id)).length, - ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size) }); + const ports = handle + .prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?') + .all(host.id) as { port: number; bind_address: string }[]; + const usedPorts = new Set( + ports + .filter( + (port) => + port.port >= host.portMin && + port.port <= host.portMax && + (port.bind_address === host.bindAddress || + ['0.0.0.0', '::'].includes(port.bind_address) || + ['0.0.0.0', '::'].includes(host.bindAddress)) + ) + .map((port) => port.port) + ); + snapshot.hosts.push({ + id: host.id, + name: host.name, + cpuCount: valid ? inventory.cpuCount! : 0, + memoryTotal: valid ? inventory.memoryTotal! : 0, + used, + running, + startup, + reason, + instances: + hostInstances.length + + incoming.filter( + (row) => !hostInstances.some((instance) => instance.id === row.instance_id) + ).length, + ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size) + }); } const destinations = new Map>(); for (const row of listServers()) { const identity = mapIdentity(row); - if (!row.host_id || !settings.hostIds.includes(row.host_id) || settings.game !== 'all' && identity?.game !== settings.game) continue; - const source = hosts.find(host => host.id === row.host_id), native = natives.find(instance => instance.id === row.native_instance_id); + if ( + !row.host_id || + !settings.hostIds.includes(row.host_id) || + (settings.game !== 'all' && identity?.game !== settings.game) + ) + continue; + const source = hosts.find((host) => host.id === row.host_id), + native = natives.find((instance) => instance.id === row.native_instance_id); // The admitted rebuild itself must not hide the pre-removal empty evidence. - const observation = row.name === ownServer ? serverObservation(row, { provisioning: false, settling: false, - telemetry: telemetryReport(row.name), submission: latestGameSubmission(row.name) }) : observe(row); + const observation = + row.name === ownServer + ? serverObservation(row, { + provisioning: false, + settling: false, + telemetry: telemetryReport(row.name), + submission: latestGameSubmission(row.name) + }) + : observe(row); let reason = protectedWork(row.name, exceptOperationId); - if (!reason && (!identity || !native || !row.map_slug)) reason = 'A native instance with a known map is required.'; - if (!reason && (!source || hostBlocker(source, now))) reason = source ? hostBlocker(source, now) : 'Source host is unavailable.'; - const operation = operations.find(item => item.server === row.name); + if (!reason && (!identity || !native || !row.map_slug)) + reason = 'A native instance with a known map is required.'; + if (!reason && (!source || hostBlocker(source, now))) + reason = source ? hostBlocker(source, now) : 'Source host is unavailable.'; + const operation = operations.find((item) => item.server === row.name); if (!reason && operation) reason = `Provisioning ${operation.id}: ${operation.stage}`; - if (!reason && native && handle.prepare(`SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown') - AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')`).get(native.id)) reason = 'An instance lifecycle command is unsettled.'; - if (!reason && (row.desired_state === 'stopped' || ['stopped','exited','created','dead','missing'].includes(row.container_state ?? ''))) reason = 'Stopped instances are not eligible.'; - if (!reason && (row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now) - && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) reason = 'Players are present according to a current RCON report.'; - if (!reason && (observation.health !== 'online' || observation.last_players === null || !freshStatusTimestamp(observation.last_seen_at, now))) reason = 'Waiting for healthy, fresh occupancy evidence.'; + if ( + !reason && + native && + handle + .prepare( + `SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown') + AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')` + ) + .get(native.id) + ) + reason = 'An instance lifecycle command is unsettled.'; + if ( + !reason && + (row.desired_state === 'stopped' || + ['stopped', 'exited', 'created', 'dead', 'missing'].includes( + row.container_state ?? '' + )) + ) + reason = 'Stopped instances are not eligible.'; + if ( + !reason && + (row.last_players ?? 0) > 0 && + freshStatusTimestamp(row.last_seen_at, now) && + (!observation.telemetry_after || + Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after)) + ) + reason = 'Players are present according to a current RCON report.'; + if ( + !reason && + (observation.health !== 'online' || + observation.last_players === null || + !freshStatusTimestamp(observation.last_seen_at, now)) + ) + reason = 'Waiting for healthy, fresh occupancy evidence.'; if (!reason && observation.last_players !== 0) reason = 'Players are present.'; - if (!reason && !getServerKey(row.name)) reason = 'Save a server key before moving this instance.'; - if (!reason && handle.prepare(`SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?`) - .get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString())) reason = 'This instance was rebalanced less than 30 minutes ago.'; - const sample = source?.inventory.instanceResources?.find(item => item.instanceId === native?.id && item.generation === native.generation); - if (!reason && (!sample || source?.inventory.instanceResourcesError || !nonnegative(sample.cpuCores) || !nonnegative(sample.memoryBytes) || !sample.memoryBytes)) reason = 'Waiting for resource measurements for the current instance generation.'; - if (!reason && sample!.uptimeSeconds < 300) reason = 'Waiting for this instance to finish its first five minutes of startup.'; - if (!reason) { try { captureMoveRecipe(row.name); } catch (error) { reason = (error as Error).message; } } - if (reason) { snapshot.exclusions.push({ server: row.name, reason }); continue; } + if (!reason && !getServerKey(row.name)) + reason = 'Save a server key before moving this instance.'; + if ( + !reason && + handle + .prepare( + `SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?` + ) + .get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString()) + ) + reason = 'This instance was rebalanced less than 30 minutes ago.'; + const sample = source?.inventory.instanceResources?.find( + (item) => item.instanceId === native?.id && item.generation === native.generation + ); + if ( + !reason && + (!sample || + source?.inventory.instanceResourcesError || + !nonnegative(sample.cpuCores) || + !nonnegative(sample.memoryBytes) || + !sample.memoryBytes) + ) + reason = 'Waiting for resource measurements for the current instance generation.'; + if (!reason && sample!.uptimeSeconds < 300) + reason = 'Waiting for this instance to finish its first five minutes of startup.'; + if (!reason) { + try { + captureMoveRecipe(row.name); + } catch (error) { + reason = (error as Error).message; + } + } + if (reason) { + snapshot.exclusions.push({ server: row.name, reason }); + continue; + } const key = JSON.stringify([native!.library_id, native!.image]); if (!destinations.has(key)) { const eligibility: Record = {}; for (const host of hosts) { - try { requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 }); assertReadyLibrary(host.id, native!.library_id); eligibility[host.id] = null; } - catch (error) { eligibility[host.id] = (error as Error).message; } + try { + requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 }); + assertReadyLibrary(host.id, native!.library_id); + eligibility[host.id] = null; + } catch (error) { + eligibility[host.id] = (error as Error).message; + } } destinations.set(key, eligibility); } - const history = handle.prepare(`SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples - WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?`) - .get(source!.id, native!.id, native!.generation, since, at) as { cpu: number | null; memory: number | null }; - const bindConflicts = (bind: string) => bind === source!.bindAddress || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(source!.bindAddress); - const portOwners = handle.prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?') + const history = handle + .prepare( + `SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples + WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?` + ) + .get(source!.id, native!.id, native!.generation, since, at) as { + cpu: number | null; + memory: number | null; + }; + const bindConflicts = (bind: string) => + bind === source!.bindAddress || + ['0.0.0.0', '::'].includes(bind) || + ['0.0.0.0', '::'].includes(source!.bindAddress); + const portOwners = handle + .prepare( + 'SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?' + ) .all(source!.id, native!.port) as { instance_id: string; bind_address: string }[]; - snapshot.instances.push({ server: row.name, instanceId: native!.id, generation: native!.generation, sourceHostId: source!.id, game: identity!.game, - releasesPort: native!.port >= source!.portMin && native!.port <= source!.portMax && bindConflicts(native!.bind_address) - && portOwners.some(owner => owner.instance_id === native!.id && bindConflicts(owner.bind_address)) - && !portOwners.some(owner => owner.instance_id !== native!.id && bindConflicts(owner.bind_address)), - relief: { cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) }, - demand: { cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) }, - destinations: destinations.get(key)! }); + snapshot.instances.push({ + server: row.name, + instanceId: native!.id, + generation: native!.generation, + sourceHostId: source!.id, + game: identity!.game, + releasesPort: + native!.port >= source!.portMin && + native!.port <= source!.portMax && + bindConflicts(native!.bind_address) && + portOwners.some( + (owner) => owner.instance_id === native!.id && bindConflicts(owner.bind_address) + ) && + !portOwners.some( + (owner) => owner.instance_id !== native!.id && bindConflicts(owner.bind_address) + ), + relief: { + cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores), + memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) + }, + demand: { + cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores), + memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) + }, + destinations: destinations.get(key)! + }); } return snapshot; } diff --git a/src/lib/server/reports/database.ts b/src/lib/server/reports/database.ts index 734ae1c..afb9297 100644 --- a/src/lib/server/reports/database.ts +++ b/src/lib/server/reports/database.ts @@ -4,4 +4,6 @@ import { tuneConnection } from '../db/connection.js'; const handle = new Database(workerData.dbPath, { readonly: true, fileMustExist: true }); tuneConnection(handle, true); -export function db() { return handle; } +export function db() { + return handle; +} diff --git a/src/lib/server/reports/index.ts b/src/lib/server/reports/index.ts index 6b5383e..bd22a04 100644 --- a/src/lib/server/reports/index.ts +++ b/src/lib/server/reports/index.ts @@ -3,43 +3,78 @@ import { resolve } from 'node:path'; import { config } from '../config.js'; import { db } from '../db/index.js'; import type { MetricsQuery, MetricsData } from '../../metrics.js'; -import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js'; +import type { + LeaderboardOptions, + LeaderboardResponse, + PlayerProfile, + LeaderboardEntry +} from '../../leaderboard.js'; import type { CapacityHistory, CapacityQuery } from '../../capacity.js'; import { ensureHostSchema } from '../hosts/schema.js'; class ReportWorker { private worker: Worker | null = null; private sequence = 0; - private pending = new Map void; reject: (error: Error) => void; timer: NodeJS.Timeout }>(); + private pending = new Map< + number, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Pending requests hold resolvers for different report result types. + { resolve: (value: any) => void; reject: (error: Error) => void; timer: NodeJS.Timeout } + >(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Each request key identifies a caller-specific report result type. private flights = new Map>(); async close() { - const worker = this.worker; this.worker = null; - for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Report service stopped')); } - this.pending.clear(); this.flights.clear(); + const worker = this.worker; + this.worker = null; + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(new Error('Report service stopped')); + } + this.pending.clear(); + this.flights.clear(); await worker?.terminate(); } run(kind: string, input: unknown): Promise { const requestInput = input as { query?: unknown; now?: number }; const timed = kind === 'metrics' || kind === 'capacity'; - const key = JSON.stringify(timed ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]); + const key = JSON.stringify( + timed + ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] + : [kind, input] + ); const flight = this.flights.get(key); if (flight) return flight; - if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.')); + if (this.pending.size >= 64) + return Promise.reject(new Error('Reports are busy. Please retry shortly.')); db(); // Finish migrations before opening the independent reader. if (!this.worker) { - const worker = this.worker = new Worker(resolve(process.env.NODE_ENV === 'production' ? 'build/reports/worker.mjs' : '.svelte-kit/reports/worker.mjs'), { workerData: { dbPath: resolve(config.dbPath) } }); + const worker = (this.worker = new Worker( + resolve( + process.env.NODE_ENV === 'production' + ? 'build/reports/worker.mjs' + : '.svelte-kit/reports/worker.mjs' + ), + { workerData: { dbPath: resolve(config.dbPath) } } + )); worker.on('message', ({ id, value, error, elapsedMs, cached }) => { - const pending = this.pending.get(id); if (!pending) return; - clearTimeout(pending.timer); this.pending.delete(id); - if (process.env.EC_PERF_LOG === 'true') console.info('[report]', { kind, elapsedMs, cached }); - if (error) pending.reject(new Error(error)); else pending.resolve(value); + const pending = this.pending.get(id); + if (!pending) return; + clearTimeout(pending.timer); + this.pending.delete(id); + if (process.env.EC_PERF_LOG === 'true') + console.info('[report]', { kind, elapsedMs, cached }); + if (error) pending.reject(new Error(error)); + else pending.resolve(value); if (!this.pending.size) worker.unref(); }); const fail = (error: Error) => { if (this.worker !== worker) return; this.worker = null; - for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(error); } - this.pending.clear(); void worker.terminate(); + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + this.pending.clear(); + void worker.terminate(); }; worker.on('error', fail); worker.on('exit', () => fail(new Error('Report worker stopped. Please retry.'))); @@ -49,12 +84,20 @@ class ReportWorker { const promise = new Promise((resolve, reject) => { const timer = setTimeout(() => { // Reset the worker rather than leaving an expensive abandoned query queued. - const worker = this.worker; this.worker = null; - for (const item of this.pending.values()) { clearTimeout(item.timer); item.reject(new Error('Report timed out. Please retry.')); } - this.pending.clear(); void worker?.terminate(); + const worker = this.worker; + this.worker = null; + for (const item of this.pending.values()) { + clearTimeout(item.timer); + item.reject(new Error('Report timed out. Please retry.')); + } + this.pending.clear(); + void worker?.terminate(); }, 60_000); - this.pending.set(id, { resolve, reject, timer }); this.worker!.postMessage({ id, kind, input }); - }).finally(() => { if (this.flights.get(key) === promise) this.flights.delete(key); }); + this.pending.set(id, { resolve, reject, timer }); + this.worker!.postMessage({ id, kind, input }); + }).finally(() => { + if (this.flights.get(key) === promise) this.flights.delete(key); + }); this.flights.set(key, promise); return promise; } @@ -62,12 +105,24 @@ class ReportWorker { const metrics = new ReportWorker(); const leaderboard = new ReportWorker(); const capacity = new ReportWorker(); -export const metricsReport = (query: MetricsQuery) => metrics.run('metrics', { query, now: Date.now() }); -export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run('leaderboard', query); -export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options }); +export const metricsReport = (query: MetricsQuery) => + metrics.run('metrics', { query, now: Date.now() }); +export const leaderboardReport = (query: LeaderboardOptions) => + leaderboard.run('leaderboard', query); +export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => + leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { + guid, + offset, + options + }); export const capacityHistoryReport = (query: CapacityQuery) => { ensureHostSchema(db()); // Game changes current placement estimates, not measured host resources. - return capacity.run('capacity', { query: { ...query, game: 'all' }, now: Date.now() }); + return capacity.run('capacity', { + query: { ...query, game: 'all' }, + now: Date.now() + }); }; -export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close(), capacity.close()]); } +export async function closeReports() { + await Promise.all([metrics.close(), leaderboard.close(), capacity.close()]); +} diff --git a/src/lib/server/reports/revisions.ts b/src/lib/server/reports/revisions.ts index 60e14f2..9e245d3 100644 --- a/src/lib/server/reports/revisions.ts +++ b/src/lib/server/reports/revisions.ts @@ -3,23 +3,39 @@ import type Database from 'better-sqlite3'; export function installReportRevisions(handle: Database.Database) { handle.exec(`CREATE TABLE IF NOT EXISTS report_revisions (topic TEXT PRIMARY KEY, revision INTEGER NOT NULL DEFAULT 0); INSERT OR IGNORE INTO report_revisions(topic) VALUES ('metrics'), ('leaderboard');`); - for (const table of ['metric_samples', 'servers']) for (const action of ['INSERT', 'UPDATE', 'DELETE']) { - const condition = table === 'servers' && action === 'UPDATE' ? 'WHEN OLD.destroyed_at IS NOT NEW.destroyed_at OR OLD.name IS NOT NEW.name' : ''; - handle.exec(`CREATE TRIGGER IF NOT EXISTS report_${table}_${action} AFTER ${action} ON ${table} ${condition} + for (const table of ['metric_samples', 'servers']) + for (const action of ['INSERT', 'UPDATE', 'DELETE']) { + const condition = + table === 'servers' && action === 'UPDATE' + ? 'WHEN OLD.destroyed_at IS NOT NEW.destroyed_at OR OLD.name IS NOT NEW.name' + : ''; + handle.exec(`CREATE TRIGGER IF NOT EXISTS report_${table}_${action} AFTER ${action} ON ${table} ${condition} BEGIN UPDATE report_revisions SET revision=revision+1 WHERE topic='metrics'; END;`); - } - if (handle.prepare("SELECT 1 FROM settings WHERE key='completed_report_revisions_v1'").get()) return; + } + if (handle.prepare("SELECT 1 FROM settings WHERE key='completed_report_revisions_v1'").get()) + return; handle.transaction(() => { - for (const table of ['leaderboard_matches', 'leaderboard_match_players', 'leaderboard_players']) { + for (const table of [ + 'leaderboard_matches', + 'leaderboard_match_players', + 'leaderboard_players' + ]) { for (const action of ['INSERT', 'UPDATE', 'DELETE']) { const name = `report_${table}_${action}`; - const versions = action === 'INSERT' ? ['NEW'] : action === 'DELETE' ? ['OLD'] : ['OLD', 'NEW']; - const condition = table === 'leaderboard_matches' - ? `WHEN ${versions.map(row => `${row}.status='completed'`).join(' OR ')}` - : table === 'leaderboard_match_players' - ? `WHEN ${versions.map(row => `(SELECT status FROM leaderboard_matches WHERE id=${row}.match_id)='completed'`).join(' OR ')}` - : action === 'UPDATE' ? 'WHEN OLD.name IS NOT NEW.name' : ''; - const event = table === 'leaderboard_players' && action === 'UPDATE' ? 'UPDATE OF name' : action; + const versions = + action === 'INSERT' ? ['NEW'] : action === 'DELETE' ? ['OLD'] : ['OLD', 'NEW']; + const condition = + table === 'leaderboard_matches' + ? `WHEN ${versions.map((row) => `${row}.status='completed'`).join(' OR ')}` + : table === 'leaderboard_match_players' + ? `WHEN ${versions.map((row) => `(SELECT status FROM leaderboard_matches WHERE id=${row}.match_id)='completed'`).join(' OR ')}` + : action === 'UPDATE' + ? 'WHEN OLD.name IS NOT NEW.name' + : ''; + const event = + table === 'leaderboard_players' && action === 'UPDATE' + ? 'UPDATE OF name' + : action; handle.exec(`DROP TRIGGER IF EXISTS ${name}; CREATE TRIGGER ${name} AFTER ${event} ON ${table} ${condition} BEGIN UPDATE report_revisions SET revision=revision+1 WHERE topic='leaderboard'; END;`); diff --git a/src/lib/server/reports/worker.ts b/src/lib/server/reports/worker.ts index bf0f497..104125c 100644 --- a/src/lib/server/reports/worker.ts +++ b/src/lib/server/reports/worker.ts @@ -15,26 +15,56 @@ parentPort!.on('message', ({ id, kind, input }) => { // Capacity refreshes every 30 seconds without adding write triggers // to the heartbeat path. It also has its own worker, so large // history ranges cannot queue ahead of fleet or player reports. - const revision = kind === 'capacity' ? 0 - : (db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number }).revision; + const revision = + kind === 'capacity' + ? 0 + : ( + db() + .prepare('SELECT revision FROM report_revisions WHERE topic=?') + .get(topic) as { revision: number } + ).revision; const timed = kind === 'metrics' || kind === 'capacity'; const now = timed ? input.now : Date.now(); - const key = JSON.stringify([kind, revision, timed ? Math.floor(now / 30_000) : 0, timed ? input.query : input]); + const key = JSON.stringify([ + kind, + revision, + timed ? Math.floor(now / 30_000) : 0, + timed ? input.query : input + ]); const saved = cache.get(key); - if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; } - const value = kind === 'metrics' ? readMetrics(input.query, now) - : kind === 'capacity' ? readCapacityHistory(input.query, now) - : kind === 'leaderboard' ? getLeaderboard(input) - : { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) }; + if (saved) { + cache.delete(key); + cache.set(key, saved); + return { value: saved.value, cached: true }; + } + const value = + kind === 'metrics' + ? readMetrics(input.query, now) + : kind === 'capacity' + ? readCapacityHistory(input.query, now) + : kind === 'leaderboard' + ? getLeaderboard(input) + : { + profile: getPlayerProfile(input.guid, input.offset, input.options), + entry: getPlayerLeaderboardEntry(input.guid, input.options) + }; const size = Buffer.byteLength(JSON.stringify(value)); if (size < 16 * 1024 * 1024) { - cache.set(key, { value, bytes: size }); bytes += size; + cache.set(key, { value, bytes: size }); + bytes += size; while (cache.size > 64 || bytes > 32 * 1024 * 1024) { - const oldest = cache.keys().next().value!; bytes -= cache.get(oldest)!.bytes; cache.delete(oldest); + const oldest = cache.keys().next().value!; + bytes -= cache.get(oldest)!.bytes; + cache.delete(oldest); } } return { value, cached: false }; })(); parentPort!.postMessage({ id, ...result, elapsedMs: performance.now() - started }); - } catch (cause) { parentPort!.postMessage({ id, error: cause instanceof Error ? cause.message : 'Report failed' }); } + } catch (cause) { + parentPort!.postMessage({ + id, + error: cause instanceof Error ? cause.message : 'Report failed' + }); + } }); diff --git a/src/lib/server/retention.ts b/src/lib/server/retention.ts index c34d239..4a57c4e 100644 --- a/src/lib/server/retention.ts +++ b/src/lib/server/retention.ts @@ -10,8 +10,16 @@ import { logResultMetadata } from './hosts/log-results.js'; import type { FleetSettings } from '../fleet-settings.js'; interface Policy { - category: string; label: string; table: string; id: string; clock: string; - days: keyof FleetSettings; eligible: string; bytes?: string; failed?: string; minimum?: boolean; + category: string; + label: string; + table: string; + id: string; + clock: string; + days: keyof FleetSettings; + eligible: string; + bytes?: string; + failed?: string; + minimum?: boolean; } const terminal = "('done','warning','failed','cancelled')"; // Cancellation stops observation, not an already-delivered command. Its source @@ -61,10 +69,26 @@ const failedCampaign = `EXISTS (SELECT 1 FROM deployment_servers s WHERE s.deplo function policies(): Policy[] { return [ - { category: 'campaign', label: 'Campaigns', table: 'deployments', id: 'id', clock: 'started_at', days: 'deployKeepDays', - eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, failed: failedCampaign, - bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, - { category: 'provisioning', label: 'Provisioning workflows', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'provisioningKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')}) + { + category: 'campaign', + label: 'Campaigns', + table: 'deployments', + id: 'id', + clock: 'started_at', + days: 'deployKeepDays', + eligible: `${campaignSafe} AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='file-content' AND c.record_id=o.id AND c.protected=1)`, + failed: failedCampaign, + bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` + }, + { + category: 'provisioning', + label: 'Provisioning workflows', + table: 'server_deployments', + id: 'id', + clock: 'finished_at', + days: 'provisioningKeepDays', + failed: "o.state='failed'", + eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')}) AND NOT EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=o.id AND r.healthy_at IS NULL) AND NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN ec_host_operations a ON a.id IN (n.operation_id,n.stage_operation_id) WHERE n.deployment_id=o.id AND a.state NOT IN ('completed','failed')) AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id WHERE m.deployment_id=o.id AND a.state NOT IN ('completed','failed')) @@ -77,41 +101,115 @@ function policies(): Policy[] { AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed')) AND NOT EXISTS (SELECT 1 FROM server_collection_members m JOIN server_collection_members sibling ON sibling.collection_id=m.collection_id JOIN server_deployments d ON d.id=sibling.deployment_id WHERE m.deployment_id=o.id AND (${unsettledDeployment}))`, - bytes: "COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)" }, - { category: 'operation', label: 'Agent actions and resolved failures', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'actionKeepDays', + bytes: 'COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)' + }, + { + category: 'operation', + label: 'Agent actions and resolved failures', + table: 'ec_host_operations', + id: 'id', + clock: 'updated_at', + days: 'actionKeepDays', failed: "(o.state='failed' OR (o.routine=1 AND o.observation_success=0))", eligible: `o.history_expired_at IS NULL AND o.state IN ('completed','failed') AND (o.routine=0 OR o.observation_success=0) AND ${unreferencedOperation}`, - bytes: "COALESCE(length(o.payload),0)+COALESCE(length(o.result),0)+COALESCE((SELECT length(payload) FROM ec_host_operation_payloads WHERE operation_id=o.id),0)" }, - { category: 'observation', label: 'Routine operations', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'observationKeepDays', eligible: "o.state='completed' AND o.observation_success=1", bytes: 'length(o.payload)+COALESCE(length(o.result),0)' }, - { category: 'maintenance', label: 'Maintenance jobs', table: 'maintenance_jobs', id: 'id', clock: 'finished_at', days: 'maintenanceKeepDays', - eligible: `o.kind!='update' AND o.state IN ('done','failed','cancelled') AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=o.provisioning_id AND (${unsettledDeployment}))` }, - { category: 'recovery', label: 'Resolved recovery incidents', table: 'watchdog_incidents', id: 'id', clock: 'finished_at', days: 'recoveryKeepDays', + bytes: 'COALESCE(length(o.payload),0)+COALESCE(length(o.result),0)+COALESCE((SELECT length(payload) FROM ec_host_operation_payloads WHERE operation_id=o.id),0)' + }, + { + category: 'observation', + label: 'Routine operations', + table: 'ec_host_operations', + id: 'id', + clock: 'updated_at', + days: 'observationKeepDays', + eligible: "o.state='completed' AND o.observation_success=1", + bytes: 'length(o.payload)+COALESCE(length(o.result),0)' + }, + { + category: 'maintenance', + label: 'Maintenance jobs', + table: 'maintenance_jobs', + id: 'id', + clock: 'finished_at', + days: 'maintenanceKeepDays', + eligible: `o.kind!='update' AND o.state IN ('done','failed','cancelled') AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.id=o.provisioning_id AND (${unsettledDeployment}))` + }, + { + category: 'recovery', + label: 'Resolved recovery incidents', + table: 'watchdog_incidents', + id: 'id', + clock: 'finished_at', + days: 'recoveryKeepDays', eligible: `o.state='resolved' AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.incident_id=o.id AND (${unsettledDeployment})) - AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN restart_rollout r ON r.operation_id=a.operation_id WHERE a.incident_id=o.id AND r.healthy_at IS NULL)` }, - { category: 'match', label: 'Matches', table: 'leaderboard_matches', id: 'id', clock: 'last_received_at', days: 'matchKeepDays', eligible: "o.status IN ('completed','interrupted')", bytes: '0' }, - { category: 'beta', label: 'Completed betas', table: 'betas', id: 'id', clock: 'closed_at', days: 'betaKeepDays', + AND NOT EXISTS (SELECT 1 FROM watchdog_actions a JOIN restart_rollout r ON r.operation_id=a.operation_id WHERE a.incident_id=o.id AND r.healthy_at IS NULL)` + }, + { + category: 'match', + label: 'Matches', + table: 'leaderboard_matches', + id: 'id', + clock: 'last_received_at', + days: 'matchKeepDays', + eligible: "o.status IN ('completed','interrupted')", + bytes: '0' + }, + { + category: 'beta', + label: 'Completed betas', + table: 'betas', + id: 'id', + clock: 'closed_at', + days: 'betaKeepDays', eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta-file-content' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM deployments d WHERE d.beta_id=o.id) AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, - bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` }, - { category: 'file-content', label: 'Campaign file bodies', table: 'deployments', id: 'id', clock: 'finished_at', days: 'fileContentKeepDays', + bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` + }, + { + category: 'file-content', + label: 'Campaign file bodies', + table: 'deployments', + id: 'id', + clock: 'finished_at', + days: 'fileContentKeepDays', eligible: `${campaignSafe} AND NOT (${failedCampaign}) AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='campaign' AND c.record_id=o.id AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='file-content' AND e.record_id=o.id)`, - bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` }, - { category: 'beta-file-content', label: 'Closed beta file snapshots', table: 'betas', id: 'id', clock: 'closed_at', days: 'fileContentKeepDays', + bytes: `(SELECT COALESCE(SUM(COALESCE(${blobSize('content_before_hash')},0)+COALESCE(${blobSize('content_after_hash')},0)),0) FROM deployment_files WHERE deployment_id=o.id)` + }, + { + category: 'beta-file-content', + label: 'Closed beta file snapshots', + table: 'betas', + id: 'id', + clock: 'closed_at', + days: 'fileContentKeepDays', eligible: `o.state IN ('promoted','rolled-back') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='beta' AND c.record_id=CAST(o.id AS TEXT) AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='beta-file-content' AND e.record_id=CAST(o.id AS TEXT)) AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN server_deployments d ON d.id=s.operation_id WHERE s.kind='beta' AND s.source_id=CAST(o.id AS TEXT) AND (${unsettledDeployment}))`, - bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` }, - { category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays', + bytes: `(SELECT COALESCE(SUM(${blobSize('content_hash')}),0) FROM beta_revision_files WHERE beta_id=o.id)` + }, + { + category: 'workflow-payload', + label: 'Completed launch file bodies', + table: 'server_deployments', + id: 'id', + clock: 'finished_at', + days: 'payloadKeepDays', failed: "o.state='failed'", eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')}) AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed')) AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`, - bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" }, - { category: 'operation-payload', label: 'Completed command file bodies', table: 'ec_host_operations', id: 'id', clock: 'updated_at', days: 'payloadKeepDays', + bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" + }, + { + category: 'operation-payload', + label: 'Completed command file bodies', + table: 'ec_host_operations', + id: 'id', + clock: 'updated_at', + days: 'payloadKeepDays', failed: "o.state='failed'", eligible: `o.state IN ('completed','failed') AND o.kind IN ('instance.create','instance.rebuild','instance.restart','instance.start','files.stage','files.read') AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='operation' AND c.record_id=o.id AND c.protected=1) @@ -120,51 +218,101 @@ function policies(): Policy[] { WHERE o.id IN (n.operation_id,n.stage_operation_id) AND ((${unsettledDeployment}) OR EXISTS ( SELECT 1 FROM retention_controls c WHERE c.category IN ('provisioning','workflow-payload') AND c.record_id=d.id AND c.protected=1))) AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='operation-payload' AND e.record_id=o.id)`, - bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" }, - { category: 'version', label: 'Unused artifact versions', table: 'script_versions', id: 'id', clock: 'uploaded_at', days: 'versionKeepDays', minimum: false, + bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type IN ('operation','operation-result') AND r.owner_id=o.id)" + }, + { + category: 'version', + label: 'Unused artifact versions', + table: 'script_versions', + id: 'id', + clock: 'uploaded_at', + days: 'versionKeepDays', + minimum: false, eligible: `NOT EXISTS (SELECT 1 FROM scripts s WHERE s.current_version_id=o.id) AND NOT EXISTS (SELECT 1 FROM staged_file_changes s WHERE s.script_version_id=o.id) - AND ${versionReferences()}`, bytes: `${blobSize('o.content_hash')}` }, - { category: 'audit', label: 'Authentication audit events', table: 'auth_audit', id: 'id', clock: 'created_at', days: 'auditKeepDays', eligible: '1', bytes: 'length(o.details)' } + AND ${versionReferences()}`, + bytes: `${blobSize('o.content_hash')}` + }, + { + category: 'audit', + label: 'Authentication audit events', + table: 'auth_audit', + id: 'id', + clock: 'created_at', + days: 'auditKeepDays', + eligible: '1', + bytes: 'length(o.details)' + } ]; } function versionReferences(): string { // Even cascading references pin versions: automatic cleanup must never erase // another feature's history as a side effect of deleting an artifact. - const tables = db().prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[]; + const tables = db().prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { + name: string; + }[]; const clauses: string[] = []; for (const { name } of tables) { if (name === 'plugin_imports') continue; - const quote = (s: string) => '"' + s.replaceAll('"','""') + '"'; - for (const fk of db().prepare(`PRAGMA foreign_key_list(${quote(name)})`).all() as { table: string; from: string }[]) { - if (fk.table === 'script_versions') clauses.push(`NOT EXISTS (SELECT 1 FROM ${quote(name)} v WHERE v.${quote(fk.from)}=o.id)`); + const quote = (s: string) => '"' + s.replaceAll('"', '""') + '"'; + for (const fk of db() + .prepare(`PRAGMA foreign_key_list(${quote(name)})`) + .all() as { table: string; from: string }[]) { + if (fk.table === 'script_versions') + clauses.push( + `NOT EXISTS (SELECT 1 FROM ${quote(name)} v WHERE v.${quote(fk.from)}=o.id)` + ); } } return clauses.join(' AND ') || '1'; } -function initialize() { ensureHostSchema(db()); ensureAuthSchema(db()); } +function initialize() { + ensureHostSchema(db()); + ensureAuthSchema(db()); +} function withoutFileBodies(value: unknown): unknown { if (Array.isArray(value)) return value.map(withoutFileBodies); if (!value || typeof value !== 'object') return value; - return Object.fromEntries(Object.entries(value).filter(([key]) => !['contentHash','contentSha256','contentBase64'].includes(key)) - .map(([key, item]) => [key, withoutFileBodies(item)])); + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => !['contentHash', 'contentSha256', 'contentBase64'].includes(key)) + .map(([key, item]) => [key, withoutFileBodies(item)]) + ); } function expirePayload(category: string, id: string): void { if (category === 'workflow-payload') { - const row = db().prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?').get(id) as { recipe: string } | undefined; - if (row) db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id); - db().prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?').run(id); + const row = db() + .prepare('SELECT recipe FROM server_launch_recipes WHERE deployment_id=?') + .get(id) as { recipe: string } | undefined; + if (row) + db() + .prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?') + .run(JSON.stringify(withoutFileBodies(JSON.parse(row.recipe))), id); + db() + .prepare('UPDATE native_provisioning SET files_json=NULL WHERE deployment_id=?') + .run(id); referenceBlobs('recipe', id, []); return; } - const row = db().prepare(`SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o - LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(id) as { payload: string; result: string | null }; + const row = db() + .prepare( + `SELECT COALESCE(p.payload,o.payload) payload,o.result FROM ec_host_operations o + LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?` + ) + .get(id) as { payload: string; result: string | null }; const payload = JSON.stringify(withoutFileBodies(JSON.parse(row.payload))); - db().prepare(`UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?`) - .run(payload, row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))), - createHash('sha256').update(row.payload).digest('hex'), id); + db() + .prepare( + `UPDATE ec_host_operations SET payload=?,result=?,request_hash=COALESCE(request_hash,?) WHERE id=?` + ) + .run( + payload, + row.result === null ? null : JSON.stringify(withoutFileBodies(JSON.parse(row.result))), + createHash('sha256').update(row.payload).digest('hex'), + id + ); db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(id); referenceBlobs('operation', id, []); referenceBlobs('operation-result', id, []); @@ -173,58 +321,130 @@ function selection(policy: Policy, settings = fleetSettings()) { const days = settings[policy.days]; const failureDays = settings.failureKeepDays; const cut = (n: number) => new Date(Date.now() - n * 86400_000).toISOString(); - const age = policy.category === 'audit' ? 'o.created_at < @numericCutoff' : `o.${policy.clock} < @cutoff`; + const age = + policy.category === 'audit' + ? 'o.created_at < @numericCutoff' + : `o.${policy.clock} < @cutoff`; const failed = policy.failed ?? '0'; - const enabled = days > 0 || policy.category === 'version' && settings.versionKeepCount > 0; + const enabled = days > 0 || (policy.category === 'version' && settings.versionKeepCount > 0); // Keep age outside the correlated failure checks so SQLite can seek the // clock index before examining dependencies. Resolved failures start from // their controls, not another scan of the entire history table. const candidates: string[] = []; - if (enabled) candidates.push(`SELECT o.rowid FROM ${policy.table} o + if (enabled) + candidates.push(`SELECT o.rowid FROM ${policy.table} o WHERE ${policy.category === 'version' && !days ? '1' : age} AND NOT (${failed})`); - if (failureDays > 0 && policy.failed) candidates.push(`SELECT o.rowid FROM retention_controls c + if (failureDays > 0 && policy.failed) + candidates.push(`SELECT o.rowid FROM retention_controls c JOIN ${policy.table} o ON o.${policy.id}=c.record_id WHERE c.category=@category AND c.resolved_at<@failureCutoff AND c.resolved_at>=o.${policy.clock} AND (${failed})`); const minimum = policy.minimum === false ? 0 : settings.historyKeepCount; - const minimumScope = policy.category === 'observation' ? "WHERE observation_success=1 AND state='completed'" - : policy.category === 'operation' ? "WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)" : ''; - const versionCount = policy.category === 'version' ? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})` : ''; - return { sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible}) + const minimumScope = + policy.category === 'observation' + ? "WHERE observation_success=1 AND state='completed'" + : policy.category === 'operation' + ? 'WHERE history_expired_at IS NULL AND (routine=0 OR observation_success=0)' + : ''; + const versionCount = + policy.category === 'version' + ? `AND o.id NOT IN (SELECT id FROM script_versions v WHERE v.script_id=o.script_id ORDER BY version DESC LIMIT ${settings.versionKeepCount})` + : ''; + return { + sql: `FROM ${policy.table} o WHERE ${candidates.length ? `o.rowid IN (${candidates.join(' UNION ALL ')})` : '0'} AND (${policy.eligible}) AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category=@category AND c.record_id=CAST(o.${policy.id} AS TEXT) AND c.protected=1) AND o.rowid NOT IN (SELECT rowid FROM ${policy.table} ${minimumScope} ORDER BY ${policy.clock} DESC,rowid DESC LIMIT ${minimum}) ${versionCount}`, - params: { category: policy.category, cutoff: cut(days), numericCutoff: Date.now() - days * 86400_000, failureCutoff: cut(failureDays) } }; + params: { + category: policy.category, + cutoff: cut(days), + numericCutoff: Date.now() - days * 86400_000, + failureCutoff: cut(failureDays) + } + }; } // better-sqlite3 accepts unused named bindings, keeping selection reusable. export function retentionPreview(settings = fleetSettings()) { initialize(); - return policies().map(policy => { + return policies().map((policy) => { const query = selection(policy, settings); - const summary = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(COALESCE(${policy.bytes ?? '0'},0)),0) AS payloadBytes ${query.sql}`).get(query.params) as { eligible: number; payloadBytes: number }; - const records = db().prepare(`SELECT CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 20`).all(query.params) as { id: string }[]; + const summary = db() + .prepare( + `SELECT COUNT(*) AS eligible,COALESCE(SUM(COALESCE(${policy.bytes ?? '0'},0)),0) AS payloadBytes ${query.sql}` + ) + .get(query.params) as { eligible: number; payloadBytes: number }; + const records = db() + .prepare( + `SELECT CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 20` + ) + .all(query.params) as { id: string }[]; return { category: policy.category, label: policy.label, ...summary, records }; }); } export function retentionControls() { - return db().prepare('SELECT category,record_id AS id,protected,resolved_at AS resolvedAt FROM retention_controls WHERE protected=1 OR resolved_at IS NOT NULL ORDER BY category,record_id').all(); + return db() + .prepare( + 'SELECT category,record_id AS id,protected,resolved_at AS resolvedAt FROM retention_controls WHERE protected=1 OR resolved_at IS NOT NULL ORDER BY category,record_id' + ) + .all(); } -export function setRetentionControl(input: { category: string; id: string; protected?: boolean; resolve?: boolean }) { +export function setRetentionControl(input: { + category: string; + id: string; + protected?: boolean; + resolve?: boolean; +}) { initialize(); ensureGameFileSchema(db()); - const policy = policies().find(policy => policy.category === input.category) - ?? (['game-file','upload'].includes(input.category) ? { category: input.category, table: 'game_file_versions',id: 'id',failed: undefined } : undefined); - if (!policy || typeof input.id !== 'string' || !input.id || input.id.length > 200 - || (input.protected !== undefined && typeof input.protected !== 'boolean') || (input.resolve !== undefined && typeof input.resolve !== 'boolean')) throw new Error('Choose a valid history category and record ID.'); - const row = db().prepare(`SELECT 1 FROM ${policy.table} o WHERE CAST(o.${policy.id} AS TEXT)=? ${policy.table === 'ec_host_operations' ? 'AND o.history_expired_at IS NULL' : ''} ${input.resolve ? `AND (${policy.failed ?? '0'})` : ''}`).get(input.id); - if (!row) throw new Error(input.resolve ? 'Only a failed record can be marked resolved.' : 'History record was not found.'); - if (input.protected && (db().prepare('SELECT 1 FROM retention_expirations WHERE category=? AND record_id=?').get(input.category,input.id) - || ['game-file','upload'].includes(input.category) && db().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(input.id))) { - throw new Error('These files have expired or cleanup has already started. Protection cannot restore them.'); + const policy = + policies().find((policy) => policy.category === input.category) ?? + (['game-file', 'upload'].includes(input.category) + ? { category: input.category, table: 'game_file_versions', id: 'id', failed: undefined } + : undefined); + if ( + !policy || + typeof input.id !== 'string' || + !input.id || + input.id.length > 200 || + (input.protected !== undefined && typeof input.protected !== 'boolean') || + (input.resolve !== undefined && typeof input.resolve !== 'boolean') + ) + throw new Error('Choose a valid history category and record ID.'); + const row = db() + .prepare( + `SELECT 1 FROM ${policy.table} o WHERE CAST(o.${policy.id} AS TEXT)=? ${policy.table === 'ec_host_operations' ? 'AND o.history_expired_at IS NULL' : ''} ${input.resolve ? `AND (${policy.failed ?? '0'})` : ''}` + ) + .get(input.id); + if (!row) + throw new Error( + input.resolve + ? 'Only a failed record can be marked resolved.' + : 'History record was not found.' + ); + if ( + input.protected && + (db() + .prepare('SELECT 1 FROM retention_expirations WHERE category=? AND record_id=?') + .get(input.category, input.id) || + (['game-file', 'upload'].includes(input.category) && + db().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(input.id))) + ) { + throw new Error( + 'These files have expired or cleanup has already started. Protection cannot restore them.' + ); } - db().prepare(`INSERT INTO retention_controls(category,record_id,protected,resolved_at) VALUES(?,?,?,?) - ON CONFLICT(category,record_id) DO UPDATE SET protected=COALESCE(?,protected),resolved_at=COALESCE(?,resolved_at)`) - .run(input.category,input.id,Number(input.protected ?? false),input.resolve ? new Date().toISOString() : null, - input.protected === undefined ? null : Number(input.protected),input.resolve ? new Date().toISOString() : null); + db() + .prepare( + `INSERT INTO retention_controls(category,record_id,protected,resolved_at) VALUES(?,?,?,?) + ON CONFLICT(category,record_id) DO UPDATE SET protected=COALESCE(?,protected),resolved_at=COALESCE(?,resolved_at)` + ) + .run( + input.category, + input.id, + Number(input.protected ?? false), + input.resolve ? new Date().toISOString() : null, + input.protected === undefined ? null : Number(input.protected), + input.resolve ? new Date().toISOString() : null + ); } /** Each policy removes at most 500 eligible records per pass; dependencies are checked again under the transaction. */ @@ -232,111 +452,258 @@ export function pruneHistory(only?: string): number { initialize(); if (only === 'observation') return pruneRoutineObservations(); let removed = only ? 0 : pruneRoutineObservations(); - for (const policy of policies().filter(policy => policy.category !== 'observation' && (!only || policy.category === only))) { + for (const policy of policies().filter( + (policy) => policy.category !== 'observation' && (!only || policy.category === only) + )) { removed += db().transaction(() => { const query = selection(policy); - const rows = db().prepare(`SELECT o.rowid AS rid,CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 500`).all(query.params) as { rid: number; id: string }[]; + const rows = db() + .prepare( + `SELECT o.rowid AS rid,CAST(o.${policy.id} AS TEXT) AS id ${query.sql} ORDER BY o.${policy.clock},o.rowid LIMIT 500` + ) + .all(query.params) as { rid: number; id: string }[]; for (const row of rows) { if (policy.category === 'provisioning') { - db().prepare('DELETE FROM native_provisioning WHERE deployment_id=?').run(row.id); - db().prepare('DELETE FROM ec_host_migrations WHERE deployment_id=?').run(row.id); - db().prepare('DELETE FROM restart_rollout WHERE operation_id=? AND healthy_at IS NOT NULL').run(row.id); + db() + .prepare('DELETE FROM native_provisioning WHERE deployment_id=?') + .run(row.id); + db() + .prepare('DELETE FROM ec_host_migrations WHERE deployment_id=?') + .run(row.id); + db() + .prepare( + 'DELETE FROM restart_rollout WHERE operation_id=? AND healthy_at IS NOT NULL' + ) + .run(row.id); } if (policy.category === 'operation' || policy.category === 'observation') { - db().prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?').run(row.id); + db() + .prepare('DELETE FROM ec_host_operation_events WHERE operation_id=?') + .run(row.id); if (policy.category === 'operation') { - const payload = db().prepare(`SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`).get(row.id) as { payload: string }; - db().prepare(`UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?`) - .run(createHash('sha256').update(payload.payload).digest('hex'),new Date().toISOString(),row.id); + const payload = db() + .prepare( + `SELECT COALESCE(p.payload,o.payload) AS payload FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?` + ) + .get(row.id) as { payload: string }; + db() + .prepare( + `UPDATE ec_host_operations SET request_hash=COALESCE(request_hash,?),payload='{}',result='{"historyExpired":true}',error=NULL,history_expired_at=? WHERE id=?` + ) + .run( + createHash('sha256').update(payload.payload).digest('hex'), + new Date().toISOString(), + row.id + ); } - db().prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?').run(row.id); + db() + .prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id=?') + .run(row.id); } - if (policy.category === 'workflow-payload' || policy.category === 'operation-payload') { + if ( + policy.category === 'workflow-payload' || + policy.category === 'operation-payload' + ) { expirePayload(policy.category, row.id); - db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); + db() + .prepare( + 'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)' + ) + .run(policy.category, row.id, new Date().toISOString()); } else if (policy.category === 'beta-file-content') { - db().prepare("UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?").run(row.id); - db().prepare('UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?').run(row.id); - db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); + db() + .prepare('UPDATE beta_revision_files SET content_hash=NULL WHERE beta_id=?') + .run(row.id); + db() + .prepare( + 'UPDATE beta_file_snapshots SET prior_content_hash=NULL WHERE beta_id=?' + ) + .run(row.id); + db() + .prepare( + 'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)' + ) + .run(policy.category, row.id, new Date().toISOString()); } else if (policy.category === 'file-content') { - db().prepare('UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?').run(row.id); - db().prepare("UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')").run(row.id); - db().prepare('INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)').run(policy.category,row.id,new Date().toISOString()); + db() + .prepare( + 'UPDATE deployment_files SET content_before_hash=NULL,content_after_hash=NULL WHERE deployment_id=?' + ) + .run(row.id); + db() + .prepare( + "UPDATE staged_file_changes SET content_hash=NULL WHERE source_kind='campaign' AND source_id=? AND state IN ('applied','cancelled')" + ) + .run(row.id); + db() + .prepare( + 'INSERT OR IGNORE INTO retention_expirations(category,record_id,expired_at) VALUES(?,?,?)' + ) + .run(policy.category, row.id, new Date().toISOString()); } else if (policy.category !== 'operation') { db().prepare(`DELETE FROM ${policy.table} WHERE rowid=?`).run(row.rid); - db().prepare('DELETE FROM retention_controls WHERE category=? AND record_id=?').run(policy.category,row.id); + db() + .prepare('DELETE FROM retention_controls WHERE category=? AND record_id=?') + .run(policy.category, row.id); } } return rows.length; })(); } const blobs = pruneFileBlobs(); - if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) db().pragma('incremental_vacuum(256)'); + if ((removed || blobs) && Number(db().pragma('freelist_count', { simple: true })) >= 256) + db().pragma('incremental_vacuum(256)'); return removed; } function samplePolicies(s = fleetSettings()) { const iso = (days: number) => new Date(Date.now() - days * 86400_000).toISOString(); return [ - { category: 'host-resources', label: 'Host resource samples', table: 'ec_host_resource_samples', condition: 'sampled_at < ?', value: iso(s.hostResourceKeepDays), enabled: s.hostResourceKeepDays > 0 }, - { category: 'instance-resources', label: 'Instance resource samples', table: 'ec_instance_resource_samples', condition: 'bucket < ?', value: iso(Math.max(1,s.instanceResourceKeepDays)), enabled: s.instanceResourceKeepDays > 0 }, - { category: 'metrics', label: 'Fleet metrics samples', table: 'metric_samples', condition: 'at < ?', value: Date.now() - s.metricsKeepDays * 86400_000, enabled: s.metricsKeepDays > 0 }, - { category: 'telemetry', label: 'Telemetry diagnostics (age expiry)', table: 'telemetry_requests', condition: 'received_at < ?', value: iso(s.telemetryKeepDays), enabled: s.telemetryKeepDays > 0 } + { + category: 'host-resources', + label: 'Host resource samples', + table: 'ec_host_resource_samples', + condition: 'sampled_at < ?', + value: iso(s.hostResourceKeepDays), + enabled: s.hostResourceKeepDays > 0 + }, + { + category: 'instance-resources', + label: 'Instance resource samples', + table: 'ec_instance_resource_samples', + condition: 'bucket < ?', + value: iso(Math.max(1, s.instanceResourceKeepDays)), + enabled: s.instanceResourceKeepDays > 0 + }, + { + category: 'metrics', + label: 'Fleet metrics samples', + table: 'metric_samples', + condition: 'at < ?', + value: Date.now() - s.metricsKeepDays * 86400_000, + enabled: s.metricsKeepDays > 0 + }, + { + category: 'telemetry', + label: 'Telemetry diagnostics (age expiry)', + table: 'telemetry_requests', + condition: 'received_at < ?', + value: iso(s.telemetryKeepDays), + enabled: s.telemetryKeepDays > 0 + } ]; } export function sampleRetentionPreview(settings = fleetSettings()) { initialize(); - const result = samplePolicies(settings).filter(p => p.category !== 'telemetry').map(p => ({ category: p.category,label: p.label,records: [],payloadBytes: 0, - eligible: p.enabled ? (db().prepare(`SELECT COUNT(*) AS n FROM ${p.table} WHERE ${p.condition}`).get(p.value) as { n: number }).n : 0 })); - const telemetry = db().prepare(`SELECT COUNT(*) AS eligible,COALESCE(SUM(bytes),0) AS payloadBytes FROM ( + const result = samplePolicies(settings) + .filter((p) => p.category !== 'telemetry') + .map((p) => ({ + category: p.category, + label: p.label, + records: [], + payloadBytes: 0, + eligible: p.enabled + ? ( + db() + .prepare(`SELECT COUNT(*) AS n FROM ${p.table} WHERE ${p.condition}`) + .get(p.value) as { n: number } + ).n + : 0 + })); + const telemetry = db() + .prepare( + `SELECT COUNT(*) AS eligible,COALESCE(SUM(bytes),0) AS payloadBytes FROM ( SELECT received_at,length(payload) AS bytes,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS per_server, ROW_NUMBER() OVER (ORDER BY id DESC) AS overall FROM telemetry_requests) - WHERE (? > 0 AND received_at < ?) OR (? > 0 AND per_server > ?) OR (? > 0 AND overall > ?)`) - .get(settings.telemetryKeepDays,new Date(Date.now()-settings.telemetryKeepDays*86400_000).toISOString(),settings.telemetryMaxPerServer, - settings.telemetryMaxPerServer,settings.telemetryMaxTotal,settings.telemetryMaxTotal) as {eligible:number;payloadBytes:number}; - result.push({category:'telemetry',label:'Telemetry diagnostics (age and count limits)',records:[],...telemetry}); + WHERE (? > 0 AND received_at < ?) OR (? > 0 AND per_server > ?) OR (? > 0 AND overall > ?)` + ) + .get( + settings.telemetryKeepDays, + new Date(Date.now() - settings.telemetryKeepDays * 86400_000).toISOString(), + settings.telemetryMaxPerServer, + settings.telemetryMaxPerServer, + settings.telemetryMaxTotal, + settings.telemetryMaxTotal + ) as { eligible: number; payloadBytes: number }; + result.push({ + category: 'telemetry', + label: 'Telemetry diagnostics (age and count limits)', + records: [], + ...telemetry + }); return result; } export function pruneSampleHistory() { initialize(); let removed = 0; - for (const p of samplePolicies().filter(p => p.enabled)) removed += db().prepare(`DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)`).run(p.value).changes; + for (const p of samplePolicies().filter((p) => p.enabled)) + removed += db() + .prepare( + `DELETE FROM ${p.table} WHERE rowid IN (SELECT rowid FROM ${p.table} WHERE ${p.condition} LIMIT 5000)` + ) + .run(p.value).changes; const s = fleetSettings(); - if (s.telemetryMaxPerServer) removed += db().prepare(`DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM - (SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)`).run(s.telemetryMaxPerServer).changes; - if (s.telemetryMaxTotal) removed += db().prepare('DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)').run(s.telemetryMaxTotal).changes; + if (s.telemetryMaxPerServer) + removed += db() + .prepare( + `DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM + (SELECT id,ROW_NUMBER() OVER (PARTITION BY server ORDER BY id DESC) AS rank FROM telemetry_requests) WHERE rank>? LIMIT 5000)` + ) + .run(s.telemetryMaxPerServer).changes; + if (s.telemetryMaxTotal) + removed += db() + .prepare( + 'DELETE FROM telemetry_requests WHERE id IN (SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 5000 OFFSET ?)' + ) + .run(s.telemetryMaxTotal).changes; return removed; } /** Drain every eligible batch, yielding between transactions so reads and receipts can run. */ export async function drainRetention(): Promise { - let total = 0, logCursor = 0; + let total = 0, + logCursor = 0; initialize(); while (true) { const removed = pruneHistory() + pruneSampleHistory(); total += removed; if (!removed) break; - await new Promise(resolve => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); } // Remove expired rows first; only compact bodies in history that will survive. - if (db().prepare("SELECT 1 FROM settings WHERE key='log_history_metadata_v1'").get()) return total; + if (db().prepare("SELECT 1 FROM settings WHERE key='log_history_metadata_v1'").get()) + return total; while (true) { - const rows = db().prepare(`SELECT rowid AS rid,id,result FROM ec_host_operations + const rows = db() + .prepare( + `SELECT rowid AS rid,id,result FROM ec_host_operations WHERE rowid>? AND kind='logs.read' AND json_valid(result) AND (json_type(result,'$.lines')='array' OR json_type(result,'$.contentBase64')='text' OR json_type(result,'$.contentSha256')='text') - ORDER BY rowid LIMIT 50`).all(logCursor) as { rid: number; id: string; result: string }[]; + ORDER BY rowid LIMIT 50` + ) + .all(logCursor) as { rid: number; id: string; result: string }[]; db().transaction(() => { for (const row of rows) { - db().prepare('UPDATE ec_host_operations SET result=? WHERE id=?') + db() + .prepare('UPDATE ec_host_operations SET result=? WHERE id=?') .run(JSON.stringify(logResultMetadata(JSON.parse(row.result))), row.id); - db().prepare("DELETE FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?").run(row.id); + db() + .prepare( + "DELETE FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?" + ) + .run(row.id); } - if (!rows.length) db().prepare("INSERT OR IGNORE INTO settings(key,value) VALUES('log_history_metadata_v1','1')").run(); + if (!rows.length) + db() + .prepare( + "INSERT OR IGNORE INTO settings(key,value) VALUES('log_history_metadata_v1','1')" + ) + .run(); })(); if (!rows.length) return total; logCursor = rows.at(-1)!.rid; - await new Promise(resolve => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); } } @@ -345,14 +712,30 @@ function pruneRoutineObservations(): number { if (!settings.observationKeepDays) return 0; const handle = db(); return handle.transaction(() => { - const rows = handle.prepare(`SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' + const rows = handle + .prepare( + `SELECT id FROM ec_host_operations WHERE observation_success=1 AND state='completed' AND updated_atrow.id)); - handle.prepare('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); - handle.prepare('DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))').run(ids); - return handle.prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))').run(ids).changes; + const ids = JSON.stringify(rows.map((row) => row.id)); + handle + .prepare( + 'DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT value FROM json_each(?))' + ) + .run(ids); + handle + .prepare( + 'DELETE FROM ec_host_operation_payloads WHERE operation_id IN (SELECT value FROM json_each(?))' + ) + .run(ids); + return handle + .prepare('DELETE FROM ec_host_operations WHERE id IN (SELECT value FROM json_each(?))') + .run(ids).changes; })(); } diff --git a/src/lib/server/settings.ts b/src/lib/server/settings.ts index 9b1dd24..1fcc65f 100644 --- a/src/lib/server/settings.ts +++ b/src/lib/server/settings.ts @@ -3,12 +3,16 @@ import { db } from './db/index.js'; import { FLEET_SETTINGS_KEY } from './db/fleet-settings.js'; export function fleetSettings(): FleetSettings { - const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string }; + const row = db() + .prepare('SELECT value FROM settings WHERE key = ?') + .get(FLEET_SETTINGS_KEY) as { value: string }; return JSON.parse(row.value); } export function saveFleetSettings(input: unknown): FleetSettings { const settings = validateFleetSettings(input); - db().prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(settings), FLEET_SETTINGS_KEY); + db() + .prepare('UPDATE settings SET value = ? WHERE key = ?') + .run(JSON.stringify(settings), FLEET_SETTINGS_KEY); return settings; } diff --git a/src/lib/server/watchdog/index.ts b/src/lib/server/watchdog/index.ts index 7d42b1d..ec80df4 100644 --- a/src/lib/server/watchdog/index.ts +++ b/src/lib/server/watchdog/index.ts @@ -1,2 +1,2 @@ // Compatibility for existing callers; Maintenance owns recovery. -export * from "../maintenance/recovery.js"; +export * from '../maintenance/recovery.js'; diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts index 8288ec2..ed96e03 100644 --- a/src/lib/telemetry.ts +++ b/src/lib/telemetry.ts @@ -15,7 +15,11 @@ export interface TelemetryPlayerStats extends TelemetryPlayer { export type GameEvent = { server: string; map: string } & ( | { event: 'game_start'; players: TelemetryPlayer[] } - | { event: 'round_change' | 'heartbeat' | 'game_end'; round: number; players: TelemetryPlayerStats[] } + | { + event: 'round_change' | 'heartbeat' | 'game_end'; + round: number; + players: TelemetryPlayerStats[]; + } ); export interface PlayerTiming { @@ -39,39 +43,58 @@ export type GameTelemetry = GameEvent & { /** Validate the four script payloads; beta state is never part of the protocol. */ export function parseGameEvent(value: unknown): GameEvent { - if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Payload must be a JSON object'); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Payload must be a JSON object'); const body = value as Record; - if (!['game_start', 'round_change', 'heartbeat', 'game_end'].includes(String(body.event))) throw new Error('event must be game_start, round_change, heartbeat, or game_end'); + if (!['game_start', 'round_change', 'heartbeat', 'game_end'].includes(String(body.event))) + throw new Error('event must be game_start, round_change, heartbeat, or game_end'); for (const key of ['server', 'map'] as const) { - if (typeof body[key] !== 'string' || !body[key].trim() || body[key].length > 255) throw new Error(`${key} must be a nonempty string (up to 255 characters)`); + if (typeof body[key] !== 'string' || !body[key].trim() || body[key].length > 255) + throw new Error(`${key} must be a nonempty string (up to 255 characters)`); } if (!Array.isArray(body.players)) throw new Error('players must be an array'); const guids = new Set(); const players = body.players.map((value, index) => { - if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`players[${index}] must be an object`); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error(`players[${index}] must be an object`); const player = value as Record; for (const key of ['name', 'guid'] as const) { - if (typeof player[key] !== 'string' || !player[key].trim()) throw new Error(`players[${index}].${key} must be a nonempty string`); + if (typeof player[key] !== 'string' || !player[key].trim()) + throw new Error(`players[${index}].${key} must be a nonempty string`); } - const identity: TelemetryPlayer = { name: player.name as string, guid: player.guid as string }; + const identity: TelemetryPlayer = { + name: player.name as string, + guid: player.guid as string + }; if (player.ping !== undefined) { - if (player.ping !== null && (!Number.isSafeInteger(player.ping) || (player.ping as number) < 0)) { + if ( + player.ping !== null && + (!Number.isSafeInteger(player.ping) || (player.ping as number) < 0) + ) { throw new Error(`players[${index}].ping must be a nonnegative integer or null`); } identity.ping = player.ping as number | null; } - if (guids.has(identity.guid)) throw new Error(`players[${index}].guid must be unique within the report`); + if (guids.has(identity.guid)) + throw new Error(`players[${index}].guid must be unique within the report`); guids.add(identity.guid); if (body.event === 'game_start') return identity; const stats = {} as Omit; for (const key of ['kills', 'headshots', 'downs', 'revives', 'score'] as const) { - if (!Number.isSafeInteger(player[key]) || (player[key] as number) < 0) throw new Error(`players[${index}].${key} must be a nonnegative integer`); + if (!Number.isSafeInteger(player[key]) || (player[key] as number) < 0) + throw new Error(`players[${index}].${key} must be a nonnegative integer`); stats[key] = player[key] as number; } return { ...identity, ...stats }; }); const common = { server: body.server as string, map: body.map as string }; if (body.event === 'game_start') return { ...common, event: 'game_start', players }; - if (!Number.isSafeInteger(body.round) || (body.round as number) < 0) throw new Error('round must be a nonnegative integer'); - return { ...common, event: body.event as 'round_change' | 'heartbeat' | 'game_end', round: body.round as number, players: players as TelemetryPlayerStats[] }; + if (!Number.isSafeInteger(body.round) || (body.round as number) < 0) + throw new Error('round must be a nonnegative integer'); + return { + ...common, + event: body.event as 'round_change' | 'heartbeat' | 'game_end', + round: body.round as number, + players: players as TelemetryPlayerStats[] + }; } diff --git a/src/lib/time-zone.ts b/src/lib/time-zone.ts index b577832..cb5f18a 100644 --- a/src/lib/time-zone.ts +++ b/src/lib/time-zone.ts @@ -5,15 +5,30 @@ export function validTimeZone(value: unknown): value is string { try { new Intl.DateTimeFormat('en-US', { timeZone: value }); return true; - } catch { return false; } + } catch { + return false; + } } export function availableTimeZones(): string[] { // Older browsers can still select common zones when enumeration is unavailable. - const fallback = [DEFAULT_TIME_ZONE, 'America/Chicago', 'America/Denver', 'America/Los_Angeles', - 'America/Anchorage', 'Pacific/Honolulu', 'Europe/London', 'Europe/Paris', - 'Asia/Kolkata', 'Asia/Tokyo', 'Australia/Sydney']; - const zones = typeof Intl.supportedValuesOf === 'function' ? Intl.supportedValuesOf('timeZone') : fallback; + const fallback = [ + DEFAULT_TIME_ZONE, + 'America/Chicago', + 'America/Denver', + 'America/Los_Angeles', + 'America/Anchorage', + 'Pacific/Honolulu', + 'Europe/London', + 'Europe/Paris', + 'Asia/Kolkata', + 'Asia/Tokyo', + 'Australia/Sydney' + ]; + const zones = + typeof Intl.supportedValuesOf === 'function' + ? Intl.supportedValuesOf('timeZone') + : fallback; return [...new Set([DEFAULT_TIME_ZONE, 'UTC', ...zones])].sort(); } @@ -22,7 +37,10 @@ export type Timestamp = string | number | Date | null | undefined; /** Database timestamps without an offset are UTC, never browser-local time. */ function parseTimestamp(value: Exclude): Date { if (value instanceof Date) return value; - if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(value)) { + if ( + typeof value === 'string' && + /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(value) + ) { return new Date(value.replace(' ', 'T') + 'Z'); } return new Date(value); @@ -37,7 +55,12 @@ export function timestampISO(value: Timestamp): string | undefined { export function createTimeFormatter(zone: string) { const timeZone = validTimeZone(zone) ? zone : DEFAULT_TIME_ZONE; const date: Intl.DateTimeFormatOptions = { year: 'numeric', month: 'short', day: 'numeric' }; - const time: Intl.DateTimeFormatOptions = { hour: 'numeric', minute: '2-digit', second: '2-digit', timeZoneName: 'short' }; + const time: Intl.DateTimeFormatOptions = { + hour: 'numeric', + minute: '2-digit', + second: '2-digit', + timeZoneName: 'short' + }; function formatter(options: Intl.DateTimeFormatOptions) { const format = new Intl.DateTimeFormat('en-US', { ...options, timeZone }); return (value: Timestamp): string => { @@ -46,11 +69,23 @@ export function createTimeFormatter(zone: string) { return Number.isFinite(parsed.getTime()) ? format.format(parsed) : '—'; }; } - const absolute = formatter({ ...date, hour: 'numeric', minute: '2-digit', timeZoneName: 'short' }); + const absolute = formatter({ + ...date, + hour: 'numeric', + minute: '2-digit', + timeZoneName: 'short' + }); const clock = formatter({ hour: 'numeric', minute: '2-digit' }); - const calendar = new Intl.DateTimeFormat('en-US', { timeZone, year: 'numeric', month: 'numeric', day: 'numeric' }); + const calendar = new Intl.DateTimeFormat('en-US', { + timeZone, + year: 'numeric', + month: 'numeric', + day: 'numeric' + }); const calendarDay = (value: Date) => { - const parts = Object.fromEntries(calendar.formatToParts(value).map(part => [part.type, part.value])); + const parts = Object.fromEntries( + calendar.formatToParts(value).map((part) => [part.type, part.value]) + ); return Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day)); }; function relative(value: Timestamp, now = Date.now()): string { @@ -64,7 +99,8 @@ export function createTimeFormatter(zone: string) { if (age < 3_600_000) return `${Math.floor(age / 60_000)} min ago`; if (age < 86_400_000) return `${Math.floor(age / 3_600_000)} h ago`; // Calendar arithmetic keeps Yesterday correct across DST and year changes. - if (calendarDay(new Date(now)) - calendarDay(parsed) === 86_400_000) return `Yesterday, ${clock(parsed)}`; + if (calendarDay(new Date(now)) - calendarDay(parsed) === 86_400_000) + return `Yesterday, ${clock(parsed)}`; return absolute(parsed); } return { diff --git a/src/lib/ui/activity-history.ts b/src/lib/ui/activity-history.ts index be99e78..db4cbd1 100644 --- a/src/lib/ui/activity-history.ts +++ b/src/lib/ui/activity-history.ts @@ -1,7 +1,12 @@ import { parseActivityQuery, type ActivityQuery, type ActivitySource } from '../activity.js'; import { createTimeFormatter } from '../time-zone.js'; -export interface ActivityScope { server?: string; source?: ActivitySource; excludeProvisioning?: boolean; excludeMaintenance?: boolean } +export interface ActivityScope { + server?: string; + source?: ActivitySource; + excludeProvisioning?: boolean; + excludeMaintenance?: boolean; +} const keys = ['server', 'host', 'type', 'source', 'state', 'days', 'q', 'limit', 'offset'] as const; /** Scoped views always retain their server/source, including after reset or a pasted URL. */ @@ -9,8 +14,10 @@ export function activityQuery(url: URL, scope: ActivityScope = {}): ActivityQuer const params = new URLSearchParams(url.searchParams); if (scope.server !== undefined) params.set('server', scope.server); if (scope.source !== undefined) params.set('source', scope.source); - if (scope.excludeProvisioning !== undefined) params.set('excludeProvisioning', String(scope.excludeProvisioning)); - if (scope.excludeMaintenance !== undefined) params.set('excludeMaintenance', String(scope.excludeMaintenance)); + if (scope.excludeProvisioning !== undefined) + params.set('excludeProvisioning', String(scope.excludeProvisioning)); + if (scope.excludeMaintenance !== undefined) + params.set('excludeMaintenance', String(scope.excludeMaintenance)); return parseActivityQuery(params); } @@ -23,13 +30,17 @@ export function activityApiHref(url: URL, scope: ActivityScope = {}): string { } /** Preserve the containing tab and return context while paging or changing filters. */ -export function activityHistoryHref(url: URL, changes: Partial>): string { +export function activityHistoryHref( + url: URL, + changes: Partial> +): string { const next = new URL(url); if (!Object.hasOwn(changes, 'offset')) next.searchParams.delete('offset'); for (const key of keys) { if (changes[key] === undefined) continue; const value = changes[key]?.trim(); - if (value && !(['offset', 'days'].includes(key) && value === '0')) next.searchParams.set(key, value); + if (value && !(['offset', 'days'].includes(key) && value === '0')) + next.searchParams.set(key, value); else next.searchParams.delete(key); } return next.pathname + next.search + next.hash; diff --git a/src/lib/ui/autoscaling.ts b/src/lib/ui/autoscaling.ts index e1cc26f..dac55ae 100644 --- a/src/lib/ui/autoscaling.ts +++ b/src/lib/ui/autoscaling.ts @@ -15,29 +15,78 @@ export function scalingReason(reason: string): string { } export function policyStatus(view: ScalingView, globallyPaused: boolean) { - if (view.policy.pausedReason) return { label: 'Needs attention', reason: scalingReason(view.policy.pausedReason), tone: 'warning' as const }; - if (globallyPaused || view.policy.mode === 'off') return { - label: globallyPaused ? 'Globally paused' : 'Paused', - reason: view.unsettled.length ? 'No new scaling requests. Outstanding jobs are still tracked.' : 'No new scaling requests.', - tone: 'neutral' as const + if (view.policy.pausedReason) + return { + label: 'Needs attention', + reason: scalingReason(view.policy.pausedReason), + tone: 'warning' as const + }; + if (globallyPaused || view.policy.mode === 'off') + return { + label: globallyPaused ? 'Globally paused' : 'Paused', + reason: view.unsettled.length + ? 'No new scaling requests. Outstanding jobs are still tracked.' + : 'No new scaling requests.', + tone: 'neutral' as const + }; + if (view.decision.action === 'create') + return { + label: 'Adding capacity', + reason: scalingReason(view.decision.reason), + tone: 'info' as const + }; + if (view.decision.action === 'destroy') + return { + label: 'Removing surplus', + reason: scalingReason(view.decision.reason), + tone: 'info' as const + }; + if ( + view.decision.reason.endsWith('Empty spare target is satisfied.') && + view.decision.deficit === 0 + ) + return { label: 'Spare targets satisfied', reason: '', tone: 'online' as const }; + if (view.decision.reason.endsWith('Waiting for a continuous empty interval.')) + return { + label: 'Waiting to remove surplus', + reason: `Lobbies must stay empty for ${view.policy.emptyMinutes} minutes.`, + tone: 'neutral' as const + }; + return { + label: 'Scaling on hold', + reason: scalingReason(view.decision.reason), + tone: 'warning' as const }; - if (view.decision.action === 'create') return { label: 'Adding capacity', reason: scalingReason(view.decision.reason), tone: 'info' as const }; - if (view.decision.action === 'destroy') return { label: 'Removing surplus', reason: scalingReason(view.decision.reason), tone: 'info' as const }; - if (view.decision.reason.endsWith('Empty spare target is satisfied.') && view.decision.deficit === 0) return { label: 'Spare targets satisfied', reason: '', tone: 'online' as const }; - if (view.decision.reason.endsWith('Waiting for a continuous empty interval.')) return { label: 'Waiting to remove surplus', reason: `Lobbies must stay empty for ${view.policy.emptyMinutes} minutes.`, tone: 'neutral' as const }; - return { label: 'Scaling on hold', reason: scalingReason(view.decision.reason), tone: 'warning' as const }; } export function lobbyGroups(view: ScalingView) { - const maps = view.policy.mapSlug === '*' ? zombieMaps(view.policy.game) : zombieMaps(view.policy.game).filter(map => map.slug === view.policy.mapSlug); + const maps = + view.policy.mapSlug === '*' + ? zombieMaps(view.policy.game) + : zombieMaps(view.policy.game).filter((map) => map.slug === view.policy.mapSlug); const assigned = new Set(); - const groups = maps.map(map => { - const members = view.snapshot.members.filter(member => (member.mapSlug ?? parseServerName(member.name)?.mapSlug) === map.slug); - members.forEach(member => assigned.add(member)); - return { slug: map.slug, label: map.friendly, members, ready: members.filter(member => member.state === 'empty').length, target: view.policy.spares }; + const groups = maps.map((map) => { + const members = view.snapshot.members.filter( + (member) => (member.mapSlug ?? parseServerName(member.name)?.mapSlug) === map.slug + ); + members.forEach((member) => assigned.add(member)); + return { + slug: map.slug, + label: map.friendly, + members, + ready: members.filter((member) => member.state === 'empty').length, + target: view.policy.spares + }; }); - const unassigned = view.snapshot.members.filter(member => !assigned.has(member)); - if (unassigned.length) groups.push({ slug: 'unassigned', label: 'Other lobbies', members: unassigned, ready: unassigned.filter(member => member.state === 'empty').length, target: 0 }); + const unassigned = view.snapshot.members.filter((member) => !assigned.has(member)); + if (unassigned.length) + groups.push({ + slug: 'unassigned', + label: 'Other lobbies', + members: unassigned, + ready: unassigned.filter((member) => member.state === 'empty').length, + target: 0 + }); return groups; } @@ -54,12 +103,18 @@ export interface ScalingActivity { } export function scalingActivity(view: ScalingView): ScalingActivity[] { - const operationIds = new Set([...view.history.map(item => item.operationId), ...view.unsettled.map(item => item.id)].filter((id): id is string => !!id)); + const operationIds = new Set( + [ + ...view.history.map((item) => item.operationId), + ...view.unsettled.map((item) => item.id) + ].filter((id): id is string => !!id) + ); const groups = new Map(); for (const item of view.history) { // Older wait decisions store the job reference in their reason rather than operationId. const reference = item.decision.reason.match(/Provisioning ([\w-]+):/)?.[1]; - const operationId = item.operationId ?? (reference && operationIds.has(reference) ? reference : null); + const operationId = + item.operationId ?? (reference && operationIds.has(reference) ? reference : null); const key = operationId ?? item.id; const existing = groups.get(key); if (existing) { @@ -72,11 +127,18 @@ export function scalingActivity(view: ScalingView): ScalingActivity[] { existing.operationState = item.operationState; existing.reason = scalingReason(item.decision.reason); } - } else groups.set(key, { - id: key, at: item.at, lastSeenAt: item.lastSeenAt, operationId, - operationState: item.operationState, action: item.decision.action, - server: item.decision.server, reason: scalingReason(item.decision.reason), evaluations: 1 - }); + } else + groups.set(key, { + id: key, + at: item.at, + lastSeenAt: item.lastSeenAt, + operationId, + operationState: item.operationState, + action: item.decision.action, + server: item.decision.server, + reason: scalingReason(item.decision.reason), + evaluations: 1 + }); } return [...groups.values()].sort((a, b) => b.lastSeenAt - a.lastSeenAt); } @@ -84,10 +146,17 @@ export function scalingActivity(view: ScalingView): ScalingActivity[] { /** Advance observation ages even when polling fails, using the fleet freshness window. */ export function observationAge(at: number, now: number): { label: string; stale: boolean } { const age = now - at; - if (!Number.isFinite(at) || !Number.isFinite(age) || age < -1000) return { label: 'Check observation time', stale: true }; + if (!Number.isFinite(at) || !Number.isFinite(age) || age < -1000) + return { label: 'Check observation time', stale: true }; const seconds = Math.max(0, Math.floor(age / 1000)); - const elapsed = seconds < 60 ? `${seconds}s` : seconds < 3600 ? `${Math.floor(seconds / 60)}m` - : seconds < 86400 ? `${Math.floor(seconds / 3600)}h` : `${Math.floor(seconds / 86400)}d`; + const elapsed = + seconds < 60 + ? `${seconds}s` + : seconds < 3600 + ? `${Math.floor(seconds / 60)}m` + : seconds < 86400 + ? `${Math.floor(seconds / 3600)}h` + : `${Math.floor(seconds / 86400)}d`; const stale = !freshStatusTimestamp(new Date(at).toISOString(), now); return { label: `${stale ? 'Stale · ' : ''}${elapsed} ago`, stale }; } diff --git a/src/lib/ui/beta-updates.ts b/src/lib/ui/beta-updates.ts index 650ed01..1812daf 100644 --- a/src/lib/ui/beta-updates.ts +++ b/src/lib/ui/beta-updates.ts @@ -1,27 +1,40 @@ /** Coalesce pushes, serialize refreshes, and recover missed events after reconnect or tab activation. */ export function betaUpdates(load: () => Promise, betaId?: () => number): () => void { let timer: ReturnType | undefined; - let stopped = false, running = false, pending = false; + let stopped = false, + running = false, + pending = false; const schedule = () => { pending = true; if (stopped || running || timer || document.hidden) return; timer = setTimeout(async () => { timer = undefined; if (stopped || document.hidden) return; - running = true; pending = false; - try { await load(); } - catch { /* The page owns its error state; later events can retry. */ } - finally { running = false; if (pending) schedule(); } + running = true; + pending = false; + try { + await load(); + } catch { + /* The page owns its error state; later events can retry. */ + } finally { + running = false; + if (pending) schedule(); + } }, 200); }; - const changed = (event: Event) => { if (!betaId || (event as CustomEvent).detail?.betaId === betaId()) schedule(); }; - const visible = () => { if (!document.hidden) schedule(); }; + const changed = (event: Event) => { + if (!betaId || (event as CustomEvent).detail?.betaId === betaId()) schedule(); + }; + const visible = () => { + if (!document.hidden) schedule(); + }; window.addEventListener('ec:beta-changed', changed); window.addEventListener('ec:pipeline-connected', schedule); window.addEventListener('ec:fleet-changed', schedule); document.addEventListener('visibilitychange', visible); return () => { - stopped = true; clearTimeout(timer); + stopped = true; + clearTimeout(timer); window.removeEventListener('ec:beta-changed', changed); window.removeEventListener('ec:pipeline-connected', schedule); window.removeEventListener('ec:fleet-changed', schedule); diff --git a/src/lib/ui/betas.ts b/src/lib/ui/betas.ts index a803fe1..56325e3 100644 --- a/src/lib/ui/betas.ts +++ b/src/lib/ui/betas.ts @@ -1,21 +1,45 @@ import type { StatusTone } from './status.js'; export interface BetaDeploymentServer { - server: string; server_deleted_at?: string | null; stage: string; error: string | null; - deployed_at: string | null; restarted_at: string | null; healthy_at: string | null; - match_seen_at: string | null; provisioning_id: string | null; + server: string; + server_deleted_at?: string | null; + stage: string; + error: string | null; + deployed_at: string | null; + restarted_at: string | null; + healthy_at: string | null; + match_seen_at: string | null; + provisioning_id: string | null; } -export function betaStatus(beta: { state: string; testOutcome?: string | null; - servers: { stage: string; server_deleted_at?: string | null }[] }): { - key: string; label: string; tone: StatusTone; +export function betaStatus(beta: { + state: string; + testOutcome?: string | null; + servers: { stage: string; server_deleted_at?: string | null }[]; +}): { + key: string; + label: string; + tone: StatusTone; } { if (beta.state === 'promoted') return { key: 'promoted', label: 'Promoted', tone: 'online' }; - if (beta.state === 'rolled-back') return { key: 'rolled-back', label: 'Rolled back', tone: 'neutral' }; - if (beta.servers.some(server => server.server_deleted_at || server.stage === 'failed') || ['failed', 'inconclusive'].includes(beta.testOutcome ?? '')) { - return { key: 'attention', label: 'Needs attention', tone: beta.testOutcome === 'failed' || beta.servers.some(server => server.stage === 'failed') ? 'danger' : 'warning' }; + if (beta.state === 'rolled-back') + return { key: 'rolled-back', label: 'Rolled back', tone: 'neutral' }; + if ( + beta.servers.some((server) => server.server_deleted_at || server.stage === 'failed') || + ['failed', 'inconclusive'].includes(beta.testOutcome ?? '') + ) { + return { + key: 'attention', + label: 'Needs attention', + tone: + beta.testOutcome === 'failed' || + beta.servers.some((server) => server.stage === 'failed') + ? 'danger' + : 'warning' + }; } if (beta.testOutcome === 'paused') return { key: 'paused', label: 'Paused', tone: 'warning' }; - if (beta.state === 'ready-promote') return { key: 'ready', label: 'Ready to promote', tone: 'online' }; + if (beta.state === 'ready-promote') + return { key: 'ready', label: 'Ready to promote', tone: 'online' }; return { key: 'testing', label: 'Testing', tone: 'info' }; } diff --git a/src/lib/ui/campaigns.ts b/src/lib/ui/campaigns.ts index 79139a6..22214f2 100644 --- a/src/lib/ui/campaigns.ts +++ b/src/lib/ui/campaigns.ts @@ -14,53 +14,105 @@ export interface CampaignSummary { } export function campaignTitle(campaign: CampaignSummary): string { - const action = campaign.action === 'drift' ? 'Maintenance repair' - : campaign.action.charAt(0).toUpperCase() + campaign.action.slice(1); + const action = + campaign.action === 'drift' + ? 'Maintenance repair' + : campaign.action.charAt(0).toUpperCase() + campaign.action.slice(1); if (campaign.action === 'drift') return action; - if (campaign.action === 'rollback' && campaign.beta_id) return `${action} · Beta #${campaign.beta_id}`; + if (campaign.action === 'rollback' && campaign.beta_id) + return `${action} · Beta #${campaign.beta_id}`; if (campaign.scripts.length > 1) return `${action} · ${campaign.scripts.length} scripts`; const script = campaign.scripts[0] ?? campaign; - if (script.script_name) return `${action} · ${script.script_name}${script.script_version !== null ? ` v${script.script_version}` : ''}`; + if (script.script_name) + return `${action} · ${script.script_name}${script.script_version !== null ? ` v${script.script_version}` : ''}`; return campaign.beta_id ? `${action} · Beta #${campaign.beta_id}` : action; } export function campaignOutcome(campaign: Pick): { - status: 'attention' | 'progress' | 'rolled-back' | 'complete' | 'inactive'; label: string; tone: StatusTone; completed: number; + status: 'attention' | 'progress' | 'rolled-back' | 'complete' | 'inactive'; + label: string; + tone: StatusTone; + completed: number; } { - const completed = campaign.servers.filter(server => server.restart_state === 'done').length; - const live = campaign.servers.filter(server => !server.server_deleted_at); - const attention = live.filter(server => ['failed', 'held', 'manual'].includes(server.restart_state)); - if (attention.length) return { status: 'attention', label: `${attention.length} ${plural(attention.length, 'needs', 'need')} attention`, tone: attention.some(server => server.restart_state === 'failed') ? 'danger' : 'warning', completed }; - if (live.some(server => server.restart_state === 'pending')) return { status: 'progress', label: `${completed}/${campaign.servers.length} complete`, tone: 'info', completed }; - if (campaign.rolled_back_by) return { status: 'rolled-back', label: 'Rolled back', tone: 'neutral', completed }; - if (campaign.servers.length && completed === campaign.servers.length) return { status: 'complete', label: 'Complete', tone: 'online', completed }; - return { status: 'inactive', label: completed ? `${completed}/${campaign.servers.length} complete` : 'Inactive', tone: 'neutral', completed }; + const completed = campaign.servers.filter((server) => server.restart_state === 'done').length; + const live = campaign.servers.filter((server) => !server.server_deleted_at); + const attention = live.filter((server) => + ['failed', 'held', 'manual'].includes(server.restart_state) + ); + if (attention.length) + return { + status: 'attention', + label: `${attention.length} ${plural(attention.length, 'needs', 'need')} attention`, + tone: attention.some((server) => server.restart_state === 'failed') + ? 'danger' + : 'warning', + completed + }; + if (live.some((server) => server.restart_state === 'pending')) + return { + status: 'progress', + label: `${completed}/${campaign.servers.length} complete`, + tone: 'info', + completed + }; + if (campaign.rolled_back_by) + return { status: 'rolled-back', label: 'Rolled back', tone: 'neutral', completed }; + if (campaign.servers.length && completed === campaign.servers.length) + return { status: 'complete', label: 'Complete', tone: 'online', completed }; + return { + status: 'inactive', + label: completed ? `${completed}/${campaign.servers.length} complete` : 'Inactive', + tone: 'neutral', + completed + }; } /** Progress comes from recorded state, not the error field, which also contains routine stage messages. */ -export function campaignServerStatus(server: { - restart_state: string; server_deleted_at?: string | null; restarted_at: string | null; -}, files: { apply_state: string }[] = []): { group: CampaignServerGroup; label: string; tone: StatusTone } { - if (server.restart_state === 'done') return { group: 'complete', label: 'Complete', tone: 'online' }; - if (server.server_deleted_at) return { group: 'inactive', label: 'Deleted before activation', tone: 'neutral' }; - if (['skipped', 'cancelled'].includes(server.restart_state)) return { group: 'inactive', ...statusPresentation(server.restart_state) }; +export function campaignServerStatus( + server: { + restart_state: string; + server_deleted_at?: string | null; + restarted_at: string | null; + }, + files: { apply_state: string }[] = [] +): { group: CampaignServerGroup; label: string; tone: StatusTone } { + if (server.restart_state === 'done') + return { group: 'complete', label: 'Complete', tone: 'online' }; + if (server.server_deleted_at) + return { group: 'inactive', label: 'Deleted before activation', tone: 'neutral' }; + if (['skipped', 'cancelled'].includes(server.restart_state)) + return { group: 'inactive', ...statusPresentation(server.restart_state) }; if (['failed', 'manual', 'held'].includes(server.restart_state)) { - return { group: 'attention', ...statusPresentation(server.restart_state), tone: server.restart_state === 'failed' ? 'danger' : 'warning' }; + return { + group: 'attention', + ...statusPresentation(server.restart_state), + tone: server.restart_state === 'failed' ? 'danger' : 'warning' + }; } - if (server.restarted_at) return { group: 'processing', label: 'Verifying health', tone: 'info' }; - if (files.some(file => file.apply_state === 'applying')) return { group: 'processing', label: 'Applying files', tone: 'info' }; + if (server.restarted_at) + return { group: 'processing', label: 'Verifying health', tone: 'info' }; + if (files.some((file) => file.apply_state === 'applying')) + return { group: 'processing', label: 'Applying files', tone: 'info' }; return { group: 'queued', label: 'Queued', tone: 'neutral' }; } /** Load only the requested history, respecting the API's per-request page bound. */ -export async function fetchCampaigns(filters: Record, count: number, signal?: AbortSignal): Promise<{ deployments: T[]; total: number }> { +export async function fetchCampaigns( + filters: Record, + count: number, + signal?: AbortSignal +): Promise<{ deployments: T[]; total: number }> { const deployments: T[] = []; let total = Infinity; while (deployments.length < Math.min(count, total)) { - const params = new URLSearchParams({ ...filters, limit: String(Math.min(500, count - deployments.length)), offset: String(deployments.length) }); + const params = new URLSearchParams({ + ...filters, + limit: String(Math.min(500, count - deployments.length)), + offset: String(deployments.length) + }); const response = await fetch(`/api/campaigns?${params}`, { signal }); if (!response.ok) throw new Error('Could not refresh campaigns.'); - const page = await response.json() as { deployments: T[]; total: number }; + const page = (await response.json()) as { deployments: T[]; total: number }; total = page.total; deployments.push(...page.deployments); if (!page.deployments.length) break; diff --git a/src/lib/ui/collection-launch.ts b/src/lib/ui/collection-launch.ts index 426223c..caa2d5c 100644 --- a/src/lib/ui/collection-launch.ts +++ b/src/lib/ui/collection-launch.ts @@ -17,35 +17,49 @@ export interface CollectionLaunchState { function changesSince(previous: CollectionPreview | null, next: CollectionPreview): string { if (!previous) return ''; - const sorted = (values: unknown[]) => JSON.stringify(values.map(value => JSON.stringify(value)).sort()); - const removals = (preview: CollectionPreview) => sorted(preview.removals.map(server => [server.name, server.reason])); - const creations = (preview: CollectionPreview) => sorted(preview.servers.map(server => { - const recipe = preview.entries[server.entryIndex]?.recipe; - return [server.name, recipe?.game, recipe?.mapSlug]; - })); + const sorted = (values: unknown[]) => + JSON.stringify(values.map((value) => JSON.stringify(value)).sort()); + const removals = (preview: CollectionPreview) => + sorted(preview.removals.map((server) => [server.name, server.reason])); + const creations = (preview: CollectionPreview) => + sorted( + preview.servers.map((server) => { + const recipe = preview.entries[server.entryIndex]?.recipe; + return [server.name, recipe?.game, recipe?.mapSlug]; + }) + ); const retained = (preview: CollectionPreview) => sorted([...new Set(preview.retainedServers)]); if (removals(previous) !== removals(next)) { return 'Servers to remove changed. Review the updated removal list before applying this collection.'; } const createsChanged = creations(previous) !== creations(next); const keepsChanged = retained(previous) !== retained(next); - if (createsChanged && keepsChanged) return 'Servers to create and keep changed. Review the updated fleet changes before applying.'; - if (createsChanged) return 'Servers to create changed. Review the updated launch list before applying.'; - if (keepsChanged) return 'Servers to keep changed. Review the updated fleet changes before applying.'; + if (createsChanged && keepsChanged) + return 'Servers to create and keep changed. Review the updated fleet changes before applying.'; + if (createsChanged) + return 'Servers to create changed. Review the updated launch list before applying.'; + if (keepsChanged) + return 'Servers to keep changed. Review the updated fleet changes before applying.'; return ''; } /** Refresh collection plans without retaining a launchable preview while its inputs are changing. */ export function createCollectionLaunchController( onChange: (state: CollectionLaunchState) => void, - dependencies: { fetch?: typeof fetch; debounceMs?: number } = {}, + dependencies: { fetch?: typeof fetch; debounceMs?: number } = {} ) { - const fetchPreview = dependencies.fetch ?? ((...args: Parameters) => fetch(...args)); + const fetchPreview = + dependencies.fetch ?? ((...args: Parameters) => fetch(...args)); const debounceMs = dependencies.debounceMs ?? 200; let selection: string | null = null; let context: string | null = null; let previous: CollectionPreview | null = null; - let state: CollectionLaunchState = { preview: null, checking: false, error: '', changeNotice: '' }; + let state: CollectionLaunchState = { + preview: null, + checking: false, + error: '', + changeNotice: '' + }; let revision = 0; let timer: ReturnType | undefined; let request: AbortController | undefined; @@ -53,8 +67,13 @@ export function createCollectionLaunchController( let disposed = false; function publish(next: CollectionLaunchState) { - if (state.preview === next.preview && state.checking === next.checking - && state.error === next.error && state.changeNotice === next.changeNotice) return; + if ( + state.preview === next.preview && + state.checking === next.checking && + state.error === next.error && + state.changeNotice === next.changeNotice + ) + return; state = next; onChange(next); } @@ -74,14 +93,19 @@ export function createCollectionLaunchController( request = controller; try { const response = await fetchPreview('/api/deployment-collections/preview', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: selection, signal: controller.signal, + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: selection, + signal: controller.signal }); const body = await response.json(); if (disposed || suspended || current !== revision) return; - if (!response.ok) throw new Error(body.error ?? 'Could not check collection readiness. Try again.'); + if (!response.ok) + throw new Error(body.error ?? 'Could not check collection readiness. Try again.'); const preview = body as CollectionPreview; const expiresAt = Date.parse(preview.expiresAt); - if (!Number.isFinite(expiresAt) || expiresAt <= Date.now()) throw new Error('The collection check expired. Try again.'); + if (!Number.isFinite(expiresAt) || expiresAt <= Date.now()) + throw new Error('The collection check expired. Try again.'); const changeNotice = changesSince(previous, preview); previous = preview; publish({ preview, checking: false, error: '', changeNotice }); @@ -90,8 +114,15 @@ export function createCollectionLaunchController( } } catch (error) { if (!disposed && !suspended && current === revision) { - publish({ preview: null, checking: false, changeNotice: '', - error: error instanceof Error ? error.message : 'Could not check collection readiness. Try again.' }); + publish({ + preview: null, + checking: false, + changeNotice: '', + error: + error instanceof Error + ? error.message + : 'Could not check collection readiness. Try again.' + }); } } finally { if (current === revision) request = undefined; @@ -117,10 +148,17 @@ export function createCollectionLaunchController( /** A null selection pauses checks during host reloads while preserving the last plan for comparison. */ select(input: CollectionLaunchInput | null) { if (disposed) return; - if (!input) { invalidate(); return; } + if (!input) { + invalidate(); + return; + } const nextContext = JSON.stringify([input.blueprintId, input.mode]); - const next = JSON.stringify({ blueprintId: input.blueprintId, mode: input.mode, - libraryIds: { t5: input.libraryIds.t5, t6: input.libraryIds.t6 }, image: input.image }); + const next = JSON.stringify({ + blueprintId: input.blueprintId, + mode: input.mode, + libraryIds: { t5: input.libraryIds.t5, t6: input.libraryIds.t6 }, + image: input.image + }); if (selection === next) return; cancel(); if (context !== nextContext) previous = null; @@ -152,6 +190,6 @@ export function createCollectionLaunchController( previous = null; context = null; disposed = true; - }, + } }; } diff --git a/src/lib/ui/collection-summary.ts b/src/lib/ui/collection-summary.ts index 49ddfa8..b916fb1 100644 --- a/src/lib/ui/collection-summary.ts +++ b/src/lib/ui/collection-summary.ts @@ -33,7 +33,13 @@ export function collectionKeyCoverage(preview: CollectionPreview): CollectionKey const coverage = new Map(); for (const server of preview.servers) { const game = preview.entries[server.entryIndex].recipe.game; - const pool = coverage.get(game) ?? { game, label: gameLabels[game], required: 0, available: 0, unassigned: 0 }; + const pool = coverage.get(game) ?? { + game, + label: gameLabels[game], + required: 0, + available: 0, + unassigned: 0 + }; pool.required++; if (!server.name) pool.unassigned++; else if (server.hasSavedKey) pool.available++; @@ -49,10 +55,19 @@ function removalMap(name: string): { game: ZombieGame | null; key: string; mapNa if (parsed && supportedGame && (!parsed.gameMode || parsed.gameMode === 'zombie')) { // Resolve friendly legacy aliases without merging T6 variants that share an engine map. const map = selectedZombieMap(supportedGame, parsed.mapCanonical, parsed.mapSlug); - if (map) return { game: supportedGame, key: `${supportedGame}:${map.slug}`, mapName: map.friendly }; + if (map) + return { + game: supportedGame, + key: `${supportedGame}:${map.slug}`, + mapName: map.friendly + }; } // A raw identity is preferable to assigning an unknown server to the wrong map. - return { game: supportedGame, key: `${supportedGame ?? 'other'}:server:${name}`, mapName: name }; + return { + game: supportedGame, + key: `${supportedGame ?? 'other'}:server:${name}`, + mapName: name + }; } /** Summarize affected inventory, including exact-mode removals outside the saved collection. */ @@ -62,7 +77,11 @@ export function collectionMapGroups(preview: CollectionPreview): CollectionMapGr const existingNames = new Map>(); const serverRows = new Map(); const addRow = (game: ZombieGame | null, row: CollectionMapRow) => { - const group = groups.get(game) ?? { game, label: game ? gameLabels[game] : 'Other servers', rows: [] }; + const group = groups.get(game) ?? { + game, + label: game ? gameLabels[game] : 'Other servers', + rows: [] + }; group.rows.push(row); groups.set(game, group); rows.set(row.key, row); @@ -72,8 +91,14 @@ export function collectionMapGroups(preview: CollectionPreview): CollectionMapGr for (const entry of preview.entries) { const { game, mapSlug, mapName } = entry.recipe; - const row = addRow(game, { key: `${game}:${mapSlug}`, mapName, target: entry.count, - existing: entry.existing, create: entry.missing, remove: 0 }); + const row = addRow(game, { + key: `${game}:${mapSlug}`, + mapName, + target: entry.count, + existing: entry.existing, + create: entry.missing, + remove: 0 + }); for (const name of entry.existingServers) { existingNames.get(row.key)!.add(name); // Backend coverage knows stored map selections even when a name is nonstandard. @@ -84,14 +109,25 @@ export function collectionMapGroups(preview: CollectionPreview): CollectionMapGr if (preview.mode === 'exact') { for (const { name } of preview.removals) { const identity = removalMap(name); - const row = serverRows.get(name) ?? rows.get(identity.key) ?? addRow(identity.game, { - key: identity.key, mapName: identity.mapName, target: 0, existing: 0, create: 0, remove: 0, - }); + const row = + serverRows.get(name) ?? + rows.get(identity.key) ?? + addRow(identity.game, { + key: identity.key, + mapName: identity.mapName, + target: 0, + existing: 0, + create: 0, + remove: 0 + }); row.remove++; const counted = existingNames.get(row.key)!; // Coverage excludes pending removals. Include those identities in displayed inventory // while counting ordinary excess servers only once. - if (!counted.has(name)) { row.existing++; counted.add(name); } + if (!counted.has(name)) { + row.existing++; + counted.add(name); + } } } @@ -101,9 +137,12 @@ export function collectionMapGroups(preview: CollectionPreview): CollectionMapGr /** Capacity remains an estimate: insufficient capacity can wait in the startup queue. */ export function collectionCapacitySummary(availability?: LaunchAvailability): string { if (availability?.requested === 0) return 'No new server capacity needed.'; - if (!availability || availability.estimatedAvailable === null) return 'Capacity estimate unavailable; launches can wait for capacity.'; + if (!availability || availability.estimatedAvailable === null) + return 'Capacity estimate unavailable; launches can wait for capacity.'; if (availability.estimatedAvailable >= availability.requested) { - return availability.requested === 1 ? 'Estimated capacity for 1 server.' : `Estimated capacity for all ${availability.requested} servers.`; + return availability.requested === 1 + ? 'Estimated capacity for 1 server.' + : `Estimated capacity for all ${availability.requested} servers.`; } return `Estimated capacity for ${availability.estimatedAvailable} of ${availability.requested} server${availability.requested === 1 ? '' : 's'}; launches can wait for capacity.`; } diff --git a/src/lib/ui/command-palette.ts b/src/lib/ui/command-palette.ts index 289c729..7763dc8 100644 --- a/src/lib/ui/command-palette.ts +++ b/src/lib/ui/command-palette.ts @@ -4,35 +4,127 @@ import { playerProfileHref } from '../leaderboard.js'; import { serverTitle } from './server-labels.js'; export type PaletteGroup = 'Pages' | 'Actions' | 'Servers' | 'Hosts' | 'Players' | 'Matches'; -export interface PaletteItem { key: string; group: PaletteGroup; label: string; detail: string; href: string } -export interface PaletteServer { name: string; map?: string | null; last_map?: string | null; host?: { name: string } | null } -export interface PaletteHost { id: string; name: string; hostname?: string; connectionState?: string } -export interface PaletteDirectory { servers: PaletteServer[]; hosts: PaletteHost[]; unavailable: string[] } +export interface PaletteItem { + key: string; + group: PaletteGroup; + label: string; + detail: string; + href: string; +} +export interface PaletteServer { + name: string; + map?: string | null; + last_map?: string | null; + host?: { name: string } | null; +} +export interface PaletteHost { + id: string; + name: string; + hostname?: string; + connectionState?: string; +} +export interface PaletteDirectory { + servers: PaletteServer[]; + hosts: PaletteHost[]; + unavailable: string[]; +} type PaletteFetch = (input: string, init: RequestInit) => Promise; -export const paletteGroups: PaletteGroup[] = ['Actions', 'Pages', 'Servers', 'Hosts', 'Players', 'Matches']; +export const paletteGroups: PaletteGroup[] = [ + 'Actions', + 'Pages', + 'Servers', + 'Hosts', + 'Players', + 'Matches' +]; function matches(query: string, ...values: Array) { const text = values.filter(Boolean).join(' ').toLocaleLowerCase(); - return query.trim().toLocaleLowerCase().split(/\s+/).every(word => text.includes(word)); + return query + .trim() + .toLocaleLowerCase() + .split(/\s+/) + .every((word) => text.includes(word)); } export function palettePageResults(role: UserRole, query: string): PaletteItem[] { - const actions: PaletteItem[] = role === 'viewer' ? [] : [{ key: 'action:create', group: 'Actions', label: 'Create server', detail: 'Choose a map and host', href: '/?action=create' }, { key: 'action:manage', group: 'Actions', label: 'Manage servers', detail: 'Server queues and lifecycle actions', href: '/provisioning' }]; - if (role === 'admin') actions.push({ key: 'action:add-host', group: 'Actions', label: 'Add host', detail: 'Enroll a host agent', href: '/hosts?action=add' }); - return [...actions.filter(item => matches(query, item.label, item.detail)), - ...visibleNavigation(role).flatMap(group => group.items.filter(item => matches(query, item.label, group.label)).map(item => ({ - key: `page:${item.href}`, group: 'Pages' as const, label: item.label, detail: group.label, href: item.href - })))]; + const actions: PaletteItem[] = + role === 'viewer' + ? [] + : [ + { + key: 'action:create', + group: 'Actions', + label: 'Create server', + detail: 'Choose a map and host', + href: '/?action=create' + }, + { + key: 'action:manage', + group: 'Actions', + label: 'Manage servers', + detail: 'Server queues and lifecycle actions', + href: '/provisioning' + } + ]; + if (role === 'admin') + actions.push({ + key: 'action:add-host', + group: 'Actions', + label: 'Add host', + detail: 'Enroll a host agent', + href: '/hosts?action=add' + }); + return [ + ...actions.filter((item) => matches(query, item.label, item.detail)), + ...visibleNavigation(role).flatMap((group) => + group.items + .filter((item) => matches(query, item.label, group.label)) + .map((item) => ({ + key: `page:${item.href}`, + group: 'Pages' as const, + label: item.label, + detail: group.label, + href: item.href + })) + ) + ]; } -export function paletteDirectoryResults(query: string, directory: PaletteDirectory | null): PaletteItem[] { +export function paletteDirectoryResults( + query: string, + directory: PaletteDirectory | null +): PaletteItem[] { if (!query.trim() || !directory) return []; - const servers: PaletteItem[] = directory.servers.filter(server => matches(query, serverTitle(server.name), server.name, server.map, server.last_map, server.host?.name)) - .slice(0, 6).map(server => ({ key: `server:${server.name}`, group: 'Servers', label: serverTitle(server.name), - detail: server.host?.name ?? 'Game server', href: `/servers/${encodeURIComponent(server.name)}` })); - const hosts: PaletteItem[] = directory.hosts.filter(host => matches(query, host.name, host.hostname, host.id)) - .slice(0, 6).map(host => ({ key: `host:${host.id}`, group: 'Hosts', label: host.name, - detail: host.connectionState === 'offline' ? 'Host agent offline' : 'Host', href: `/hosts/${encodeURIComponent(host.id)}` })); + const servers: PaletteItem[] = directory.servers + .filter((server) => + matches( + query, + serverTitle(server.name), + server.name, + server.map, + server.last_map, + server.host?.name + ) + ) + .slice(0, 6) + .map((server) => ({ + key: `server:${server.name}`, + group: 'Servers', + label: serverTitle(server.name), + detail: server.host?.name ?? 'Game server', + href: `/servers/${encodeURIComponent(server.name)}` + })); + const hosts: PaletteItem[] = directory.hosts + .filter((host) => matches(query, host.name, host.hostname, host.id)) + .slice(0, 6) + .map((host) => ({ + key: `host:${host.id}`, + group: 'Hosts', + label: host.name, + detail: host.connectionState === 'offline' ? 'Host agent offline' : 'Host', + href: `/hosts/${encodeURIComponent(host.id)}` + })); return [...servers, ...hosts]; } @@ -40,49 +132,107 @@ export function paletteDirectoryResults(query: string, directory: PaletteDirecto export function paletteMatchJump(query: string): PaletteItem[] { const id = /^(?:match\s*#?\s*|#)?([1-9]\d*)$/i.exec(query.trim())?.[1]; if (!id || !Number.isSafeInteger(Number(id))) return []; - return [{ key: `match:${id}`, group: 'Matches', label: `Open match #${id}`, detail: 'Go to a recorded match', href: `/matches/${id}` }]; + return [ + { + key: `match:${id}`, + group: 'Matches', + label: `Open match #${id}`, + detail: 'Go to a recorded match', + href: `/matches/${id}` + } + ]; } async function read(url: string, signal: AbortSignal, fetcher: PaletteFetch) { const response = await fetcher(url, { signal, cache: 'no-store' }); if (!response.ok) throw new Error('Search request failed.'); const body: unknown = await response.json(); - if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Search response is unavailable.'); + if (!body || typeof body !== 'object' || Array.isArray(body)) + throw new Error('Search response is unavailable.'); return body as Record; } /** Load only the current directory; historical entities use bounded queries below. */ -export async function loadPaletteDirectory(signal: AbortSignal, fetcher: PaletteFetch = fetch): Promise { +export async function loadPaletteDirectory( + signal: AbortSignal, + fetcher: PaletteFetch = fetch +): Promise { const [servers, hosts] = await Promise.allSettled([ - read('/api/servers', signal, fetcher), read('/api/hosts', signal, fetcher) + read('/api/servers', signal, fetcher), + read('/api/hosts', signal, fetcher) ]); return { - servers: servers.status === 'fulfilled' && Array.isArray(servers.value.servers) ? servers.value.servers.filter(server => server && typeof server.name === 'string') : [], - hosts: hosts.status === 'fulfilled' && Array.isArray(hosts.value.hosts) ? hosts.value.hosts.filter(host => host && typeof host.id === 'string' && typeof host.name === 'string') : [], - unavailable: [servers.status === 'rejected' ? 'Server' : '', hosts.status === 'rejected' ? 'Host' : ''].filter(Boolean) + servers: + servers.status === 'fulfilled' && Array.isArray(servers.value.servers) + ? servers.value.servers.filter( + (server) => server && typeof server.name === 'string' + ) + : [], + hosts: + hosts.status === 'fulfilled' && Array.isArray(hosts.value.hosts) + ? hosts.value.hosts.filter( + (host) => host && typeof host.id === 'string' && typeof host.name === 'string' + ) + : [], + unavailable: [ + servers.status === 'rejected' ? 'Server' : '', + hosts.status === 'rejected' ? 'Host' : '' + ].filter(Boolean) }; } -export async function searchPaletteRecords(query: string, signal: AbortSignal, fetcher: PaletteFetch = fetch): Promise<{ items: PaletteItem[]; unavailable: string[] }> { +export async function searchPaletteRecords( + query: string, + signal: AbortSignal, + fetcher: PaletteFetch = fetch +): Promise<{ items: PaletteItem[]; unavailable: string[] }> { const term = query.trim().slice(0, 100); if (term.length < 2) return { items: [], unavailable: [] }; const playersParams = new URLSearchParams({ q: term, limit: '5', metric: 'highestRound' }); const matchesParams = new URLSearchParams({ q: term, limit: '5', status: 'completed' }); const [players, recordedMatches] = await Promise.allSettled([ - read(`/api/leaderboard?${playersParams}`, signal, fetcher), read(`/api/matches?${matchesParams}`, signal, fetcher) + read(`/api/leaderboard?${playersParams}`, signal, fetcher), + read(`/api/matches?${matchesParams}`, signal, fetcher) ]); const items: PaletteItem[] = []; if (players.status === 'fulfilled' && Array.isArray(players.value.entries)) { for (const player of players.value.entries.slice(0, 5)) { - if (!player || typeof player.guid !== 'string' || typeof player.name !== 'string') continue; - items.push({ key: `player:${player.guid}`, group: 'Players', label: player.name, detail: 'Player profile', href: playerProfileHref(player.guid) }); + if (!player || typeof player.guid !== 'string' || typeof player.name !== 'string') + continue; + items.push({ + key: `player:${player.guid}`, + group: 'Players', + label: player.name, + detail: 'Player profile', + href: playerProfileHref(player.guid) + }); } } if (recordedMatches.status === 'fulfilled' && Array.isArray(recordedMatches.value.matches)) { for (const match of recordedMatches.value.matches.slice(0, 5)) { - if (!match || !Number.isSafeInteger(match.id) || match.id < 1 || match.status !== 'completed' || typeof match.map !== 'string' || typeof match.server !== 'string') continue; - items.push({ key: `match:${match.id}`, group: 'Matches', label: `${match.map} · Match #${match.id}`, detail: serverTitle(match.server), href: `/matches/${match.id}` }); + if ( + !match || + !Number.isSafeInteger(match.id) || + match.id < 1 || + match.status !== 'completed' || + typeof match.map !== 'string' || + typeof match.server !== 'string' + ) + continue; + items.push({ + key: `match:${match.id}`, + group: 'Matches', + label: `${match.map} · Match #${match.id}`, + detail: serverTitle(match.server), + href: `/matches/${match.id}` + }); } } - return { items, unavailable: [players.status === 'rejected' ? 'Player' : '', recordedMatches.status === 'rejected' ? 'Match' : ''].filter(Boolean) }; + return { + items, + unavailable: [ + players.status === 'rejected' ? 'Player' : '', + recordedMatches.status === 'rejected' ? 'Match' : '' + ].filter(Boolean) + }; } diff --git a/src/lib/ui/components/ActivityLog.svelte b/src/lib/ui/components/ActivityLog.svelte index c6e5676..1abd2fb 100644 --- a/src/lib/ui/components/ActivityLog.svelte +++ b/src/lib/ui/components/ActivityLog.svelte @@ -2,7 +2,14 @@ import { untrack } from 'svelte'; import { page } from '$app/stores'; import { goto } from '$app/navigation'; - import { activityTypes, activitySources, activityStates, type ActivityPage, type ActivityQuery, type ActivitySource } from '$lib/activity.js'; + import { + activityTypes, + activitySources, + activityStates, + type ActivityPage, + type ActivityQuery, + type ActivitySource + } from '$lib/activity.js'; import { contextHref } from '$lib/navigation-context.js'; import { activityApiHref, activityHistoryHref, activityQuery } from '$ui/activity-history.js'; import { pollVisible } from '$ui/poll.js'; @@ -17,7 +24,19 @@ import Freshness from './Freshness.svelte'; import HistoricalServerIdentity from './HistoricalServerIdentity.svelte'; - let { server, source, initial, excludeProvisioning = false, excludeMaintenance = false }: { server?: string; source?: ActivitySource; initial?: ActivityPage; excludeProvisioning?: boolean; excludeMaintenance?: boolean } = $props(); + let { + server, + source, + initial, + excludeProvisioning = false, + excludeMaintenance = false + }: { + server?: string; + source?: ActivitySource; + initial?: ActivityPage; + excludeProvisioning?: boolean; + excludeMaintenance?: boolean; + } = $props(); const id = $props.id(); const headingId = `${id}-activity-heading`; let data = $state(untrack(() => initial)); @@ -25,169 +44,588 @@ let error = $state(''); let retry = $state(0); let filtersOpen = $state(false); - let checkedAt = $state(untrack(() => initial ? Date.now() : null)); + let checkedAt = $state(untrack(() => (initial ? Date.now() : null))); let results = $state(); - let search = $state(untrack(() => activityQuery($page.url, { server, source, excludeProvisioning, excludeMaintenance }).q ?? '')); + let search = $state( + untrack( + () => + activityQuery($page.url, { + server, + source, + excludeProvisioning, + excludeMaintenance + }).q ?? '' + ) + ); let pendingUrl: URL | undefined; let navigationId = 0; - const filters = $derived(activityQuery($page.url, { server, source, excludeProvisioning, excludeMaintenance })); - const apiUrl = $derived(activityApiHref($page.url, { server, source, excludeProvisioning, excludeMaintenance })); - let loadedUrl = $state(untrack(() => initial ? apiUrl : '')); - const typeOptions = $derived(Object.entries(activityTypes).filter(([key]) => - (!excludeProvisioning || !['create', 'start', 'restart', 'rebuild', 'stop', 'remove'].includes(key)) - && (!excludeMaintenance || !['update', 'health'].includes(key)))); - const sourceOptions = $derived(Object.entries(activitySources).filter(([key]) => - (!excludeProvisioning || !['provisioning', 'autoscaling'].includes(key)) - && (!excludeMaintenance || !['maintenance', 'recovery'].includes(key)))); - const historyDescription = $derived(excludeMaintenance - ? 'File changes and host work appear here.' - : excludeProvisioning ? 'File changes, maintenance, and host work appear here.' - : 'Server operations, file changes, maintenance, and host work appear here.'); - const filterCount = $derived([filters.type, filters.state, source === undefined ? filters.source : '', server === undefined ? filters.server : '', filters.host, filters.days, filters.q].filter(Boolean).length); + const filters = $derived( + activityQuery($page.url, { server, source, excludeProvisioning, excludeMaintenance }) + ); + const apiUrl = $derived( + activityApiHref($page.url, { server, source, excludeProvisioning, excludeMaintenance }) + ); + let loadedUrl = $state(untrack(() => (initial ? apiUrl : ''))); + const typeOptions = $derived( + Object.entries(activityTypes).filter( + ([key]) => + (!excludeProvisioning || + !['create', 'start', 'restart', 'rebuild', 'stop', 'remove'].includes(key)) && + (!excludeMaintenance || !['update', 'health'].includes(key)) + ) + ); + const sourceOptions = $derived( + Object.entries(activitySources).filter( + ([key]) => + (!excludeProvisioning || !['provisioning', 'autoscaling'].includes(key)) && + (!excludeMaintenance || !['maintenance', 'recovery'].includes(key)) + ) + ); + const historyDescription = $derived( + excludeMaintenance + ? 'File changes and host work appear here.' + : excludeProvisioning + ? 'File changes, maintenance, and host work appear here.' + : 'Server operations, file changes, maintenance, and host work appear here.' + ); + const filterCount = $derived( + [ + filters.type, + filters.state, + source === undefined ? filters.source : '', + server === undefined ? filters.server : '', + filters.host, + filters.days, + filters.q + ].filter(Boolean).length + ); const serverOptions = $derived.by(() => { - const names = new Set([...$fleet.servers.map(item => item.name), ...(data?.rows.flatMap(row => row.server ? [row.server] : []) ?? [])]); + const names = new Set([ + ...$fleet.servers.map((item) => item.name), + ...(data?.rows.flatMap((row) => (row.server ? [row.server] : [])) ?? []) + ]); if (filters.server) names.add(filters.server); return [...names].sort((a, b) => serverTitle(a).localeCompare(serverTitle(b))); }); const hostOptions = $derived.by(() => { const hosts = new Map(); for (const item of $fleet.servers) if (item.host) hosts.set(item.host.id, item.host.name); - for (const row of data?.rows ?? []) if (row.hostId) hosts.set(row.hostId, row.hostName ?? 'Historical host'); + for (const row of data?.rows ?? []) + if (row.hostId) hosts.set(row.hostId, row.hostName ?? 'Historical host'); if (filters.host && !hosts.has(filters.host)) hosts.set(filters.host, 'Selected host'); return [...hosts].sort((a, b) => a[1].localeCompare(b[1])); }); - $effect(() => { search = filters.q ?? ''; }); + $effect(() => { + search = filters.q ?? ''; + }); $effect(() => { const report = initial; - if (report) untrack(() => { data = report; loadedUrl = apiUrl; loading = false; error = ''; checkedAt = Date.now(); }); + if (report) + untrack(() => { + data = report; + loadedUrl = apiUrl; + loading = false; + error = ''; + checkedAt = Date.now(); + }); }); $effect(() => { const url = apiUrl; - retry; - untrack(() => { if (loadedUrl !== url) loading = true; error = ''; }); - return pollVisible(async signal => { - try { - const response = await fetch(url, { signal, cache: 'no-store' }); - if (!response.ok) throw new Error('Could not load activity.'); - const body: ActivityPage = await response.json(); - if (signal.aborted || apiUrl !== url) return; - // Keep a focused operation link in place when newer rows arrive. - if (loadedUrl !== url || !results?.contains(document.activeElement)) { data = body; checkedAt = Date.now(); } - loadedUrl = url; loading = false; error = ''; - } catch (cause) { - if (signal.aborted || apiUrl !== url) return; - error = cause instanceof Error ? cause.message : 'Could not load activity.'; - loading = false; - return false; - } - }, 10_000, untrack(() => loadedUrl !== url || retry > 0)); + void retry; + untrack(() => { + if (loadedUrl !== url) loading = true; + error = ''; + }); + return pollVisible( + async (signal) => { + try { + const response = await fetch(url, { signal, cache: 'no-store' }); + if (!response.ok) throw new Error('Could not load activity.'); + const body: ActivityPage = await response.json(); + if (signal.aborted || apiUrl !== url) return; + // Keep a focused operation link in place when newer rows arrive. + if (loadedUrl !== url || !results?.contains(document.activeElement)) { + data = body; + checkedAt = Date.now(); + } + loadedUrl = url; + loading = false; + error = ''; + } catch (cause) { + if (signal.aborted || apiUrl !== url) return; + error = cause instanceof Error ? cause.message : 'Could not load activity.'; + loading = false; + return false; + } + }, + 10_000, + untrack(() => loadedUrl !== url || retry > 0) + ); }); function update(changes: Partial>) { const href = activityHistoryHref(pendingUrl ?? $page.url, changes); pendingUrl = new URL(href, $page.url); const id = ++navigationId; void goto(href, { noScroll: true, keepFocus: true }) - .catch(() => { error = 'Could not update activity filters. Please try again.'; }) - .finally(() => { if (id === navigationId) pendingUrl = undefined; }); + .catch(() => { + error = 'Could not update activity filters. Please try again.'; + }) + .finally(() => { + if (id === navigationId) pendingUrl = undefined; + }); + } + function reset() { + update({ type: '', state: '', source: '', server: '', host: '', days: '', q: '' }); + } + function submitSearch() { + update({ q: search.trim() }); } - function reset() { update({ type: '', state: '', source: '', server: '', host: '', days: '', q: '' }); } - function submitSearch() { update({ q: search.trim() }); }

Activity history

{#snippet filterFields(suffix: string)}
- - - {#if source === undefined}{/if} - {#if server === undefined}{/if} - - - + + + {#if source === undefined}{/if} + {#if server === undefined}{/if} + + +
{/snippet}
{@render filterFields('')}
-
{data?.total ?? '—'} {data?.total === 1 ? 'activity' : 'activities'}{source ? ` · ${activitySources[source]}` : ''}
- +
+ {data?.total ?? '—'} + {data?.total === 1 ? 'activity' : 'activities'}{source + ? ` · ${activitySources[source]}` + : ''} +
+ {@render filterFields('-mobile')} - {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet} -
Newest first · Times in {$timeZone}{#if filterCount}{/if}{#if !error}{/if}
- {#if error}{/if} +
+ Newest first · Times in {$timeZone}{#if filterCount}{/if}{#if !error}{/if} +
+ {#if error}{/if} {#if loading} - {:else if loadedUrl !== apiUrl && error} + {:else if loadedUrl !== apiUrl && error} {:else if data?.rows.length}
    {#each data.rows as row (row.id)}
  1. -

    {row.title}

    {#if row.detail}

    {row.detail}

    {/if} -
    {#if row.server && server === undefined}{/if} +
    +

    + {row.title} +

    + {#if row.detail}

    {row.detail}

    {/if} +
    + {#if row.server && server === undefined}{/if} {#if row.hostName}{row.hostName}{/if}
    -
    - {(row.sources.length ? row.sources : [row.source]).map(item => activitySources[item]).join(' · ')} +
    + + {(row.sources.length ? row.sources : [row.source]) + .map((item) => activitySources[item]) + .join(' · ')}
    2. {/each}
- {:else if !error} - {#snippet actions()}{#if filterCount}{/if}{/snippet} - {/if} + {:else if !error} + {#snippet actions()}{#if filterCount}{/if}{/snippet} + {/if} {#if data && !loading && loadedUrl === apiUrl && (data.total > 0 || data.offset > 0)} -
diff --git a/src/lib/ui/components/AddHostDrawer.svelte b/src/lib/ui/components/AddHostDrawer.svelte index caa39a1..2323826 100644 --- a/src/lib/ui/components/AddHostDrawer.svelte +++ b/src/lib/ui/components/AddHostDrawer.svelte @@ -2,25 +2,46 @@ import { timeFormat } from '$ui/stores/time-zone.js'; import Dialog from './primitives/Dialog.svelte'; import Button from './primitives/Button.svelte'; - let { open = $bindable(false), onadded }: { open?: boolean; onadded: () => Promise } = $props(); + let { open = $bindable(false), onadded }: { open?: boolean; onadded: () => Promise } = + $props(); let name = $state(''); - let portMin = $state(28960); let portMax = $state(28980); - let busy = $state(false); let error = $state(''); let copied = $state(false); + let portMin = $state(28960); + let portMax = $state(28980); + let busy = $state(false); + let error = $state(''); + let copied = $state(false); let enrollment = $state<{ command: string; expiresAt: string; hostId: string } | null>(null); async function add(event: SubmitEvent) { - event.preventDefault(); if (busy) return; - busy = true; error = ''; copied = false; + event.preventDefault(); + if (busy) return; + busy = true; + error = ''; + copied = false; try { - const response = await fetch('/api/hosts', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name, portMin, portMax }) }); - const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not add host.'); + const response = await fetch('/api/hosts', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name, portMin, portMax }) + }); + const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not add host.'); enrollment = body.enrollment; await onadded(); - } catch (cause) { error = (cause as Error).message; } finally { busy = false; } + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } } async function copy() { if (!enrollment) return; - try { await navigator.clipboard.writeText(enrollment.command); copied = true; error = ''; } - catch { error = 'Could not copy automatically. Select and copy the command below.'; } + try { + await navigator.clipboard.writeText(enrollment.command); + copied = true; + error = ''; + } catch { + error = 'Could not copy automatically. Select and copy the command below.'; + } } @@ -28,35 +49,142 @@
{#if error}{/if} {#if enrollment} -

{name} created. Run this command with administrator access on that machine.

- -
-

This single-use command expires {$timeFormat.dateTime(enrollment.expiresAt)}. Generate a new installer from host settings if it expires.

- +

+ {name} created. Run this command with administrator access on that machine. +

+ +
+ +
+

+ This single-use command expires {$timeFormat.dateTime(enrollment.expiresAt)}. + Generate a new installer from host settings if it expires. +

+ {:else} -

Use a Debian 12 or 13 amd64 machine. The agent connects over HTTPS and prepares the runtime and game files automatically.

-
- -
Advanced settings
- - -
- -
+

+ Use a Debian 12 or 13 amd64 machine. The agent connects over HTTPS and prepares the + runtime and game files automatically. +

+
+
+ +
+ Advanced settings +
+ + +
+
+ +
+
{/if}
diff --git a/src/lib/ui/components/ArtifactDetail.svelte b/src/lib/ui/components/ArtifactDetail.svelte index ce1c15d..3d1a768 100644 --- a/src/lib/ui/components/ArtifactDetail.svelte +++ b/src/lib/ui/components/ArtifactDetail.svelte @@ -28,39 +28,66 @@ type Family = 'script' | 'config' | 'plugin'; - interface Props { id: number; family: Family; } + interface Props { + id: number; + family: Family; + } let { id, family }: Props = $props(); const basePath = $derived(`/${family}s`); const crumbLabel = $derived(family.charAt(0).toUpperCase() + family.slice(1) + 's'); interface Artifact { - id: number; name: string; target_rel_path: string; - scope_type: 'game' | 'map' | null; scope_value: string | null; + id: number; + name: string; + target_rel_path: string; + scope_type: 'game' | 'map' | null; + scope_value: string | null; current_version_id: number | null; kind: 'script' | 'config' | 'plugin' | 'baseline'; created_at: string; } interface Version { - id: number; version: number; sha256: string; size: number; + id: number; + version: number; + sha256: string; + size: number; release_version: string | null; source: PluginSource | null; - notes: string | null; state: 'untested' | 'tested' | 'test-failed'; + notes: string | null; + state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string; } - interface ServerRow { name: string; game: string | null; map: string | null; } - interface ScopeOptions { games: string[]; mapsByGame: Record; } + interface ScopeOptions { + games: string[]; + mapsByGame: Record; + } let artifact = $state(null); let versions = $state([]); let matchedServers = $state([]); - let installedVersions = $state>([]); - interface ScopeImpact { before: string[]; after: string[]; added: string[]; removed: string[]; affected: string[]; composed: boolean; detached: string[]; } + let installedVersions = $state< + Array<{ server: string; version: number; versionId: number; recordedAt: string }> + >([]); + interface ScopeImpact { + before: string[]; + after: string[]; + added: string[]; + removed: string[]; + affected: string[]; + composed: boolean; + detached: string[]; + } let scopeReviewOpen = $state(false); let scopeImpact = $state(null); let scopePreviewError = $state(''); let scopePreviewLoading = $state(false); - let scopeComplete = $state<{ count: number; composed: boolean; detached: number; href: string } | null>(null); + let scopeComplete = $state<{ + count: number; + composed: boolean; + detached: number; + href: string; + } | null>(null); let runningBetaId = $state(null); let loading = $state(true); let flash = $state(null); @@ -91,32 +118,51 @@ const currentVersion = $derived( artifact?.current_version_id - ? versions.find((v) => v.id === artifact!.current_version_id) ?? null + ? (versions.find((v) => v.id === artifact!.current_version_id) ?? null) : null ); const latestVersion = $derived(versions[0] ?? null); $effect(() => { - const type = scopeType; const value = editedScopeValue; const artifactId = id; + const type = scopeType; + const value = editedScopeValue; + const artifactId = id; if (!scopeReviewOpen) return; - scopeImpact = null; scopePreviewError = ''; - if (type !== 'none' && !value) { scopePreviewLoading = false; return; } - const controller = new AbortController(); scopePreviewLoading = true; + scopeImpact = null; + scopePreviewError = ''; + if (type !== 'none' && !value) { + scopePreviewLoading = false; + return; + } + const controller = new AbortController(); + scopePreviewLoading = true; void (async () => { try { - const response = await fetch(`/api/scripts/${artifactId}/scope?${new URLSearchParams({ type, value })}`, { signal: controller.signal }); + const response = await fetch( + `/api/scripts/${artifactId}/scope?${new URLSearchParams({ type, value })}`, + { signal: controller.signal } + ); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not preview scope.'); if (!controller.signal.aborted) scopeImpact = body; - } catch (error) { if (!controller.signal.aborted) scopePreviewError = (error as Error).message; } - finally { if (!controller.signal.aborted) scopePreviewLoading = false; } + } catch (error) { + if (!controller.signal.aborted) scopePreviewError = (error as Error).message; + } finally { + if (!controller.signal.aborted) scopePreviewLoading = false; + } })(); return () => controller.abort(); }); function scopeLabel(s: Artifact): string { if (!s.scope_type) return 'unscoped'; - return s.scope_type === 'map' ? mapLabel(s.scope_value ?? '') : s.scope_value === 't5' ? 'Black Ops I' : s.scope_value === 't6' ? 'Black Ops II' : s.scope_value ?? 'Unscoped'; + return s.scope_type === 'map' + ? mapLabel(s.scope_value ?? '') + : s.scope_value === 't5' + ? 'Black Ops I' + : s.scope_value === 't6' + ? 'Black Ops II' + : (s.scope_value ?? 'Unscoped'); } async function load({ seedForm = true }: { seedForm?: boolean } = {}): Promise { @@ -127,16 +173,28 @@ fetch('/api/scripts/' + requestedId), seedForm ? fetch('/api/scope-options') : Promise.resolve(null) ]); - if (!detailRes.ok) throw new Error(detailRes.status === 404 ? 'This file no longer exists.' : 'Could not refresh file details.'); + if (!detailRes.ok) + throw new Error( + detailRes.status === 404 + ? 'This file no longer exists.' + : 'Could not refresh file details.' + ); const body = await detailRes.json(); const scopes = scopeRes?.ok ? await scopeRes.json() : null; if (requestedId !== id || current !== request) return; - artifact = body.script; versions = body.versions; matchedServers = body.matchedServers; runningBetaId = body.runningBetaId; installedVersions = body.installedVersions ?? []; + artifact = body.script; + versions = body.versions; + matchedServers = body.matchedServers; + runningBetaId = body.runningBetaId; + installedVersions = body.installedVersions ?? []; if (scopes) scopeOptions = scopes; if (seedForm) seedScopeForm(); loadError = ''; - } catch (error) { if (requestedId === id && current === request) loadError = (error as Error).message; } - finally { if (requestedId === id && current === request) loading = false; } + } catch (error) { + if (requestedId === id && current === request) loadError = (error as Error).message; + } finally { + if (requestedId === id && current === request) loading = false; + } } function seedScopeForm(): void { @@ -148,13 +206,25 @@ } else if (scopeType === 'map') { scopeMap = artifact.scope_value ?? ''; const owner = fleetServers.find((sv) => sv.map === scopeMap); - scopeGame = owner?.game ?? Object.keys(scopeOptions.mapsByGame).find(game => scopeOptions.mapsByGame[game].includes(scopeMap)) ?? ''; + scopeGame = + owner?.game ?? + Object.keys(scopeOptions.mapsByGame).find((game) => + scopeOptions.mapsByGame[game].includes(scopeMap) + ) ?? + ''; } else { - scopeGame = ''; scopeMap = ''; + scopeGame = ''; + scopeMap = ''; } } - $effect(() => { id; loading = true; artifact = null; limit = 25; void load(); }); + $effect(() => { + void id; + loading = true; + artifact = null; + limit = 25; + void load(); + }); onMount(() => pollVisible(() => load({ seedForm: false }), 10000, false)); function openScopeModal(): void { @@ -186,7 +256,13 @@ if (!dialog) return; if (e.target === dialog && !uploadingVersion && !updatingScope) { const bounds = dialog.getBoundingClientRect(); - if (e.clientX < bounds.left || e.clientX > bounds.right || e.clientY < bounds.top || e.clientY > bounds.bottom) dialog.close(); + if ( + e.clientX < bounds.left || + e.clientX > bounds.right || + e.clientY < bounds.top || + e.clientY > bounds.bottom + ) + dialog.close(); } } @@ -198,22 +274,30 @@ async function uploadVersion(e: Event): Promise { e.preventDefault(); if (!versionFile || uploadingVersion) return; - uploadingVersion = true; flash = null; + uploadingVersion = true; + flash = null; const form = new FormData(); form.append('content', versionFile); if (family === 'plugin') form.append('releaseVersion', releaseVersion.trim()); if (versionNotes.trim()) form.append('notes', versionNotes.trim()); try { const res = await fetch(`/api/scripts/${id}/versions`, { method: 'POST', body: form }); - const body = (await res.json()) as { ok: boolean; error?: string; betaRequired?: boolean }; + const body = (await res.json()) as { + ok: boolean; + error?: string; + betaRequired?: boolean; + }; if (!res.ok || !body.ok) { flash = body.error ?? `HTTP ${res.status}`; return; } - versionFile = null; versionNotes = ''; + versionFile = null; + versionNotes = ''; uploadDialog?.close(); await load(); - } catch (error) { flash = (error as Error).message; } finally { + } catch (error) { + flash = (error as Error).message; + } finally { uploadingVersion = false; } } @@ -225,7 +309,8 @@ flash = `Pick a ${scopeType}`; return; } - updatingScope = true; flash = null; + updatingScope = true; + flash = null; try { const res = await fetch(`/api/scripts/${id}/scope`, { method: 'POST', @@ -235,27 +320,54 @@ scopeValue: scopeType === 'none' ? null : editedScopeValue }) }); - const body = (await res.json()) as { ok: boolean; error?: string; kind?: string; detached?: string[] }; + const body = (await res.json()) as { + ok: boolean; + error?: string; + kind?: string; + detached?: string[]; + }; if (!res.ok || !body.ok) { flash = body.error ?? `HTTP ${res.status}`; return; } - scopeComplete = { count: scopeImpact.affected.length, composed: scopeImpact.composed, detached: body.detached?.length ?? 0, href: scopeImpact.affected.length === 1 ? '/maintenance?kind=drift&server=' + encodeURIComponent(scopeImpact.affected[0]) : '/maintenance?kind=drift' }; - scopeReviewOpen = false; scopeDialog?.close(); + scopeComplete = { + count: scopeImpact.affected.length, + composed: scopeImpact.composed, + detached: body.detached?.length ?? 0, + href: + scopeImpact.affected.length === 1 + ? '/maintenance?kind=drift&server=' + + encodeURIComponent(scopeImpact.affected[0]) + : '/maintenance?kind=drift' + }; + scopeReviewOpen = false; + scopeDialog?.close(); await load(); - } catch (error) { flash = (error as Error).message; } finally { + } catch (error) { + flash = (error as Error).message; + } finally { updatingScope = false; } } async function deleteArtifact(): Promise { - if (!await confirmAction({title:'Delete ' + family + '?',message:'Delete ' + (artifact?.name ?? 'this file') + ' and its saved versions?', confirmLabel:'Delete', danger:true})) return; + if ( + !(await confirmAction({ + title: 'Delete ' + family + '?', + message: 'Delete ' + (artifact?.name ?? 'this file') + ' and its saved versions?', + confirmLabel: 'Delete', + danger: true + })) + ) + return; try { - const response = await fetch('/api/scripts/' + id, {method:'DELETE'}); + const response = await fetch('/api/scripts/' + id, { method: 'DELETE' }); const body = await response.json(); if (!response.ok || !body.ok) throw new Error(body.error ?? 'Could not delete file.'); await goto(basePath); - } catch (error) { flash = (error as Error).message; } + } catch (error) { + flash = (error as Error).message; + } } let gameOptions = $derived([ @@ -269,13 +381,15 @@ let scopeTypeOptions = [ { value: 'none', label: 'Unscoped' }, { value: 'game', label: 'Game' }, - { value: 'map', label: 'Map' } + { value: 'map', label: 'Map' } ]; {artifact?.name ?? crumbLabel} · Electric Cherry -{#if loadError}{/if} +{#if loadError}{/if} {#if loading}
Loading…
{:else if !artifact && !loadError} @@ -289,13 +403,47 @@ {/snippet} - {#if scopeComplete}
Scope saved. {scopeComplete.composed ? `Intended configuration may change on ${scopeComplete.count} eligible servers.` : scopeComplete.detached ? `Inventory tracking detached on ${scopeComplete.detached} servers; live files remain.` : `${scopeComplete.count} servers are in the old or new scope.`} Maintenance detects resulting drift and rebuilds eligible servers when automation is enabled. View drift recovery →
{/if} - {#if latestVersion}

Latest {artifactVersionLabel(latestVersion)}

{latestVersion.notes || 'No version notes provided.'}

{#if runningBetaId !== null}{:else if latestVersion.id !== a.current_version_id}{:else}Latest version is promoted{/if}
{/if} + {#if scopeComplete}
+ Scope saved. + {scopeComplete.composed + ? `Intended configuration may change on ${scopeComplete.count} eligible servers.` + : scopeComplete.detached + ? `Inventory tracking detached on ${scopeComplete.detached} servers; live files remain.` + : `${scopeComplete.count} servers are in the old or new scope.`} Maintenance detects + resulting drift and rebuilds eligible servers when automation is enabled. + View drift recovery → +
{/if} + {#if latestVersion}
+
+

Latest {artifactVersionLabel(latestVersion)}

+ +

{latestVersion.notes || 'No version notes provided.'}

+
+ {#if runningBetaId !== null}{:else if latestVersion.id !== a.current_version_id}{:else}Latest version is promoted{/if} +
{/if} {#if runningBetaId}
- Beta #{runningBetaId} is running for this {family}. - Rollback or promote it before making changes. + Beta #{runningBetaId} + is running for this {family}. Rollback or promote it before making changes.
{/if} @@ -303,62 +451,109 @@

Details

- -
-
Target file, scope, and eligible servers
-
- Target file - {a.target_rel_path} +
+ Target file, scope, and eligible servers +
+
+ Target file + {a.target_rel_path} +
+
+ Scope + + + {scopeLabel(a)} + + +
+
+ Eligible servers + + {#if !a.scope_type} + Unscoped — deploy from a beta + {:else if matchedServers.length === 0} + No matching servers + {:else} + {#each matchedServers as name}{/each} + {/if} + +
+
+ Promoted version + + {#if currentVersion} + {artifactVersionLabel(currentVersion)} · {currentVersion.sha256.slice( + 0, + 8 + )} + Promoted + {:else} + Never promoted — test a version first + {/if} + +
-
- Scope - - - {scopeLabel(a)} - - -
-
- Eligible servers - - {#if !a.scope_type} - Unscoped — deploy from a beta - {:else if matchedServers.length === 0} - No matching servers - {:else} - {#each matchedServers as name}{/each} - {/if} - -
-
- Promoted version - - {#if currentVersion} - {artifactVersionLabel(currentVersion)} · {currentVersion.sha256.slice(0, 8)} - Promoted - {:else} - Never promoted — test a version first - {/if} - -
-
+ - {#if installedVersions.length}
Recorded installed files · {installedVersions.length} servers

Last recorded file writes, not a verification of the currently running process.

{:else if family !== 'config'}

No installed file versions recorded. Eligibility does not mean a file is installed.

{/if} + {#if installedVersions.length}
+ Recorded installed files · {installedVersions.length} servers +

+ Last recorded file writes, not a verification of the currently running process. +

+ +
{:else if family !== 'config'}

+ No installed file versions recorded. Eligibility does not mean a file is installed. +

{/if} {#if a.kind === 'config' || a.kind === 'baseline'} - load({ seedForm: false })} /> + load({ seedForm: false })} + /> {/if} @@ -367,23 +562,41 @@ {versions.length} {#if versions.length === 0} - + {:else}
- {#each versions.slice(0,limit) as v (v.id)} + {#each versions.slice(0, limit) as v (v.id)}
{artifactVersionLabel(v)} - {v.notes ?? ''} - {#if v.source}GitHub · {v.source.asset_name} ↗{/if} + {v.notes ?? ''} + {#if v.source}GitHub · {v.source.asset_name} ↗{/if} {#if a.current_version_id === v.id} Promoted {:else} - {#if runningBetaId === null}{/if} + {#if runningBetaId === null}{/if} {/if} -
File detailsSHA-256: {v.sha256}Size: {v.size.toLocaleString()} bytes
+
+ File detailsSHA-256: {v.sha256}Size: {v.size.toLocaleString()} bytes +
{/each}
@@ -392,7 +605,14 @@ {#if family === 'plugin'} -
load()} />
+
+ load()} + /> +
{/if} @@ -403,27 +623,92 @@ {#if flash}{/if} - scopeReviewOpen = false} oncancel={event => { if (updatingScope) event.preventDefault(); }} bind:this={scopeDialog} onclick={(e) => onDialogClick(scopeDialog, e)}> + (scopeReviewOpen = false)} + oncancel={(event) => { + if (updatingScope) event.preventDefault(); + }} + bind:this={scopeDialog} + onclick={(e) => onDialogClick(scopeDialog, e)} + >

Edit scope

- +
- {#if scopeType === 'game' || scopeType === 'map'} - (scopeMap = '')} + options={gameOptions} + disabled={updatingScope} + /> {/if} {#if scopeType === 'map'} - {/if}
- {#if scopePreviewLoading}

Reviewing eligible servers…

{:else if scopePreviewError}

{scopePreviewError}

{:else if scopeImpact}
{scopeImpact.after.length} eligible · {scopeImpact.added.length} added · {scopeImpact.removed.length} removed{#if scopeImpact.added.length}

Added: {scopeImpact.added.join(', ')}

{/if}{#if scopeImpact.removed.length}

Removed: {scopeImpact.removed.join(', ')}

{/if}

{scopeImpact.composed ? 'This changes intended configuration composition.' : scopeType === 'none' ? `${scopeImpact.detached.length} recorded installations will become untracked. Maintenance will detect orphaned files as drift.` : 'Scope describes eligibility for promoted files.'}

When automation is enabled, Maintenance rebuilds servers with drift after verifying a safe server state. Rebuilds delete the existing instance and all of its files.

{/if} + {#if scopePreviewLoading}

+ Reviewing eligible servers… +

{:else if scopePreviewError}

+ {scopePreviewError} +

{:else if scopeImpact}
+ {scopeImpact.after.length} eligible · {scopeImpact.added.length} added · + {scopeImpact.removed.length} removed{#if scopeImpact.added.length}

+ Added: {scopeImpact.added.join(', ')} +

{/if}{#if scopeImpact.removed.length}

+ Removed: {scopeImpact.removed.join(', ')} +

{/if} +

+ {scopeImpact.composed + ? 'This changes intended configuration composition.' + : scopeType === 'none' + ? `${scopeImpact.detached.length} recorded installations will become untracked. Maintenance will detect orphaned files as drift.` + : 'Scope describes eligibility for promoted files.'} +

+

+ When automation is enabled, Maintenance rebuilds servers with drift + after verifying a safe server state. Rebuilds delete the existing + instance and all of its files. +

+
{/if}
- - +
@@ -432,25 +717,56 @@
- { if (uploadingVersion) event.preventDefault(); }} bind:this={uploadDialog} onclick={(e) => onDialogClick(uploadDialog, e)}> + { + if (uploadingVersion) event.preventDefault(); + }} + bind:this={uploadDialog} + onclick={(e) => onDialogClick(uploadDialog, e)} + >

Upload new version

- +
- + {#if family === 'plugin'} - - + + {/if} - +
- + + {#if choosingPreset} -
- {#each builtInPlans as preset, index (preset.name)} - - {/each} -
- {#if savedPresets.length} - - {/if} +
+ {#each builtInPlans as preset, index (preset.name)} + + {/each} +
+ {#if savedPresets.length} + + {/if} {/if} {#if message}{/if} {#if error}{/if} - {#if issues.length}{/if} + {#if issues.length}{/if}
-
Your active plan

{matchingPreset?.name ?? 'Custom plan'}

- {plan.requirements.length} requirement{plan.requirements.length === 1 ? '' : 's'} +
+ Your active plan +

{matchingPreset?.name ?? 'Custom plan'}

+
+ {plan.requirements.length} requirement{plan.requirements.length === 1 + ? '' + : 's'}
-
Must pass on
{coverageSummary(plan)}
-
Testing window
{plan.timeoutHours} hours from opening
-
If a check fails
{failureSummary}
+
+
Must pass on
+
{coverageSummary(plan)}
+
+
+
Testing window
+
{plan.timeoutHours} hours from opening
+
+
+
If a check fails
+
{failureSummary}
+
-

Every requirement below must pass on each qualifying server. You can edit any check.

+

+ Every requirement below must pass on each qualifying server. You can edit any check. +

{#if plan.requirements.some(needsGameTelemetry)} -

Gameplay progress comes from game start, round change, and game end reports.

+

+ Gameplay progress comes from game start, round change, + and game end reports. +

{/if}
- {#each plan.requirements as r, index (r.id)} -
-
- {String(index + 1).padStart(2, '0')} -
{requirementCategory(r)}
{requirementSummary(r)}
- -
-
- Edit requirement {requirementLabels[r.kind]} -
-
- - {#if r.kind !== 'manual'} - - {/if} - {#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)} - - {/if} - {#if r.kind === 'matches'} - - {/if} - {#if r.kind === 'matches'} - - {/if} -
- {#if needsGameTelemetry(r)} -

Uses game events reported by this server automatically.

- {:else if r.kind === 'recovery'} -

After the other requirements pass, restart when empty and repeat them. Every cycle requires fresh evidence and checklist sign-off.

- {:else if r.kind === 'failure-budget'} -

Counts transitions from healthy to unhealthy. Missing or stale monitoring resets continuous healthy time.

- {:else if r.kind === 'manual'} -

A tester records their name and observation on the beta detail page.

- {/if} + {#each plan.requirements as r, index (r.id)} +
+
+ {String(index + 1).padStart(2, '0')} +
+ {requirementCategory(r)} +
{requirementSummary(r)}
+
+
-
-
- {:else} -
No requirements yet

Choose a preset above or add your first check below. A beta needs at least one requirement.

- {/each} +
+ Edit requirement {requirementLabels[r.kind]} +
+
+ + {#if r.kind !== 'manual'} + + {/if} + {#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)} + + {/if} + {#if r.kind === 'matches'} + + {/if} + {#if r.kind === 'matches'} + + {/if} +
+ {#if needsGameTelemetry(r)} +

Uses game events reported by this server automatically.

+ {:else if r.kind === 'recovery'} +

+ After the other requirements pass, restart when empty and repeat + them. Every cycle requires fresh evidence and checklist + sign-off. +

+ {:else if r.kind === 'failure-budget'} +

+ Counts transitions from healthy to unhealthy. Missing or stale + monitoring resets continuous healthy time. +

+ {:else if r.kind === 'manual'} +

+ A tester records their name and observation on the beta detail + page. +

+ {/if} +
+
+ + {:else} +
+ No requirements yet +

+ Choose a preset above or add your first check below. A beta needs at least + one requirement. +

+
+ {/each}
- - + +
- Coverage & waves{coverageSummary(plan)} · {waveSummary}Configure + Coverage & waves{coverageSummary(plan)} · {waveSummary}Configure
{#if allServersRequired} -

Every requested instance must pass. Change the instance counts in the Servers step.

+

+ Every requested instance must pass. Change the instance counts in the + Servers step. +

{:else} - - {#if plan.scope === 'quorum'}{/if} + + {#if plan.scope === 'quorum'}{/if} {/if} - +
-

Each wave must fully pass before files reach the next wave. Coverage rules apply after all selected servers have started testing, and every file must pass on at least one matching instance. Restarts always wait for empty servers.

+

+ Each wave must fully pass before files reach the next wave. Coverage rules apply + after all selected servers have started testing, and every file must pass on at + least one matching instance. Restarts always wait for empty servers. +

- Deadline & failure handling{plan.timeoutHours}-hour window · {failureSummary}Configure + Deadline & failure handling{plan.timeoutHours}-hour window · {failureSummary}Configure
- - - + + +
-

Waiting for empty servers uses this window. Pausing and retrying do not extend it. Missing evidence at the deadline is inconclusive. Health checks continue until you manually promote.

+

+ Waiting for empty servers uses this window. Pausing and retrying do not extend + it. Missing evidence at the deadline is inconclusive. Health checks continue + until you manually promote. +

- Save this plan as a presetReuse your requirements and settings on future betas.Save + Save this plan as a presetReuse your requirements and settings on future betas.Save
- - + +
diff --git a/src/lib/ui/components/BetaRevisionDialog.svelte b/src/lib/ui/components/BetaRevisionDialog.svelte index 632f0c7..4582823 100644 --- a/src/lib/ui/components/BetaRevisionDialog.svelte +++ b/src/lib/ui/components/BetaRevisionDialog.svelte @@ -5,96 +5,243 @@ import ServerIdentity from './ServerIdentity.svelte'; import { artifactVersionLabel } from '$lib/artifact-version.js'; - interface FileEntry { id: number; name: string; kind: string; target_rel_path: string; - version: { version: number; release_version: string | null; sha256: string } | null } - let { open = $bindable(false), betaId, revision, files, servers, timeoutHours, onUpdate }: - { open?: boolean; betaId: number; revision: number; files: FileEntry[]; servers: string[]; - timeoutHours?: number; onUpdate: () => Promise } = $props(); - interface Replacement { file: File; sha: string; before: string | null; after: string | null } + interface FileEntry { + id: number; + name: string; + kind: string; + target_rel_path: string; + version: { version: number; release_version: string | null; sha256: string } | null; + } + let { + open = $bindable(false), + betaId, + revision, + files, + servers, + timeoutHours, + onUpdate + }: { + open?: boolean; + betaId: number; + revision: number; + files: FileEntry[]; + servers: string[]; + timeoutHours?: number; + onUpdate: () => Promise; + } = $props(); + interface Replacement { + file: File; + sha: string; + before: string | null; + after: string | null; + } let replacements = $state>({}); - let notes = $state(''), error = $state(''), reviewing = $state(false), busy = $state(false); + let notes = $state(''), + error = $state(''), + reviewing = $state(false), + busy = $state(false); const count = $derived(Object.keys(replacements).length); - $effect(() => { if (open) { replacements = {}; notes = ''; error = ''; reviewing = false; } }); - function fileUrl(scriptId: number) { return `/api/betas/${betaId}/revisions/${revision}/files/${scriptId}`; } + $effect(() => { + if (open) { + replacements = {}; + notes = ''; + error = ''; + reviewing = false; + } + }); + function fileUrl(scriptId: number) { + return `/api/betas/${betaId}/revisions/${revision}/files/${scriptId}`; + } async function choose(entry: FileEntry, file?: File) { - delete replacements[entry.id]; error = ''; + delete replacements[entry.id]; + error = ''; if (!file) return; busy = true; try { if (!file.size) throw new Error('Choose a non-empty file.'); const bytes = await file.arrayBuffer(); - const inspection = new FormData(); inspection.set('content', file); - const inspected = await fetch('/api/scripts/inspect', { method: 'POST', body: inspection }); + const inspection = new FormData(); + inspection.set('content', file); + const inspected = await fetch('/api/scripts/inspect', { + method: 'POST', + body: inspection + }); const inspectionBody = await inspected.json(); - if (!inspected.ok) throw new Error(inspectionBody.error ?? 'Could not inspect replacement file.'); + if (!inspected.ok) + throw new Error(inspectionBody.error ?? 'Could not inspect replacement file.'); const sha = inspectionBody.sha256 as string; - if (sha === entry.version?.sha256) throw new Error(`${entry.name}: this file is identical to the current version.`); - let before: string | null = null, after: string | null = null; + if (sha === entry.version?.sha256) + throw new Error(`${entry.name}: this file is identical to the current version.`); + let before: string | null = null, + after: string | null = null; if (entry.kind !== 'plugin' && bytes.byteLength <= 128 * 1024) { const response = await fetch(fileUrl(entry.id)); - if (!response.ok) throw new Error('Could not load the current file for review. Reload the beta and try again.'); + if (!response.ok) + throw new Error( + 'Could not load the current file for review. Reload the beta and try again.' + ); const old = await response.arrayBuffer(); - if (old.byteLength <= 128 * 1024 && !new Uint8Array(old).includes(0) && !new Uint8Array(bytes).includes(0)) { - before = new TextDecoder().decode(old); after = new TextDecoder().decode(bytes); + if ( + old.byteLength <= 128 * 1024 && + !new Uint8Array(old).includes(0) && + !new Uint8Array(bytes).includes(0) + ) { + before = new TextDecoder().decode(old); + after = new TextDecoder().decode(bytes); } } replacements[entry.id] = { file, sha, before, after }; - } catch (err) { error = (err as Error).message; } - finally { busy = false; } + } catch (err) { + error = (err as Error).message; + } finally { + busy = false; + } } async function deploy() { - busy = true; error = ''; + busy = true; + error = ''; try { const form = new FormData(); - form.set('expectedRevision', String(revision)); form.set('notes', notes); - for (const [scriptId, replacement] of Object.entries(replacements)) form.set(`file:${scriptId}`, replacement.file); - const response = await fetch(`/api/betas/${betaId}/revisions`, { method: 'POST', body: form }); + form.set('expectedRevision', String(revision)); + form.set('notes', notes); + for (const [scriptId, replacement] of Object.entries(replacements)) + form.set(`file:${scriptId}`, replacement.file); + const response = await fetch(`/api/betas/${betaId}/revisions`, { + method: 'POST', + body: form + }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not update beta.'); - open = false; await onUpdate(); - } catch (err) { error = (err as Error).message; } - finally { busy = false; } + open = false; + await onUpdate(); + } catch (err) { + error = (err as Error).message; + } finally { + busy = false; + } } - + {#if error}{/if} {#if !reviewing}

Choose replacements for the files you changed.

-
{#each files as entry} - - {/each}
- +
+ {#each files as entry} + + {/each} +
+ {:else} -

{count} file{count === 1 ? '' : 's'} changed. All {servers.length} selected servers will repeat the test policy, using its existing wave order. Each server installs the bundle and restarts when empty.

-

Previous results move to revision history. Promotion requires this revision to pass. - {#if timeoutHours}A fresh {timeoutHours}-hour test window starts when you deploy, including time waiting for empty servers.{/if} - Rejecting the beta still restores the files from before revision 1.

-
Servers ({servers.length})
    {#each servers as name}
    • {/each}
- {#each files.filter(entry => replacements[entry.id]) as entry} +

+ {count} file{count === 1 ? '' : 's'} changed. All {servers.length} selected + servers will repeat the test policy, using its existing wave order. Each server installs the + bundle and restarts when empty. +

+

+ Previous results move to revision history. Promotion requires this revision to pass. + {#if timeoutHours}A fresh {timeoutHours}-hour test window starts when you deploy, + including time waiting for empty servers.{/if} + Rejecting the beta still restores the files from before revision 1. +

+
+ Servers ({servers.length}) +
    + {#each servers as name}
    • {/each} +
+
+ {#each files.filter((entry) => replacements[entry.id]) as entry} {@const replacement = replacements[entry.id]} -

{entry.name}

{entry.target_rel_path} ← {replacement.file.name} ({replacement.file.size.toLocaleString()} bytes)

-

SHA {entry.version?.sha256.slice(0,12)} → {replacement.sha.slice(0,12)}

- {#if replacement.after !== null} +
+

{entry.name}

+

+ {entry.target_rel_path} ← {replacement.file.name} ({replacement.file.size.toLocaleString()} + bytes) +

+

+ SHA {entry.version?.sha256.slice(0, 12)} → {replacement.sha.slice(0, 12)} +

+ {#if replacement.after !== null} {:else}

Binary or large file: review the uploaded file and checksum.

{/if}
{/each} {#if notes}

Change note: {notes}

{/if} {/if} {#snippet actions()} - - {#if reviewing} - {:else}{/if} + + {#if reviewing} + {:else}{/if} {/snippet}
diff --git a/src/lib/ui/components/BetaTestProgress.svelte b/src/lib/ui/components/BetaTestProgress.svelte index 23111b7..3900e65 100644 --- a/src/lib/ui/components/BetaTestProgress.svelte +++ b/src/lib/ui/components/BetaTestProgress.svelte @@ -12,162 +12,723 @@ import ListFooter from './primitives/ListFooter.svelte'; import Button from './primitives/Button.svelte'; import { confirmAction } from '$ui/confirm.js'; - import { coverageSummary, requirementSummary, needsGameTelemetry } from '$lib/beta-plan-summary.js'; + import { + coverageSummary, + requirementSummary, + needsGameTelemetry + } from '$lib/beta-plan-summary.js'; import { betaWaitingCondition, betaEvidenceSummary } from '$lib/beta-evidence.js'; import type { BetaRequirement, BetaTestView } from '$lib/beta-plan.js'; - let { test, betaId, closed, deadline, retryBlocked, deletedParticipants = false, deployments, busy = $bindable(false), onUpdate }: { test:BetaTestView; betaId:number; closed:boolean; deadline?:string; retryBlocked?:string; deletedParticipants?:boolean; deployments?:BetaDeploymentServer[]; busy?:boolean; onUpdate:()=>Promise } = $props(); - let notes = $state>({}); - let evidenceOpen = $state(false), evidenceLimit = $state(25), error = $state(''), message = $state(''); - let evidenceServer = $state(''), evidenceKind = $state(''), evidenceQuery = $state(''); + let { + test, + betaId, + closed, + deadline, + retryBlocked, + deletedParticipants = false, + deployments, + busy = $bindable(false), + onUpdate + }: { + test: BetaTestView; + betaId: number; + closed: boolean; + deadline?: string; + retryBlocked?: string; + deletedParticipants?: boolean; + deployments?: BetaDeploymentServer[]; + busy?: boolean; + onUpdate: () => Promise; + } = $props(); + let notes = $state>({}); + let evidenceOpen = $state(false), + evidenceLimit = $state(25), + error = $state(''), + message = $state(''); + let evidenceServer = $state(''), + evidenceKind = $state(''), + evidenceQuery = $state(''); let evidenceSection = $state(); - const hasDeletedServers = $derived(deletedParticipants || test.servers.some(server => server.server_deleted_at) || deployments?.some(server => server.server_deleted_at)); - const canSign = $derived(!closed && !hasDeletedServers && ['active','passed'].includes(test.outcome)); - const expired = $derived(test.outcome === 'inconclusive' || (!!deadline && test.outcome !== 'passed' && Date.parse(deadline) <= Date.now())); - const ranked = $derived([...test.servers].sort((a,b) => (a.progress.some(r => r.failed) ? 0 : a.passed ? 2 : 1) - (b.progress.some(r => r.failed) ? 0 : b.passed ? 2 : 1) || a.server.localeCompare(b.server))); + const hasDeletedServers = $derived( + deletedParticipants || + test.servers.some((server) => server.server_deleted_at) || + deployments?.some((server) => server.server_deleted_at) + ); + const canSign = $derived( + !closed && !hasDeletedServers && ['active', 'passed'].includes(test.outcome) + ); + const expired = $derived( + test.outcome === 'inconclusive' || + (!!deadline && test.outcome !== 'passed' && Date.parse(deadline) <= Date.now()) + ); + const ranked = $derived( + [...test.servers].sort( + (a, b) => + (a.progress.some((r) => r.failed) ? 0 : a.passed ? 2 : 1) - + (b.progress.some((r) => r.failed) ? 0 : b.passed ? 2 : 1) || + a.server.localeCompare(b.server) + ) + ); let expanded = $state>({}); const combined = $derived.by(() => { const rows = [...(deployments ?? [])]; for (const server of test.servers) { - if (!rows.some(row => row.server === server.server)) rows.push({ server: server.server, server_deleted_at: server.server_deleted_at, - stage: 'unknown', error: null, deployed_at: null, restarted_at: null, healthy_at: null, match_seen_at: null, provisioning_id: null }); + if (!rows.some((row) => row.server === server.server)) + rows.push({ + server: server.server, + server_deleted_at: server.server_deleted_at, + stage: 'unknown', + error: null, + deployed_at: null, + restarted_at: null, + healthy_at: null, + match_seen_at: null, + provisioning_id: null + }); } - return rows.map(deployment => { - const result = test.servers.find(server => server.server === deployment.server); - const deletedAt = deployment.server_deleted_at ?? result?.server_deleted_at ?? null; - return { deployment: { ...deployment, server_deleted_at: deletedAt }, result: result ? { ...result, server_deleted_at: deletedAt } : undefined }; - }) - .sort((a,b) => rank(a) - rank(b) || a.deployment.server.localeCompare(b.deployment.server)); + return rows + .map((deployment) => { + const result = test.servers.find((server) => server.server === deployment.server); + const deletedAt = deployment.server_deleted_at ?? result?.server_deleted_at ?? null; + return { + deployment: { ...deployment, server_deleted_at: deletedAt }, + result: result ? { ...result, server_deleted_at: deletedAt } : undefined + }; + }) + .sort( + (a, b) => + rank(a) - rank(b) || a.deployment.server.localeCompare(b.deployment.server) + ); }); - function rank(row: { deployment: BetaDeploymentServer; result?: BetaTestView['servers'][number] }) { - return row.deployment.stage === 'failed' || row.result?.progress.some(check => check.failed) ? 0 : row.result?.passed ? 2 : 1; + function rank(row: { + deployment: BetaDeploymentServer; + result?: BetaTestView['servers'][number]; + }) { + return row.deployment.stage === 'failed' || + row.result?.progress.some((check) => check.failed) + ? 0 + : row.result?.passed + ? 2 + : 1; } - const filteredEvidence = $derived(test.evidence.filter(event => (!evidenceServer || event.server === evidenceServer) - && (!evidenceKind || (evidenceKind === 'gameplay' ? ['game_start','round_change','heartbeat','game_end'].includes(event.kind) : evidenceKind === 'health' ? ['health','status-sample'].includes(event.kind) : event.kind === evidenceKind)) - && (!evidenceQuery || `${betaEvidenceSummary(event)} ${event.payload}`.toLowerCase().includes(evidenceQuery.toLowerCase())))); - $effect(() => { evidenceServer; evidenceKind; evidenceQuery; evidenceLimit = 25; }); - export async function action(action:string, extra:Record={}) { - if (action === 'retry' && !await confirmAction({title:'Start a fresh test attempt?',message:`Reset current progress and checklist sign-offs on ${test.servers.length} deployed servers. They will restart when empty and must collect fresh evidence. Previous evidence remains in history. The deadline stays ${deadline ? $timeFormat.dateTime(deadline) : 'unchanged'}.`,confirmLabel:'Reset and restart testing'})) return; - busy = true; error = ''; message = ''; + const filteredEvidence = $derived( + test.evidence.filter( + (event) => + (!evidenceServer || event.server === evidenceServer) && + (!evidenceKind || + (evidenceKind === 'gameplay' + ? ['game_start', 'round_change', 'heartbeat', 'game_end'].includes( + event.kind + ) + : evidenceKind === 'health' + ? ['health', 'status-sample'].includes(event.kind) + : event.kind === evidenceKind)) && + (!evidenceQuery || + `${betaEvidenceSummary(event)} ${event.payload}` + .toLowerCase() + .includes(evidenceQuery.toLowerCase())) + ) + ); + $effect(() => { + void evidenceServer; + void evidenceKind; + void evidenceQuery; + evidenceLimit = 25; + }); + export async function action(action: string, extra: Record = {}) { + if ( + action === 'retry' && + !(await confirmAction({ + title: 'Start a fresh test attempt?', + message: `Reset current progress and checklist sign-offs on ${test.servers.length} deployed servers. They will restart when empty and must collect fresh evidence. Previous evidence remains in history. The deadline stays ${deadline ? $timeFormat.dateTime(deadline) : 'unchanged'}.`, + confirmLabel: 'Reset and restart testing' + })) + ) + return; + busy = true; + error = ''; + message = ''; try { - const response = await fetch(`/api/betas/${betaId}/test`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({action,...extra})}), body = await response.json(); + const response = await fetch(`/api/betas/${betaId}/test`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action, ...extra }) + }), + body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not update the test.'); await onUpdate(); - message = action === 'sign-off' ? 'Observation recorded.' : action === 'retry' ? 'Fresh attempt queued. Deployed servers restart when empty; the deadline is unchanged.' : action === 'pause' ? 'Testing paused. The original deadline still applies.' : 'Testing resumed. Continuous timing checks restart from fresh observations.'; - } catch(e) { error = (e as Error).message; } - finally { busy = false; } + message = + action === 'sign-off' + ? 'Observation recorded.' + : action === 'retry' + ? 'Fresh attempt queued. Deployed servers restart when empty; the deadline is unchanged.' + : action === 'pause' + ? 'Testing paused. The original deadline still applies.' + : 'Testing resumed. Continuous timing checks restart from fresh observations.'; + } catch (e) { + error = (e as Error).message; + } finally { + busy = false; + } } - async function showEvidence(server:string, requirement:BetaRequirement) { - evidenceServer = server; evidenceKind = needsGameTelemetry(requirement) ? 'gameplay' : requirement.kind === 'manual' ? 'manual-sign-off' : ['failure-budget','healthy-minutes'].includes(requirement.kind) ? 'health' : ''; - evidenceQuery = ''; evidenceOpen = true; await tick(); evidenceSection?.scrollIntoView({block:'start',behavior:'smooth'}); evidenceSection?.querySelector('summary')?.focus(); + async function showEvidence(server: string, requirement: BetaRequirement) { + evidenceServer = server; + evidenceKind = needsGameTelemetry(requirement) + ? 'gameplay' + : requirement.kind === 'manual' + ? 'manual-sign-off' + : ['failure-budget', 'healthy-minutes'].includes(requirement.kind) + ? 'health' + : ''; + evidenceQuery = ''; + evidenceOpen = true; + await tick(); + evidenceSection?.scrollIntoView({ block: 'start', behavior: 'smooth' }); + evidenceSection?.querySelector('summary')?.focus(); }
-

{deployments ? 'Test servers' : 'Test results'}

{#if !deployments}{/if}
+
+

{deployments ? 'Test servers' : 'Test results'}

+ {#if !deployments}{/if} +
{#if !closed && !deployments} - {#if !hasDeletedServers}
- {#if test.outcome === 'paused' && !expired}{/if} - {#if test.outcome === 'active'}{/if} - {#if ['paused','failed'].includes(test.outcome) && !expired}{/if} -
{/if} - {#if retryBlocked && ['paused','failed'].includes(test.outcome)}

{retryBlocked}

{/if} - {#if expired}

The testing window ended. Use “Update beta” to test changed files in a fresh revision, or roll back and reuse the plan with a longer window.

- {:else if test.outcome === 'paused' && !hasDeletedServers}

Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.

{/if} + {#if !hasDeletedServers}
+ {#if test.outcome === 'paused' && !expired}{/if} + {#if test.outcome === 'active'}{/if} + {#if ['paused', 'failed'].includes(test.outcome) && !expired}{/if} +
{/if} + {#if retryBlocked && ['paused', 'failed'].includes(test.outcome)}

{retryBlocked}

{/if} + {#if expired}

+ The testing window ended. Use “Update beta” to test changed files in a fresh + revision, or roll back and reuse the plan with a longer window. +

+ {:else if test.outcome === 'paused' && !hasDeletedServers}

+ Resume retains completed milestones; continuous health and in-progress timing + intervals start fresh. Pausing does not extend the deadline. +

{/if} {/if} {#if error}{/if} {#if message}

{message}

{/if} - {#if !test.servers.length}

{closed ? 'Test results were not recorded.' : hasDeletedServers ? 'No test run was recorded before a participating server was deleted.' : 'Awaiting the first test run.'}

{/if} + {#if !test.servers.length}

+ {closed + ? 'Test results were not recorded.' + : hasDeletedServers + ? 'No test run was recorded before a participating server was deleted.' + : 'Awaiting the first test run.'} +

{/if} {#if deployments} -
- - {#each combined as { deployment, result } (deployment.server)} - {@const failed = deployment.stage === 'failed' || result?.progress.some(check => check.failed)} - {@const open = expanded[deployment.server] ?? Boolean(failed)} - - - - - - - - {/each} -
ServerDeploymentTest resultDetails
{#if result?.passed}Passed - {:else if result?.progress.some(check => check.failed)}Failed - {:else if !result}{closed || deployment.server_deleted_at ? 'Not recorded' : 'Awaiting test run'} - {:else}{closed ? 'Incomplete' : test.outcome === 'paused' ? 'Paused' : 'Waiting'}{/if} -
+
+ + + {#each combined as { deployment, result } (deployment.server)} + {@const failed = + deployment.stage === 'failed' || + result?.progress.some((check) => check.failed)} + {@const open = expanded[deployment.server] ?? Boolean(failed)} + + + + + + + + {/each} +
ServerDeploymentTest resultDetails
{#if result?.passed}Passed + {:else if result?.progress.some((check) => check.failed)}Failed + {:else if !result}{closed || deployment.server_deleted_at + ? 'Not recorded' + : 'Awaiting test run'} + {:else}{closed + ? 'Incomplete' + : test.outcome === 'paused' + ? 'Paused' + : 'Waiting'}{/if} +
+
{:else} - {#each ranked as server (server.server)} -
- {server.progress.filter(p=>p.passed).length} / {server.progress.length} checks passed{#if test.plan.requirements.some(r=>r.kind==='recovery')}Recovery cycles: {server.phase}{/if} - {@render checks(server)} -
- {/each} + {#each ranked as server (server.server)} +
+ {server.progress.filter((p) => p.passed).length} / {server.progress.length} checks + passed{#if test.plan.requirements.some((r) => r.kind === 'recovery')}Recovery cycles: {server.phase}{/if} + {@render checks(server)} +
+ {/each} {/if} -
Test policy

{coverageSummary(test.plan)} · Every requirement must pass on each qualifying server.

{test.plan.waveSize ? `${test.plan.waveSize} servers per wave.` : 'All selected servers deployed together.'} {test.plan.failureAction === 'rollback' ? 'Failed checks trigger automatic rollback.' : test.plan.failureAction === 'pause' ? 'Failed checks pause testing for investigation.' : 'Failed checks hold testing for review.'} {test.plan.rollbackRestart === 'when-empty' ? 'Rollback restarts wait for empty servers.' : 'Rollback changes wait until the rollback campaign is resumed.'} Health checks continue after passing until promotion.

+
+ Test policy +

{coverageSummary(test.plan)} · Every requirement must pass on each qualifying server.

+

+ {test.plan.waveSize + ? `${test.plan.waveSize} servers per wave.` + : 'All selected servers deployed together.'} + {test.plan.failureAction === 'rollback' + ? 'Failed checks trigger automatic rollback.' + : test.plan.failureAction === 'pause' + ? 'Failed checks pause testing for investigation.' + : 'Failed checks hold testing for review.'} + {test.plan.rollbackRestart === 'when-empty' + ? 'Rollback restarts wait for empty servers.' + : 'Rollback changes wait until the rollback campaign is resumed.'} Health checks continue + after passing until promotion. +

+
- Evidence ({test.evidence.length}{test.evidence.length===200 ? ' most recent' : ''}) + Evidence ({test.evidence.length}{test.evidence.length === 200 + ? ' most recent' + : ''}) {#if evidenceOpen} -

Newest first. Includes retained evidence from earlier attempts and recovery cycles; current progress uses only the current attempt.

-
- {#each filteredEvidence.slice(0,evidenceLimit) as event(event.id)}

{betaEvidenceSummary(event)}

Raw payload · {event.kind}
{event.payload}
{/each} - - {#if !filteredEvidence.length}

{test.evidence.length ? 'No evidence matches these filters.' : 'No evidence recorded yet. Check deployment progress and telemetry diagnostics.'}

{/if} +

+ Newest first. Includes retained evidence from earlier attempts and recovery cycles; + current progress uses only the current attempt. +

+
+ +
+ {#each filteredEvidence.slice(0, evidenceLimit) as event (event.id)}
+
+ +
+

{betaEvidenceSummary(event)}

+
+ Raw payload · {event.kind} + 
{event.payload}
+
+
{/each} + + {#if !filteredEvidence.length}

+ {test.evidence.length + ? 'No evidence matches these filters.' + : 'No evidence recorded yet. Check deployment progress and telemetry diagnostics.'} +

{/if} {/if}
{#snippet checks(server: BetaTestView['servers'][number])} - {#if test.plan.requirements.some(needsGameTelemetry)}

{#if server.observation?.lastGameplayAt}Latest gameplay: {server.observation.lastEvent?.replaceAll('_',' ')} · · {server.observation.players ?? 'Unknown'} players{#if server.observation.round !== null} · Round {server.observation.round}{/if}{:else}No gameplay reports in this attempt yet.{/if} {#if !server.server_deleted_at}Telemetry diagnostics{/if}

{/if} - {#each server.progress as progress (progress.id)} - {@const requirement = test.plan.requirements.find(r=>r.id===progress.id)} - {#if requirement}
-
{progress.passed ? 'Passed' : progress.failed ? 'Failed' : 'Waiting'} {requirementSummary(requirement)}{progress.detail}
- {#if !progress.passed && !closed && !hasDeletedServers}

{betaWaitingCondition(requirement,server.observation)}

{/if} - {#if !['manual','failure-budget'].includes(requirement.kind)}{/if} + {#if test.plan.requirements.some(needsGameTelemetry)}

+ {#if server.observation?.lastGameplayAt}Latest gameplay: {server.observation.lastEvent?.replaceAll( + '_', + ' ' + )} · + + · {server.observation.players ?? 'Unknown'} players{#if server.observation.round !== null} + · Round {server.observation.round}{/if}{:else}No gameplay reports in this + attempt yet.{/if} + {#if !server.server_deleted_at}Telemetry diagnostics{/if} +

{/if} + {#each server.progress as progress (progress.id)} + {@const requirement = test.plan.requirements.find((r) => r.id === progress.id)} + {#if requirement}
+
+ {progress.passed + ? 'Passed' + : progress.failed + ? 'Failed' + : 'Waiting'} + {requirementSummary(requirement)}{progress.detail} +
+ {#if !progress.passed && !closed && !hasDeletedServers}

+ {betaWaitingCondition(requirement, server.observation)} +

{/if} + {#if !['manual', 'failure-budget'].includes(requirement.kind)}{/if} {#if requirement.kind === 'manual' && !progress.passed && !closed && !hasDeletedServers} - {#if canSign && server.observingAt}
- {:else if canSign}

Waiting for this server's restart and health verification to finish before tester sign-off.

- {:else}

Sign-off is unavailable while this test is {test.outcome}. {test.outcome === 'paused' ? 'Resume testing first.' : 'Use the recovery instructions above.'}

{/if} + {#if canSign && server.observingAt}
+ +
+ {:else if canSign}

+ Waiting for this server's restart and health verification to finish + before tester sign-off. +

+ {:else}

+ Sign-off is unavailable while this test is {test.outcome}. {test.outcome === + 'paused' + ? 'Resume testing first.' + : 'Use the recovery instructions above.'} +

{/if} {/if} - +
{/if} - {/each} + {/each} {/snippet} diff --git a/src/lib/ui/components/BlueprintBuilder.svelte b/src/lib/ui/components/BlueprintBuilder.svelte index c3cab6b..864d0ae 100644 --- a/src/lib/ui/components/BlueprintBuilder.svelte +++ b/src/lib/ui/components/BlueprintBuilder.svelte @@ -4,8 +4,15 @@ import type { BlueprintSummary } from '$lib/deployment-plan.js'; import type { ZombieGame, ZombieMapEntry } from '$lib/map-aliases.js'; - let { open = $bindable(false), blueprint = null, onsaved }: - { open?: boolean; blueprint?: BlueprintSummary | null; onsaved: (blueprint: BlueprintSummary, launch: boolean) => void } = $props(); + let { + open = $bindable(false), + blueprint = null, + onsaved + }: { + open?: boolean; + blueprint?: BlueprintSummary | null; + onsaved: (blueprint: BlueprintSummary, launch: boolean) => void; + } = $props(); let dialog: HTMLDialogElement | undefined = $state(); let name = $state(''); let description = $state(''); @@ -15,25 +22,73 @@ let busy = $state(false); let error = $state(''); let total = $derived(rows.reduce((sum, row) => sum + (Number(row.count) || 0), 0)); - const duplicate = (row: typeof rows[number]) => rows.some(other => other.key !== row.key && other.game === row.game && other.mapSlug === row.mapSlug); - let valid = $derived(name.trim().length > 0 && rows.length > 0 && total <= 100 && rows.every(row => row.mapSlug && Number.isInteger(row.count) && row.count >= 1 && row.count <= 15 && !duplicate(row))); - let reason = $derived(!name.trim() ? 'Enter a collection name to save.' : total > 100 ? `Reduce the total by ${total - 100} to stay within 100 target lobbies.` : rows.some(duplicate) ? 'Choose each game and map once; increase its lobby count instead.' : !valid ? 'Choose a map and 1–15 target lobbies for every row.' : ''); - let canAdd = $derived((['t5', 't6'] as const).some(game => catalogs[game].some(map => !rows.some(row => row.game === game && row.mapSlug === map.slug)))); + const duplicate = (row: (typeof rows)[number]) => + rows.some( + (other) => + other.key !== row.key && other.game === row.game && other.mapSlug === row.mapSlug + ); + let valid = $derived( + name.trim().length > 0 && + rows.length > 0 && + total <= 100 && + rows.every( + (row) => + row.mapSlug && + Number.isInteger(row.count) && + row.count >= 1 && + row.count <= 15 && + !duplicate(row) + ) + ); + let reason = $derived( + !name.trim() + ? 'Enter a collection name to save.' + : total > 100 + ? `Reduce the total by ${total - 100} to stay within 100 target lobbies.` + : rows.some(duplicate) + ? 'Choose each game and map once; increase its lobby count instead.' + : !valid + ? 'Choose a map and 1–15 target lobbies for every row.' + : '' + ); + let canAdd = $derived( + (['t5', 't6'] as const).some((game) => + catalogs[game].some( + (map) => !rows.some((row) => row.game === game && row.mapSlug === map.slug) + ) + ) + ); let wasOpen = false; let session = 0; let nextKey = 0; $effect(() => { if (!dialog) return; - if (open && !wasOpen) { wasOpen = true; dialog.showModal(); void initialize(); } - else if (!open && wasOpen) { wasOpen = false; session++; dialog.close(); } + if (open && !wasOpen) { + wasOpen = true; + dialog.showModal(); + void initialize(); + } else if (!open && wasOpen) { + wasOpen = false; + session++; + dialog.close(); + } }); async function initialize() { const current = ++session; - name = blueprint?.name ?? ''; description = blueprint?.description ?? ''; - rows = blueprint?.entries.map(entry => ({ key: nextKey++, game: entry.game, mapSlug: entry.mapSlug, count: entry.count })) ?? []; - error = ''; loaded = false; busy = false; + name = blueprint?.name ?? ''; + description = blueprint?.description ?? ''; + rows = + blueprint?.entries.map((entry) => ({ + key: nextKey++, + game: entry.game, + mapSlug: entry.mapSlug, + count: entry.count + })) ?? []; + error = ''; + loaded = false; + busy = false; try { const res = await fetch('/api/maps'); if (!res.ok) throw new Error('Could not load maps. Try again.'); @@ -42,77 +97,373 @@ catalogs = { t5: body.t5Zombie, t6: body.t6Zombie }; if (!rows.length) addMap(); loaded = true; - } catch (err) { if (current === session) error = (err as Error).message; } + } catch (err) { + if (current === session) error = (err as Error).message; + } } function addMap() { for (const game of ['t5', 't6'] as const) { - const map = catalogs[game].find(map => !rows.some(row => row.game === game && row.mapSlug === map.slug)); - if (map) { rows = [...rows, { key: nextKey++, game, mapSlug: map.slug, count: 1 }]; return; } + const map = catalogs[game].find( + (map) => !rows.some((row) => row.game === game && row.mapSlug === map.slug) + ); + if (map) { + rows = [...rows, { key: nextKey++, game, mapSlug: map.slug, count: 1 }]; + return; + } } } async function save(launch: boolean) { if (busy || !valid) return; - busy = true; error = ''; + busy = true; + error = ''; try { - const res = await fetch(blueprint ? `/api/blueprints/${encodeURIComponent(blueprint.id)}` : '/api/blueprints', { - method: blueprint ? 'PUT' : 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name, description, entries: rows.map(({ game, mapSlug, count }) => ({ game, mapSlug, count })) }) }); + const res = await fetch( + blueprint + ? `/api/blueprints/${encodeURIComponent(blueprint.id)}` + : '/api/blueprints', + { + method: blueprint ? 'PUT' : 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name, + description, + entries: rows.map(({ game, mapSlug, count }) => ({ game, mapSlug, count })) + }) + } + ); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not save collection.'); - open = false; onsaved(body.blueprint, launch); - } catch (err) { error = (err as Error).message; } - finally { busy = false; } + open = false; + onsaved(body.blueprint, launch); + } catch (err) { + error = (err as Error).message; + } finally { + busy = false; + } } - { if (isDialogBackdrop(event, dialog) && !busy) open = false; }} oncancel={event => { if (busy) event.preventDefault(); }} onclose={() => open = false}> + { + if (isDialogBackdrop(event, dialog) && !busy) open = false; + }} + oncancel={(event) => { + if (busy) event.preventDefault(); + }} + onclose={() => (open = false)} +>
-
One collection. Your whole lineup.

{blueprint ? 'Edit your collection' : 'Build a server collection'}

-

Choose the total lobbies you want for each map. Existing and queued servers count; only missing lobbies are created. Mix Black Ops I and II in the same collection.

+
+
+ One collection. Your whole lineup. +

{blueprint ? 'Edit your collection' : 'Build a server collection'}

+
+ +
+

+ Choose the total lobbies you want for each map. Existing and queued servers count; only + missing lobbies are created. Mix Black Ops I and II in the same collection. +

{#if error}{/if} - {#if !loaded}

Loading maps…

{#if error}{/if} + {#if !loaded}

Loading maps…

+ {#if error}{/if} {:else} - { event.preventDefault(); void save(true); }}> -
+ { + event.preventDefault(); + void save(true); + }} + > +
+ +
-

Your lineup

100} role="status">{total} / 100 target lobbies
+
+

Your lineup

+ 100} role="status" + >{total} / 100 target lobbies +
{#each rows as row, index (row.key)}
{String(index + 1).padStart(2, '0')} - - - - + + + +
{/each}
-

Collections save your maps and target lobby counts. Every new server uses the latest promoted scripts, plugins, and configuration. Before launch, you’ll review every new server ID, port, and key.

+

+ Collections save your maps and target lobby counts. Every new server uses the + latest promoted scripts, plugins, and configuration. Before launch, you’ll + review every new server ID, port, and key. +

{#if reason}

{reason}

{/if} -
+
+ +
{/if}
diff --git a/src/lib/ui/components/BrandMark.svelte b/src/lib/ui/components/BrandMark.svelte index 6ed18f3..5fa3ad0 100644 --- a/src/lib/ui/components/BrandMark.svelte +++ b/src/lib/ui/components/BrandMark.svelte @@ -26,15 +26,32 @@ letter-spacing: 0.12em; line-height: 1; } - .mark-sm .word { font-size: var(--text-sm); letter-spacing: 1.5px; } - .mark-md .word { font-size: var(--text-md); } - .mark-lg .word { font-size: var(--text-2xl); letter-spacing: 3px; } + .mark-sm .word { + font-size: var(--text-sm); + letter-spacing: 1.5px; + } + .mark-md .word { + font-size: var(--text-md); + } + .mark-lg .word { + font-size: var(--text-2xl); + letter-spacing: 3px; + } .bolt { fill: var(--accent-primary-hover); filter: drop-shadow(0 0 5px rgba(124, 212, 255, 0.75)); } - .mark-sm .bolt { width: 12px; height: 20px; } - .mark-md .bolt { width: 14px; height: 24px; } - .mark-lg .bolt { width: 22px; height: 36px; } + .mark-sm .bolt { + width: 12px; + height: 20px; + } + .mark-md .bolt { + width: 14px; + height: 24px; + } + .mark-lg .bolt { + width: 22px; + height: 36px; + } diff --git a/src/lib/ui/components/CapacitySamples.svelte b/src/lib/ui/components/CapacitySamples.svelte index d56dec6..0ac766c 100644 --- a/src/lib/ui/components/CapacitySamples.svelte +++ b/src/lib/ui/components/CapacitySamples.svelte @@ -15,57 +15,138 @@ $effect(() => { if (!open) return; - const url = `/api/hosts/${encodeURIComponent(hostId)}/samples?` + new URLSearchParams({ scope, phase, page: String(page) }); + const url = + `/api/hosts/${encodeURIComponent(hostId)}/samples?` + + new URLSearchParams({ scope, phase, page: String(page) }); void revision; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 15000); let active = true; - loading = true; error = ''; result = null; + loading = true; + error = ''; + result = null; void (async () => { try { const response = await fetch(url, { signal: controller.signal }); const body = await response.json(); - if (!response.ok) throw new Error(body.message ?? body.error ?? 'Could not load samples.'); + if (!response.ok) + throw new Error(body.message ?? body.error ?? 'Could not load samples.'); if (active) result = body; } catch (cause) { - if (active) error = controller.signal.aborted ? 'Sample request timed out. Try refreshing.' : (cause as Error).message; - } finally { clearTimeout(timer); if (active) loading = false; } + if (active) + error = controller.signal.aborted + ? 'Sample request timed out. Try refreshing.' + : (cause as Error).message; + } finally { + clearTimeout(timer); + if (active) loading = false; + } })(); - return () => { active = false; clearTimeout(timer); controller.abort(); }; + return () => { + active = false; + clearTimeout(timer); + controller.abort(); + }; });
View samples {#if open} -

Retained minute peaks from the last 24 hours, used without extra allowance padding. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.

+

+ Retained minute peaks from the last 24 hours, used without extra allowance padding. CPU + and memory peaks may come from different heartbeats. Choose the fleet to inspect + measurements from other hosts used by fleet allowances. +

- - - + + +
{#if error}{/if} {#if loading}

Loading samples…

{:else if result} - {#if result.total === 0}

No {phase === 'all' ? '' : phase === 'startup' ? 'startup ' : 'running '}samples recorded for this source in the last 24 hours.

+ {#if result.total === 0}

+ No {phase === 'all' ? '' : phase === 'startup' ? 'startup ' : 'running '}samples + recorded for this source in the last 24 hours. +

{:else} -

{result.total.toLocaleString()} samples · newest first. Counts can differ from the allowance above, which is recalculated periodically.

+

+ {result.total.toLocaleString()} samples · newest first. Counts can differ from the + allowance above, which is recalculated periodically. +

-
- - {#if scope === 'fleet'}{/if} - {#each result.samples as sample} - - {#if scope === 'fleet'}{/if} - - - {/each} +
+
CPU and memory peaks per container and minute; times are local.
TimeHostServerPhaseCPU coresMemory
{sample.hostName ?? sample.hostId}{sample.server ?? sample.instanceId}Generation {sample.generation}{sample.phase === 'startup' ? 'Starting' : 'Running'}{sample.cpuCores.toFixed(3)}{(sample.memoryBytes / 1048576).toFixed(1)} MiB
+ + {#if scope === 'fleet'}{/if} + {#each result.samples as sample} + + {#if scope === 'fleet'}{/if} + + + {/each}
CPU and memory peaks per container and minute; times are local.
TimeHostServerPhaseCPU coresMemory
{sample.hostName ?? sample.hostId}{sample.server ?? sample.instanceId}Generation {sample.generation}{sample.phase === 'startup' ? 'Starting' : 'Running'}{sample.cpuCores.toFixed(3)}{(sample.memoryBytes / 1048576).toFixed(1)} MiB
{/if} {/if} @@ -73,13 +154,75 @@
diff --git a/src/lib/ui/components/ChatPanel.svelte b/src/lib/ui/components/ChatPanel.svelte index 922cf62..cc2186b 100644 --- a/src/lib/ui/components/ChatPanel.svelte +++ b/src/lib/ui/components/ChatPanel.svelte @@ -8,12 +8,22 @@ import Button from './primitives/Button.svelte'; import Freshness from './Freshness.svelte'; - let { server, history, hostOffline = false, hostName }: { server?: string; history?: ContextualChatEntry[]; hostOffline?: boolean; hostName?: string } = $props(); + let { + server, + history, + hostOffline = false, + hostName + }: { + server?: string; + history?: ContextualChatEntry[]; + hostOffline?: boolean; + hostName?: string; + } = $props(); let updatedAt = $state(null); let liveEntries = $state([]); const entries = $derived(history ?? liveEntries); let timeline = $derived(groupChatTimeline(entries)); - let hasMessages = $derived(entries.some(entry => entry.type === 'message')); + let hasMessages = $derived(entries.some((entry) => entry.type === 'message')); let liveLoaded = $state(false); const loaded = $derived(history !== undefined || liveLoaded); let loading = $state(false); @@ -36,23 +46,33 @@ const request = new AbortController(); controller = request; try { - const response = await fetch(`/api/servers/${encodeURIComponent(server)}/chat`, { signal: request.signal }); + const response = await fetch(`/api/servers/${encodeURIComponent(server)}/chat`, { + signal: request.signal + }); const data = await response.json().catch(() => null); - if (!response.ok || !data?.ok) throw new Error('Could not refresh chat. Try again shortly.'); + if (!response.ok || !data?.ok) + throw new Error('Could not refresh chat. Try again shortly.'); if (request.signal.aborted || hostOffline) return; if (paused && !manual) return; liveEntries = data.entries ?? []; missing = !!data.missing; - liveLoaded = true; updatedAt = Date.now(); + liveLoaded = true; + updatedAt = Date.now(); error = ''; await tick(); if (follow && body) body.scrollTop = body.scrollHeight; } catch (err) { - if (!request.signal.aborted) error = err instanceof Error ? err.message : 'Could not refresh chat.'; - } finally { loading = false; manualRefreshing = false; } + if (!request.signal.aborted) + error = err instanceof Error ? err.message : 'Could not refresh chat.'; + } finally { + loading = false; + manualRefreshing = false; + } } - $effect(() => { if (hostOffline) controller?.abort(); }); + $effect(() => { + if (hostOffline) controller?.abort(); + }); function onScroll() { if (body) follow = body.scrollHeight - body.scrollTop - body.clientHeight < 40; @@ -64,8 +84,11 @@ } onMount(() => { if (!server) return; - const stop = pollVisible(() => paused ? Promise.resolve() : refresh(), 5000); - return () => { stop(); controller?.abort(); }; + const stop = pollVisible(() => (paused ? Promise.resolve() : refresh()), 5000); + return () => { + stop(); + controller?.abort(); + }; }); @@ -74,54 +97,128 @@ {/snippet} {#snippet clockTime(value: string)} - + {/snippet} {#snippet chatContext(context: ChatContext)}
-
Match
{#if context.matchElapsedMs === null}{@render unavailable()}{:else}{formatDuration(context.matchElapsedMs)}{/if}
-
Round
{#if context.round === null}{@render unavailable()}{:else}{context.round}{/if}
+
+
Match
+
+ {#if context.matchElapsedMs === null}{@render unavailable()}{:else}{formatDuration( + context.matchElapsedMs + )}{/if} +
+
+
+
Round
+
+ {#if context.round === null}{@render unavailable()}{:else}{context.round}{/if} +
+
{/snippet} {#snippet matchBoundary(entry: ChatBoundaryEntry)}
- {entry.event === 'start' ? 'Match started' : entry.event === 'end' ? 'Match ended' : 'New match'} - {#if entry.context.realTime}{@render clockTime(entry.context.realTime)}{/if} - {#if entry.context.matchElapsedMs !== null}Match {formatDuration(entry.context.matchElapsedMs)}{/if} - {#if entry.context.round !== null}Round {entry.context.round}{/if} + {entry.event === 'start' + ? 'Match started' + : entry.event === 'end' + ? 'Match ended' + : 'New match'} + {#if entry.context.realTime}{@render clockTime(entry.context.realTime)}{/if} + {#if entry.context.matchElapsedMs !== null}Match {formatDuration(entry.context.matchElapsedMs)}{/if} + {#if entry.context.round !== null}Round {entry.context.round}{/if}
{/snippet}
-

Chat

{#if !server}Match history{:else if hostOffline}Host offline{:else if paused}Paused{:else if !error}{/if}
+
+

Chat

+ {#if !server}Match history{:else if hostOffline}Host offline{:else if paused}Paused{:else if !error}{/if} +
{#if server}
- {#if hasMessages}{/if} - {#if !follow}{/if} + {#if hasMessages}{/if} + {#if !follow}{/if}
{/if}
- {#if hostOffline}

Chat resumes when {hostName ?? 'the host agent'} reconnects.{loaded ? ' Showing the last loaded messages.' : ''}

- {:else if error}{/if} + {#if hostOffline}

+ Chat resumes when {hostName ?? 'the host agent'} reconnects.{loaded + ? ' Showing the last loaded messages.' + : ''} +

+ {:else if error}{/if} {#if !loaded && !error && !hostOffline}

Loading chat…

{:else if loaded && !hasMessages} -

{history !== undefined ? 'No chat messages were saved for this match.' : missing ? 'Waiting for the game log. Player messages will appear here.' : 'No player messages in this match yet.'}

+

+ {history !== undefined + ? 'No chat messages were saved for this match.' + : missing + ? 'Waiting for the game log. Player messages will appear here.' + : 'No player messages in this match yet.'} +

{:else if hasMessages} -
+
    {#each timeline as item} {#if item.type === 'boundaries'}
  1. {#if item.entries.length > 1} {#key JSON.stringify(item.entries)} -
    { if (event.currentTarget.open) follow = false; }}> - {item.entries.length - 1} earlier match {item.entries.length === 2 ? 'event' : 'events'} +
    { + if (event.currentTarget.open) follow = false; + }} + > + {item.entries.length - 1} earlier match {item.entries + .length === 2 + ? 'event' + : 'events'}
      {#each item.entries.slice(0, -1) as entry}
    1. {@render matchBoundary(entry)}
      2. @@ -135,10 +232,17 @@ {:else}
    2. - {#if item.entry.context.realTime}{@render clockTime(item.entry.context.realTime)}{:else}Time: {@render unavailable()}{/if} + {#if item.entry.context.realTime}{@render clockTime( + item.entry.context.realTime + )}{:else}Time: + {@render unavailable()}{/if}
      -
      {item.entry.message.player}{@render chatContext(item.entry.context)}
      +
      + {item.entry.message.player}{@render chatContext(item.entry.context)} +

      {item.entry.message.message}

      3. @@ -150,44 +254,214 @@
diff --git a/src/lib/ui/components/CommandPalette.svelte b/src/lib/ui/components/CommandPalette.svelte index b240609..25ce3cf 100644 --- a/src/lib/ui/components/CommandPalette.svelte +++ b/src/lib/ui/components/CommandPalette.svelte @@ -3,8 +3,16 @@ import { goto } from '$app/navigation'; import type { UserRole } from '$lib/auth.js'; import Dialog from './primitives/Dialog.svelte'; - import { loadPaletteDirectory, paletteDirectoryResults, paletteGroups, paletteMatchJump, palettePageResults, searchPaletteRecords, - type PaletteDirectory, type PaletteItem } from '$ui/command-palette.js'; + import { + loadPaletteDirectory, + paletteDirectoryResults, + paletteGroups, + paletteMatchJump, + palettePageResults, + searchPaletteRecords, + type PaletteDirectory, + type PaletteItem + } from '$ui/command-palette.js'; let { open = $bindable(false), role }: { open?: boolean; role: UserRole } = $props(); const id = $props.id(); @@ -20,54 +28,111 @@ let navigating = $state(false); const items = $derived.by(() => { const unique = new Map(); - for (const item of [...palettePageResults(role, query), ...paletteDirectoryResults(query, directory), ...paletteMatchJump(query), ...records]) unique.set(item.key, item); - return paletteGroups.flatMap(group => [...unique.values()].filter(item => item.group === group)); + for (const item of [ + ...palettePageResults(role, query), + ...paletteDirectoryResults(query, directory), + ...paletteMatchJump(query), + ...records + ]) + unique.set(item.key, item); + return paletteGroups.flatMap((group) => + [...unique.values()].filter((item) => item.group === group) + ); }); - const activeIndex = $derived(Math.max(0, items.findIndex(item => item.key === selected))); + const activeIndex = $derived( + Math.max( + 0, + items.findIndex((item) => item.key === selected) + ) + ); const unavailable = $derived([...(directory?.unavailable ?? []), ...recordErrors]); const searching = $derived(directoryLoading || recordsLoading); $effect(() => { - if (!open) { query = ''; records = []; directory = null; navigationError = ''; return; } + if (!open) { + query = ''; + records = []; + directory = null; + navigationError = ''; + return; + } let cancelled = false; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 15_000); directoryLoading = true; - void loadPaletteDirectory(controller.signal).then(result => { if (!cancelled) directory = result; }) - .finally(() => { clearTimeout(timeout); if (!cancelled) directoryLoading = false; }); - void tick().then(() => { if (!cancelled) input?.focus(); }); - return () => { cancelled = true; controller.abort(); clearTimeout(timeout); }; + void loadPaletteDirectory(controller.signal) + .then((result) => { + if (!cancelled) directory = result; + }) + .finally(() => { + clearTimeout(timeout); + if (!cancelled) directoryLoading = false; + }); + void tick().then(() => { + if (!cancelled) input?.focus(); + }); + return () => { + cancelled = true; + controller.abort(); + clearTimeout(timeout); + }; }); $effect(() => { const term = query.trim(); - selected = ''; records = []; recordErrors = []; - if (!open || term.length < 2) { recordsLoading = false; return; } + selected = ''; + records = []; + recordErrors = []; + if (!open || term.length < 2) { + recordsLoading = false; + return; + } let cancelled = false; const controller = new AbortController(); let timeout: ReturnType | undefined; recordsLoading = true; const debounce = setTimeout(() => { timeout = setTimeout(() => controller.abort(), 15_000); - void searchPaletteRecords(term, controller.signal).then(result => { - if (!cancelled) { records = result.items; recordErrors = result.unavailable; } - }).finally(() => { clearTimeout(timeout); if (!cancelled) recordsLoading = false; }); + void searchPaletteRecords(term, controller.signal) + .then((result) => { + if (!cancelled) { + records = result.items; + recordErrors = result.unavailable; + } + }) + .finally(() => { + clearTimeout(timeout); + if (!cancelled) recordsLoading = false; + }); }, 220); - return () => { cancelled = true; clearTimeout(debounce); clearTimeout(timeout); controller.abort(); }; + return () => { + cancelled = true; + clearTimeout(debounce); + clearTimeout(timeout); + controller.abort(); + }; }); $effect(() => { const option = activeIndex; if (!open || !items.length) return; - void tick().then(() => document.getElementById(`${id}-option-${option}`)?.scrollIntoView({ block: 'nearest' })); + void tick().then(() => + document.getElementById(`${id}-option-${option}`)?.scrollIntoView({ block: 'nearest' }) + ); }); async function choose(item: PaletteItem) { if (navigating) return; - navigating = true; navigationError = ''; - try { await goto(item.href); open = false; } - catch { navigationError = 'Could not open that result. Try again.'; } - finally { navigating = false; } + navigating = true; + navigationError = ''; + try { + await goto(item.href); + open = false; + } catch { + navigationError = 'Could not open that result. Try again.'; + } finally { + navigating = false; + } } function keydown(event: KeyboardEvent) { @@ -77,54 +142,169 @@ const movement = event.key === 'ArrowDown' ? 1 : -1; selected = items[(activeIndex + movement + items.length) % items.length].key; } else if (event.key === 'Enter') { - event.preventDefault(); void choose(items[activeIndex]); + event.preventDefault(); + void choose(items[activeIndex]); } } - +
- -

Try a lobby, player name, or #match number. ↑ ↓ to choose · Enter to open

-
+ +

+ Try a lobby, player name, or #match number. ↑ ↓ to choose · Enter to open +

+
{#each paletteGroups as group} - {@const groupItems = items.filter(item => item.group === group)} + {@const groupItems = items.filter((item) => item.group === group)} {#if groupItems.length}
{group}
{#each groupItems as item (item.key)} {@const index = items.indexOf(item)} - {/each}
{/if} {/each}
-

{searching ? 'Searching…' : items.length ? `${items.length} results` : 'No matches. Try a different name or a match number.'}

- {#if unavailable.length}

{unavailable.join(', ')} search is unavailable. Other results are still available.

{/if} +

+ {searching + ? 'Searching…' + : items.length + ? `${items.length} results` + : 'No matches. Try a different name or a match number.'} +

+ {#if unavailable.length}

+ {unavailable.join(', ')} search is unavailable. Other results are still available. +

{/if} {#if navigationError}{/if}
diff --git a/src/lib/ui/components/ConfigEditor.svelte b/src/lib/ui/components/ConfigEditor.svelte index d02aa30..8f625fa 100644 --- a/src/lib/ui/components/ConfigEditor.svelte +++ b/src/lib/ui/components/ConfigEditor.svelte @@ -6,10 +6,29 @@ import ConfigInspector from './ConfigInspector.svelte'; import Button from './primitives/Button.svelte'; import { configWarnings } from '$lib/config-workbench.js'; - interface Version { id: number; version: number; notes: string | null; } - let { id, kind, versions, currentVersionId, runningBetaId, scopeType, scopeValue, onsaved }: { - id: number; kind: 'baseline' | 'config'; versions: Version[]; currentVersionId: number | null; - runningBetaId: number | null; scopeType: string | null; scopeValue: string | null; onsaved: () => Promise; + interface Version { + id: number; + version: number; + notes: string | null; + } + let { + id, + kind, + versions, + currentVersionId, + runningBetaId, + scopeType, + scopeValue, + onsaved + }: { + id: number; + kind: 'baseline' | 'config'; + versions: Version[]; + currentVersionId: number | null; + runningBetaId: number | null; + scopeType: string | null; + scopeValue: string | null; + onsaved: () => Promise; } = $props(); let selected = $state(''); let text = $state(''); @@ -24,132 +43,406 @@ let loadedVersion = $state(''); let recovered = $state<{ version: string; text: string; notes: string } | null>(null); const dirty = $derived(text !== original || notes.trim().length > 0); - const ignoredWarnings = $derived(configWarnings(text, kind === 'config').filter(warning => warning.includes('ignored') || warning.includes('no value'))); + const ignoredWarnings = $derived( + configWarnings(text, kind === 'config').filter( + (warning) => warning.includes('ignored') || warning.includes('no value') + ) + ); const storageKey = $derived(`electric-cherry.config-draft.${id}`); onMount(() => { - selected = String(versions[0]?.id ?? ''); ready = true; + selected = String(versions[0]?.id ?? ''); + ready = true; try { const saved = JSON.parse(localStorage.getItem(storageKey) ?? 'null'); - if (saved && typeof saved.text === 'string' && typeof saved.version === 'string' && typeof saved.notes === 'string') recovered = saved; - } catch { /* Draft storage is optional. */ } + if ( + saved && + typeof saved.text === 'string' && + typeof saved.version === 'string' && + typeof saved.notes === 'string' + ) + recovered = saved; + } catch { + /* Draft storage is optional. */ + } }); let allowNavigation = false; beforeNavigate(({ cancel, to, type }) => { if (!dirty || allowNavigation || type === 'leave' || !to) return; cancel(); - void confirmAction({title:'Leave this editor?', message:'You have unsaved changes. A draft is kept in this browser when storage is available.', confirmLabel:'Leave editor'}).then(async accepted => { + void confirmAction({ + title: 'Leave this editor?', + message: + 'You have unsaved changes. A draft is kept in this browser when storage is available.', + confirmLabel: 'Leave editor' + }).then(async (accepted) => { if (!accepted) return; allowNavigation = true; - try { await goto(to.url); } finally { allowNavigation = false; } + try { + await goto(to.url); + } finally { + allowNavigation = false; + } }); }); - function beforeUnload(event: BeforeUnloadEvent) { if (dirty) { event.preventDefault(); event.returnValue = ''; } } + function beforeUnload(event: BeforeUnloadEvent) { + if (dirty) { + event.preventDefault(); + event.returnValue = ''; + } + } $effect(() => { const version = selected; - const refresh = retry; + void retry; if (!ready || !version) return; const controller = new AbortController(); - loading = true; error = ''; loadedVersion = ''; + loading = true; + error = ''; + loadedVersion = ''; (async () => { try { - const response = await fetch(`/api/configs/${id}?version=${version}`, { signal: controller.signal }); + const response = await fetch(`/api/configs/${id}?version=${version}`, { + signal: controller.signal + }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load config contents.'); - if (!controller.signal.aborted) { text = body.text; original = body.text; notes = ''; loadedVersion = version; } - } catch (err) { if (!controller.signal.aborted) error = (err as Error).message; } - finally { if (!controller.signal.aborted) loading = false; } + if (!controller.signal.aborted) { + text = body.text; + original = body.text; + notes = ''; + loadedVersion = version; + } + } catch (err) { + if (!controller.signal.aborted) error = (err as Error).message; + } finally { + if (!controller.signal.aborted) loading = false; + } })(); return () => controller.abort(); }); $effect(() => { if (!ready || loading || !loadedVersion || !dirty) return; - try { localStorage.setItem(storageKey, JSON.stringify({ version: loadedVersion, text, notes })); } catch { /* Editing still works without storage. */ } + try { + localStorage.setItem( + storageKey, + JSON.stringify({ version: loadedVersion, text, notes }) + ); + } catch { + /* Editing still works without storage. */ + } }); - function clearDraft() { try { localStorage.removeItem(storageKey); } catch { /* Storage may be disabled. */ } recovered = null; } + function clearDraft() { + try { + localStorage.removeItem(storageKey); + } catch { + /* Storage may be disabled. */ + } + recovered = null; + } async function changeVersion(event: Event) { const target = event.currentTarget as HTMLSelectElement; - const next = target.value; target.value = selected; - if (dirty && !await confirmAction({title:'Discard current edits?',message:'Open the selected version and discard your current changes.',confirmLabel:'Discard edits',danger:true})) return; - clearDraft(); selected = next; success = ''; + const next = target.value; + target.value = selected; + if ( + dirty && + !(await confirmAction({ + title: 'Discard current edits?', + message: 'Open the selected version and discard your current changes.', + confirmLabel: 'Discard edits', + danger: true + })) + ) + return; + clearDraft(); + selected = next; + success = ''; } function restoreDraft() { if (!recovered) return; // Keep the current version as the comparison base, including when the // recovered draft was started from a version that has since been deleted. - text = recovered.text; notes = recovered.notes; recovered = null; + text = recovered.text; + notes = recovered.notes; + recovered = null; + } + async function discard() { + if ( + await confirmAction({ + title: 'Discard unsaved changes?', + message: 'Restore the version you opened in this editor.', + confirmLabel: 'Discard changes', + danger: true + }) + ) { + text = original; + notes = ''; + clearDraft(); + } } - async function discard() { if (await confirmAction({title:'Discard unsaved changes?',message:'Restore the version you opened in this editor.',confirmLabel:'Discard changes',danger:true})) { text = original; notes = ''; clearDraft(); } } async function importFile(event: Event) { const input = event.currentTarget as HTMLInputElement; const file = input.files?.[0]; - if (file) { try { text = await file.text(); } catch { error = 'Could not read that file.'; } } + if (file) { + try { + text = await file.text(); + } catch { + error = 'Could not read that file.'; + } + } input.value = ''; } async function save() { if (saving || loading || runningBetaId !== null || !text.trim()) return; - saving = true; error = ''; success = ''; + saving = true; + error = ''; + success = ''; const form = new FormData(); form.append('content', new Blob([text], { type: 'text/plain' }), 'config.cfg'); if (notes.trim()) form.append('notes', notes.trim()); try { - const response = await fetch(`/api/scripts/${id}/versions`, { method: 'POST', body: form }); + const response = await fetch(`/api/scripts/${id}/versions`, { + method: 'POST', + body: form + }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not save this version.'); - original = text; notes = ''; clearDraft(); + original = text; + notes = ''; + clearDraft(); success = `v${body.version.version} saved as untested. Open a beta to test and promote it.`; - await onsaved(); selected = String(body.version.id); - } catch (err) { error = (err as Error).message; } - finally { saving = false; } + await onsaved(); + selected = String(body.version.id); + } catch (err) { + error = (err as Error).message; + } finally { + saving = false; + } }
-
Edit · preview · version

Configuration editor

+
+
+ Edit · preview · version +

Configuration editor

+
{#if dirty}Unsaved draft{/if}
-
- +
+ +
- {#if recovered && !loading}
A previous draft is available in this browser.
{/if} - {#if success}
{success} Test this version →
{/if} - {#if error}{/if} + {#if recovered && !loading}
+ A previous draft is available in this browser. +
{/if} + {#if success}
+ {success} + Test this version → +
{/if} + {#if error}{/if} {#if loading}

Loading config contents…

{:else if loadedVersion} - - -
{runningBetaId !== null ? 'Finish the active beta to save another version.' : 'A new version must pass through a beta before promotion.'}{#if ignoredWarnings.length}{ignoredWarnings.length} ignored or incomplete lines — review the open configuration notes before saving.{/if} - - + + +
+ {runningBetaId !== null + ? 'Finish the active beta to save another version.' + : 'A new version must pass through a beta before promotion.'}{#if ignoredWarnings.length}{ignoredWarnings.length} ignored or incomplete lines — review the open configuration + notes before saving.{/if} + +
{/if}
- {#if loadedVersion && !loading}{/if} + {#if loadedVersion && !loading}{/if}
diff --git a/src/lib/ui/components/ConfigInspector.svelte b/src/lib/ui/components/ConfigInspector.svelte index 2923687..2ab4915 100644 --- a/src/lib/ui/components/ConfigInspector.svelte +++ b/src/lib/ui/components/ConfigInspector.svelte @@ -4,7 +4,19 @@ import type { ConfigInspection, ConfigServer, ConfigSource } from '$lib/config-workbench.js'; import Button from './primitives/Button.svelte'; import { saveLibraryContext } from '$ui/library-context.js'; - let { artifactId, draftText, scopeType, scopeValue, fixedServer }: { fixedServer?: string; artifactId?: number; draftText?: string; scopeType?: string | null; scopeValue?: string | null } = $props(); + let { + artifactId, + draftText, + scopeType, + scopeValue, + fixedServer + }: { + fixedServer?: string; + artifactId?: number; + draftText?: string; + scopeType?: string | null; + scopeValue?: string | null; + } = $props(); let servers = $state([]); let selected = $state(''); let inspection = $state(null); @@ -19,143 +31,587 @@ let reload = $state(0); let ready = $state(false); let desiredServer = ''; - const eligible = $derived(servers.filter(s => (!fixedServer || s.name === fixedServer) && (!artifactId || (scopeType === 'game' ? s.game === scopeValue : scopeType === 'map' ? s.map === scopeValue : false)))); - const settings = $derived((inspection?.settings ?? []).filter(s => (!onlyOverrides || s.history.length > 1) && `${s.key} ${s.value} ${s.source.name}`.toLowerCase().includes(query.toLowerCase()))); - const overlaps = $derived(inspection?.settings.filter(s => s.history.length > 1).length ?? 0); - const previousValues = $derived(new Map((promoted?.settings ?? []).map(s => [s.key, s.value]))); - const changes = $derived(promoted && inspection ? inspection.settings.filter(s => previousValues.get(s.key) !== s.value) : []); - const removed = $derived(promoted?.settings.filter(s => inspection && !inspection.settings.some(current => current.key === s.key)) ?? []); - function label(source: ConfigSource): string { return `${source.name}${source.draft ? ' · draft' : source.version ? ` · v${source.version}` : ''}`; } - function overriddenCount(source: ConfigSource) { return inspection?.settings.filter(s => (s.source.id !== source.id || s.source.kind !== source.kind) && s.history.some(h => h.source.id === source.id && h.source.kind === source.kind)).length ?? 0; } + const eligible = $derived( + servers.filter( + (s) => + (!fixedServer || s.name === fixedServer) && + (!artifactId || + (scopeType === 'game' + ? s.game === scopeValue + : scopeType === 'map' + ? s.map === scopeValue + : false)) + ) + ); + const settings = $derived( + (inspection?.settings ?? []).filter( + (s) => + (!onlyOverrides || s.history.length > 1) && + `${s.key} ${s.value} ${s.source.name}`.toLowerCase().includes(query.toLowerCase()) + ) + ); + const overlaps = $derived(inspection?.settings.filter((s) => s.history.length > 1).length ?? 0); + const previousValues = $derived( + new Map((promoted?.settings ?? []).map((s) => [s.key, s.value])) + ); + const changes = $derived( + promoted && inspection + ? inspection.settings.filter((s) => previousValues.get(s.key) !== s.value) + : [] + ); + const removed = $derived( + promoted?.settings.filter( + (s) => inspection && !inspection.settings.some((current) => current.key === s.key) + ) ?? [] + ); + function label(source: ConfigSource): string { + return `${source.name}${source.draft ? ' · draft' : source.version ? ` · v${source.version}` : ''}`; + } + function overriddenCount(source: ConfigSource) { + return ( + inspection?.settings.filter( + (s) => + (s.source.id !== source.id || s.source.kind !== source.kind) && + s.history.some( + (h) => h.source.id === source.id && h.source.kind === source.kind + ) + ).length ?? 0 + ); + } async function loadServers() { - loading = true; error = ''; + loading = true; + error = ''; try { const response = await fetch('/api/configs/inspect'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load servers.'); servers = body.servers; ready = true; - } catch (err) { error = (err as Error).message; } - finally { loading = false; } + } catch (err) { + error = (err as Error).message; + } finally { + loading = false; + } } - onMount(() => { desiredServer = new URLSearchParams(window.location.search).get('server') ?? ''; void loadServers(); }); - $effect(() => { if (!eligible.some(s => s.name === selected)) selected = eligible.find(s => s.name === desiredServer)?.name ?? eligible[0]?.name ?? ''; }); - $effect(() => { if (ready && selected && !fixedServer) saveLibraryContext({ server: selected }); }); + onMount(() => { + desiredServer = new URLSearchParams(window.location.search).get('server') ?? ''; + void loadServers(); + }); + $effect(() => { + if (!eligible.some((s) => s.name === selected)) + selected = + eligible.find((s) => s.name === desiredServer)?.name ?? eligible[0]?.name ?? ''; + }); + $effect(() => { + if (ready && selected && !fixedServer) saveLibraryContext({ server: selected }); + }); $effect(() => { const server = selected; const text = draftText; const id = artifactId; - const refresh = reload; - if (!ready || !server) { inspection = null; if (ready) loading = false; return; } + void reload; + if (!ready || !server) { + inspection = null; + if (ready) loading = false; + return; + } const controller = new AbortController(); - loading = true; error = ''; - if (inspectedServer !== server) { inspection = null; promoted = null; expanded = []; inspectedServer = server; } - const timer = setTimeout(async () => { - try { - const response = await fetch(id !== undefined && text !== undefined ? '/api/configs/inspect' : `/api/configs/inspect?server=${encodeURIComponent(server)}`, { - signal: controller.signal, - ...(id !== undefined && text !== undefined ? { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, artifactId: id, text }) } : {}) - }); - const body = await response.json(); - if (!response.ok) throw new Error(body.error ?? 'Could not inspect this config.'); - if (!controller.signal.aborted) { inspection = body.inspection; promoted = body.promoted ?? null; } - } catch (err) { if (!controller.signal.aborted) error = (err as Error).message; } - finally { if (!controller.signal.aborted) loading = false; } - }, text === undefined ? 0 : 350); - return () => { clearTimeout(timer); controller.abort(); }; + loading = true; + error = ''; + if (inspectedServer !== server) { + inspection = null; + promoted = null; + expanded = []; + inspectedServer = server; + } + const timer = setTimeout( + async () => { + try { + const response = await fetch( + id !== undefined && text !== undefined + ? '/api/configs/inspect' + : `/api/configs/inspect?server=${encodeURIComponent(server)}`, + { + signal: controller.signal, + ...(id !== undefined && text !== undefined + ? { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ server, artifactId: id, text }) + } + : {}) + } + ); + const body = await response.json(); + if (!response.ok) + throw new Error(body.error ?? 'Could not inspect this config.'); + if (!controller.signal.aborted) { + inspection = body.inspection; + promoted = body.promoted ?? null; + } + } catch (err) { + if (!controller.signal.aborted) error = (err as Error).message; + } finally { + if (!controller.signal.aborted) loading = false; + } + }, + text === undefined ? 0 : 350 + ); + return () => { + clearTimeout(timer); + controller.abort(); + }; }); function download() { if (!inspection) return; const url = URL.createObjectURL(new Blob([inspection.text], { type: 'text/plain' })); - const anchor = document.createElement('a'); anchor.href = url; anchor.download = `${selected}${artifactId ? '-draft' : ''}.cfg`; anchor.click(); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = `${selected}${artifactId ? '-draft' : ''}.cfg`; + anchor.click(); setTimeout(() => URL.revokeObjectURL(url), 1000); }
-
{#if !fixedServer}{artifactId ? 'Draft preview' : 'Follow every value to its source'}{/if}

{artifactId ? 'On this server' : 'Promoted configuration'}

- +
+
+ {#if !fixedServer}{artifactId ? 'Draft preview' : 'Follow every value to its source'}{/if} +

{artifactId ? 'On this server' : 'Promoted configuration'}

+
+
-

{artifactId ? 'Your draft composed with the other promoted configs. Saving creates an untested version.' : 'The intended file from promoted configs and the selected map. Live files and active betas may differ.'}

- {#if !fixedServer}{/if} - {#if error}{/if} - {#if !fixedServer || loading}

{loading ? inspection ? 'Updating preview… Showing the last completed result.' : 'Composing configuration…' : inspection ? 'Preview up to date.' : ''}

{/if} - {#if !eligible.length && !loading}

{artifactId ? 'No registered servers match this scope.' : 'Register a server to inspect its configuration.'}

- {:else if inspection?.reason}

{fixedServer && inspection.reason.startsWith('No promoted baseline') ? `No promoted baseline for ${servers.find(item => item.name === fixedServer)?.game === 't6' ? 'Black Ops II' : servers.find(item => item.name === fixedServer)?.game === 't5' ? 'Black Ops I' : 'this game'}.` : inspection.reason}

{#if fixedServer}Open config library →{/if} +

+ {artifactId + ? 'Your draft composed with the other promoted configs. Saving creates an untested version.' + : 'The intended file from promoted configs and the selected map. Live files and active betas may differ.'} +

+ {#if !fixedServer}{/if} + {#if error}{/if} + {#if !fixedServer || loading}

+ {loading + ? inspection + ? 'Updating preview… Showing the last completed result.' + : 'Composing configuration…' + : inspection + ? 'Preview up to date.' + : ''} +

{/if} + {#if !eligible.length && !loading}

+ {artifactId + ? 'No registered servers match this scope.' + : 'Register a server to inspect its configuration.'} +

+ {:else if inspection?.reason}

+ {fixedServer && inspection.reason.startsWith('No promoted baseline') + ? `No promoted baseline for ${servers.find((item) => item.name === fixedServer)?.game === 't6' ? 'Black Ops II' : servers.find((item) => item.name === fixedServer)?.game === 't5' ? 'Black Ops I' : 'this game'}.` + : inspection.reason} +

+ {#if fixedServer}Open config library →{/if} {:else if inspection}
    {#each inspection.sources as source, index} -
  1. {index + 1}.{#if source.id}{label(source)} - {:else}{source.name}{/if}{overriddenCount(source)} settings superseded later
    2. +
  2. + {index + 1}.{#if source.id}{label(source)} + {:else}{source.name}{/if}{overriddenCount(source)} settings superseded later +
    3. {/each}
-

Overlays apply in creation order; later overlays win for the same key. The selected map takes precedence over map rotation.

- {#if promoted}

Compared with promoted settings: {changes.length} changed or added, {removed.length} removed.

{#if removed.length}
Removed settings
    {#each removed as setting}
  • {setting.key} · previously {setting.value}
    • {/each}
{/if}{/if} -
- {#if raw} +

+ Overlays apply in creation order; later overlays win for the same key. The selected map + takes precedence over map rotation. +

+ {#if promoted}

+ Compared with promoted settings: {changes.length} changed or added, {removed.length} removed. +

+ {#if removed.length}
+ Removed settings +
    + {#each removed as setting}
  • + {setting.key} · previously {setting.value} +
    • {/each} +
+
{/if}{/if} +
+
+ +
+ +
+ {#if raw} {:else} -
+
+ +
{#each settings as setting (setting.key)} -
{ const open = event.currentTarget.open; expanded = open ? [...new Set([...expanded, setting.key])] : expanded.filter(key => key !== setting.key); }}>
{setting.key}{setting.value}
{label(setting.source)} {#if setting.history.length > 1}{setting.history.length - 1} overridden{/if}{#if promoted && previousValues.get(setting.key) !== setting.value}{previousValues.has(setting.key) ? `Changed from ${previousValues.get(setting.key)}` : 'Added by draft'}{/if} -
    {#each setting.history as entry, index}
  1. {label(entry.source)}{entry.source.kind === 'map' ? 'Automatic' : `Line ${entry.line}`} · {index === setting.history.length - 1 ? 'Final value' : 'Overridden'}
    {entry.value}
    2. {/each}
+
{ + const open = event.currentTarget.open; + expanded = open + ? [...new Set([...expanded, setting.key])] + : expanded.filter((key) => key !== setting.key); + }} + > +
+ {setting.key}{setting.value} +
+ {label(setting.source)} + {#if setting.history.length > 1}{setting.history.length - 1} overridden{/if}{#if promoted && previousValues.get(setting.key) !== setting.value}{previousValues.has(setting.key) + ? `Changed from ${previousValues.get(setting.key)}` + : 'Added by draft'}{/if} +
    + {#each setting.history as entry, index}
  1. +
    + {label(entry.source)}{entry.source.kind === 'map' + ? 'Automatic' + : `Line ${entry.line}`} · {index === + setting.history.length - 1 + ? 'Final value' + : 'Overridden'} +
    + {entry.value} +
    2. {/each} +
{:else}

No matching settings.

{/each}
-

Shows set / seta assignments and map rotation. Other commands remain visible in the composed file.

+

+ Shows set / seta assignments and map rotation. Other commands remain visible in the + composed file. +

{/if} - {#if inspection.warnings.length}
{inspection.warnings.length} configuration {inspection.warnings.length === 1 ? 'note' : 'notes'}
    {#each inspection.warnings as warning}
  • {warning}
    • {/each}
{/if} + {#if inspection.warnings.length}
+ {inspection.warnings.length} configuration {inspection.warnings.length === 1 + ? 'note' + : 'notes'} +
    + {#each inspection.warnings as warning}
  • {warning}
    • {/each} +
+
{/if} {/if}
diff --git a/src/lib/ui/components/ConfigLibrary.svelte b/src/lib/ui/components/ConfigLibrary.svelte index ec1c823..1a04fc1 100644 --- a/src/lib/ui/components/ConfigLibrary.svelte +++ b/src/lib/ui/components/ConfigLibrary.svelte @@ -16,14 +16,30 @@ import { libraryLimit, saveLibraryContext } from '$ui/library-context.js'; import { gameTitle } from '$ui/server-labels.js'; - interface Version { id: number; version: number; state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string; } + interface Version { + id: number; + version: number; + state: 'untested' | 'tested' | 'test-failed'; + uploaded_at: string; + } interface Artifact { - id: number; name: string; kind: 'baseline' | 'config'; scope_type: 'game' | 'map' | null; - scope_value: string | null; current_version_id: number | null; currentVersion: Version | null; - latestVersion: Version | null; versionCount: number; servers: string[]; runningBetaId: number | null; + id: number; + name: string; + kind: 'baseline' | 'config'; + scope_type: 'game' | 'map' | null; + scope_value: string | null; + current_version_id: number | null; + currentVersion: Version | null; + latestVersion: Version | null; + versionCount: number; + servers: string[]; + runningBetaId: number | null; } let artifacts = $state([]); - let scopeOptions = $state<{ games: string[]; mapsByGame: Record }>({ games: [], mapsByGame: {} }); + let scopeOptions = $state<{ games: string[]; mapsByGame: Record }>({ + games: [], + mapsByGame: {} + }); let loading = $state(true); let error = $state(''); let query = $state(''); @@ -33,10 +49,23 @@ let view = $state('library'); let limit = $state(25); let mounted = $state(false); - afterNavigate(() => { mounted = true; }); + afterNavigate(() => { + mounted = true; + }); let lastFilters = ''; - $effect(() => { const key = JSON.stringify([query, game, scope, status]); if (!mounted) return; if (lastFilters && lastFilters !== key) limit = 25; lastFilters = key; }); - $effect(() => { if (mounted) saveLibraryContext({ q: query, game, scope, status, view, limit }, { view: 'library', limit: 25 }); }); + $effect(() => { + const key = JSON.stringify([query, game, scope, status]); + if (!mounted) return; + if (lastFilters && lastFilters !== key) limit = 25; + lastFilters = key; + }); + $effect(() => { + if (mounted) + saveLibraryContext( + { q: query, game, scope, status, view, limit }, + { view: 'library', limit: 25 } + ); + }); let dialog: HTMLDialogElement; let kind = $state<'baseline' | 'config'>('config'); let name = $state(''); @@ -47,161 +76,585 @@ let notes = $state(''); let saving = $state(false); let createError = $state(''); - const baselines = $derived(artifacts.filter(a => a.kind === 'baseline')); - const gameOptions = $derived([...new Set([...scopeOptions.games, ...artifacts.filter(a => a.scope_type === 'game').map(a => a.scope_value!).filter(Boolean)])].sort()); - const baselineExists = $derived(kind === 'baseline' && baselines.some(a => a.scope_type === 'game' && a.scope_value === scopeGame)); - const selectedScope = $derived(kind === 'baseline' || scopeType === 'game' ? scopeGame : scopeMap); - const filtered = $derived(artifacts.filter(a => { - if (query && !`${a.name} ${a.scope_value ?? ''} ${a.servers.join(' ')}`.toLowerCase().includes(query.toLowerCase())) return false; - if (game && !(a.scope_type === 'game' ? a.scope_value === game : (scopeOptions.mapsByGame[game] ?? []).includes(a.scope_value ?? ''))) return false; - if (scope && a.scope_type !== scope) return false; - if (status === 'pending' && (!a.latestVersion || a.latestVersion.id === a.current_version_id)) return false; - if (status === 'promoted' && !a.current_version_id) return false; - if (status === 'beta' && !a.runningBetaId) return false; - if (status === 'failed' && a.latestVersion?.state !== 'test-failed') return false; - return true; - })); - const filteredBaselines = $derived(filtered.filter(a => a.kind === 'baseline').sort((a, b) => (a.scope_value ?? '').localeCompare(b.scope_value ?? ''))); - const filteredOverlays = $derived(filtered.filter(a => a.kind === 'config')); + const baselines = $derived(artifacts.filter((a) => a.kind === 'baseline')); + const gameOptions = $derived( + [ + ...new Set([ + ...scopeOptions.games, + ...artifacts + .filter((a) => a.scope_type === 'game') + .map((a) => a.scope_value!) + .filter(Boolean) + ]) + ].sort() + ); + const baselineExists = $derived( + kind === 'baseline' && + baselines.some((a) => a.scope_type === 'game' && a.scope_value === scopeGame) + ); + const selectedScope = $derived( + kind === 'baseline' || scopeType === 'game' ? scopeGame : scopeMap + ); + const filtered = $derived( + artifacts.filter((a) => { + if ( + query && + !`${a.name} ${a.scope_value ?? ''} ${a.servers.join(' ')}` + .toLowerCase() + .includes(query.toLowerCase()) + ) + return false; + if ( + game && + !(a.scope_type === 'game' + ? a.scope_value === game + : (scopeOptions.mapsByGame[game] ?? []).includes(a.scope_value ?? '')) + ) + return false; + if (scope && a.scope_type !== scope) return false; + if ( + status === 'pending' && + (!a.latestVersion || a.latestVersion.id === a.current_version_id) + ) + return false; + if (status === 'promoted' && !a.current_version_id) return false; + if (status === 'beta' && !a.runningBetaId) return false; + if (status === 'failed' && a.latestVersion?.state !== 'test-failed') return false; + return true; + }) + ); + const filteredBaselines = $derived( + filtered + .filter((a) => a.kind === 'baseline') + .sort((a, b) => (a.scope_value ?? '').localeCompare(b.scope_value ?? '')) + ); + const filteredOverlays = $derived(filtered.filter((a) => a.kind === 'config')); async function load() { - loading = true; error = ''; + loading = true; + error = ''; try { - const responses = await Promise.all([fetch('/api/configs?kind=baseline'), fetch('/api/configs?kind=config'), fetch('/api/scope-options')]); - if (responses.some(r => !r.ok)) throw new Error('Could not load the config library. Please try again.'); - const [base, overlay, scopes] = await Promise.all(responses.map(r => r.json())); - artifacts = [...base.scripts, ...overlay.scripts]; scopeOptions = scopes; - } catch (err) { error = (err as Error).message; } - finally { loading = false; } + const responses = await Promise.all([ + fetch('/api/configs?kind=baseline'), + fetch('/api/configs?kind=config'), + fetch('/api/scope-options') + ]); + if (responses.some((r) => !r.ok)) + throw new Error('Could not load the config library. Please try again.'); + const [base, overlay, scopes] = await Promise.all(responses.map((r) => r.json())); + artifacts = [...base.scripts, ...overlay.scripts]; + scopeOptions = scopes; + } catch (err) { + error = (err as Error).message; + } finally { + loading = false; + } + } + onMount(() => { + const params = new URLSearchParams(window.location.search); + query = params.get('q') ?? ''; + game = params.get('game') ?? ''; + scope = params.get('scope') ?? ''; + status = params.get('status') ?? ''; + view = params.get('view') === 'inspect' ? 'inspect' : 'library'; + limit = libraryLimit(params); + void load(); + }); + function openCreate(type: 'config' | 'baseline' = 'config') { + kind = type; + createError = ''; + dialog.showModal(); + } + function resetFilters() { + query = ''; + game = ''; + scope = ''; + status = ''; } - onMount(() => { const params = new URLSearchParams(window.location.search); query = params.get('q') ?? ''; game = params.get('game') ?? ''; scope = params.get('scope') ?? ''; status = params.get('status') ?? ''; view = params.get('view') === 'inspect' ? 'inspect' : 'library'; limit = libraryLimit(params); void load(); }); - function openCreate(type: 'config' | 'baseline' = 'config') { kind = type; createError = ''; dialog.showModal(); } - function resetFilters() { query = ''; game = ''; scope = ''; status = ''; } async function importFile(event: Event) { const input = event.currentTarget as HTMLInputElement; const file = input.files?.[0]; if (file) { - try { text = await file.text(); if (!name) name = file.name.replace(/\.[^.]+$/, ''); } - catch { createError = 'Could not read that file.'; } + try { + text = await file.text(); + if (!name) name = file.name.replace(/\.[^.]+$/, ''); + } catch { + createError = 'Could not read that file.'; + } } input.value = ''; } async function create(event: SubmitEvent) { event.preventDefault(); if (saving || baselineExists || !name.trim() || !selectedScope || !text.trim()) return; - if (kind === 'config' && !/^\s*set\s+\S+\s+/m.test(text)) { createError = 'Add at least one set KEY VALUE line.'; return; } - saving = true; createError = ''; + if (kind === 'config' && !/^\s*set\s+\S+\s+/m.test(text)) { + createError = 'Add at least one set KEY VALUE line.'; + return; + } + saving = true; + createError = ''; const form = new FormData(); - form.append('name', name.trim()); form.append('kind', kind); - form.append('scopeType', kind === 'baseline' ? 'game' : scopeType); form.append('scopeValue', selectedScope); - form.append('text', text); if (notes.trim()) form.append('notes', notes.trim()); + form.append('name', name.trim()); + form.append('kind', kind); + form.append('scopeType', kind === 'baseline' ? 'game' : scopeType); + form.append('scopeValue', selectedScope); + form.append('text', text); + if (notes.trim()) form.append('notes', notes.trim()); try { const response = await fetch('/api/configs', { method: 'POST', body: form }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not create this config.'); - dialog.close(); await goto(`/configs/${body.script.id}`); - } catch (err) { createError = (err as Error).message; } - finally { saving = false; } + dialog.close(); + await goto(`/configs/${body.script.id}`); + } catch (err) { + createError = (err as Error).message; + } finally { + saving = false; + } }
- {#snippet actions()}{#if view === 'inspect' || loading || artifacts.length || error}{/if}{/snippet} + {#snippet actions()}{#if view === 'inspect' || loading || artifacts.length || error}{/if}{/snippet} -
-
- {#if error}{/if} - {#if view === 'inspect'} - - {:else} - {#if loading || artifacts.length || error}
- - - - - {#if filtered.length && (query || game || scope || status)}{/if} -
{/if} - {#if loading}
Loading your config library…
- {:else if !artifacts.length && !error} - - {#snippet actions()}{/snippet} - - {:else if !filtered.length && !error} - {#snippet actions()}{/snippet} - +
+ +
+
+ {#if error}{/if} + {#if view === 'inspect'} + {:else} - {#if filteredBaselines.length} -

Baselines {filteredBaselines.length}

{@render configTable(filteredBaselines)}
- {/if} - {#if filteredOverlays.length} -

Overlays {filteredOverlays.length}

{@render configTable(filteredOverlays.slice(0,limit))}
+ {#if loading || artifacts.length || error}
+ + + + + {#if filtered.length && (query || game || scope || status)}{/if} +
{/if} + {#if loading}
Loading your config library…
+ {:else if !artifacts.length && !error} + + {#snippet actions()}{/snippet} + + {:else if !filtered.length && !error} + {#snippet actions()}{/snippet} + + {:else} + {#if filteredBaselines.length} +
+

Baselines {filteredBaselines.length}

+ {@render configTable(filteredBaselines)} +
+ {/if} + {#if filteredOverlays.length} +
+

Overlays {filteredOverlays.length}

+ {@render configTable(filteredOverlays.slice(0, limit))} +
+ {/if} {/if} + {/if} - - {/if}
{#snippet configTable(rows: Artifact[])} -
- - {#each rows as artifact (artifact.id)} - - - - - {/each} -
ConfigScopeVersion / statusEligible servers
{artifact.name}{artifact.scope_type === 'game' ? `All ${gameTitle(artifact.scope_value)} servers` : artifact.scope_type === 'map' ? mapLabel(artifact.scope_value ?? '') : 'Unscoped'}{artifact.servers.length}
+
+ + + {#each rows as artifact (artifact.id)} + + + + + {/each} +
ConfigScopeVersion / statusEligible servers
{artifact.name}{artifact.scope_type === 'game' + ? `All ${gameTitle(artifact.scope_value)} servers` + : artifact.scope_type === 'map' + ? mapLabel(artifact.scope_value ?? '') + : 'Unscoped'}{artifact.servers.length}
+
{/snippet} - { if (saving) event.preventDefault(); }}> + { + if (saving) event.preventDefault(); + }} +>
-
Create version 1

New config

-
-
- {#if kind === 'config'}{#if scopeType === 'map'}{/if}{/if} +
+
+ Create version 1 +

New config

+
+
- {#if baselineExists}

This game already has a baseline. Open it from the library to save a new version.

{/if} - {#if !gameOptions.length}

Register a server first to make its game and map available here.

{/if} - +
+ +
+
+ + {#if kind === 'config'}{#if scopeType === 'map'}{/if}{/if} +
+ {#if baselineExists}

+ This game already has a baseline. Open it from the library to save a new version. +

{/if} + {#if !gameOptions.length}

+ Register a server first to make its game and map available here. +

{/if} + - + {#if createError}{/if} -

Saved as untested. Test in a beta before promotion.

+
+

Saved as untested. Test in a beta before promotion.

+ +
diff --git a/src/lib/ui/components/ConfigTextEditor.svelte b/src/lib/ui/components/ConfigTextEditor.svelte index dafae18..b5f22a7 100644 --- a/src/lib/ui/components/ConfigTextEditor.svelte +++ b/src/lib/ui/components/ConfigTextEditor.svelte @@ -1,74 +1,274 @@
- - + +
{text.split(/\r?\n/).length} lines
{#if mode === 'raw'} - + {:else}
- -

Edit values and inline comments. Other commands and formatting stay in the raw file.

+ +

+ Edit values and inline comments. Other commands and formatting stay in the raw file. +

{#each visible as entry (entry.index)} -
{/if}
{#if warnings.length} -
w.includes('ignored') || w.includes('no value'))}>{warnings.length} configuration {warnings.length === 1 ? 'note' : 'notes'} — review before saving -
    {#each warnings as warning}
  • {warning}
    • {/each}
+
w.includes('ignored') || w.includes('no value'))} + > + {warnings.length} configuration {warnings.length === 1 ? 'note' : 'notes'} — review before + saving +
    + {#each warnings as warning}
  • {warning}
    • {/each} +
{/if} diff --git a/src/lib/ui/components/Confirmation.svelte b/src/lib/ui/components/Confirmation.svelte index 0f31913..5f16474 100644 --- a/src/lib/ui/components/Confirmation.svelte +++ b/src/lib/ui/components/Confirmation.svelte @@ -4,11 +4,31 @@ import Button from './primitives/Button.svelte'; - $confirmation !== null, (open) => { if (!open) resolveConfirmation(false); }} title={$confirmation?.title ?? 'Confirm action'}> + $confirmation !== null, + (open) => { + if (!open) resolveConfirmation(false); + } + } + title={$confirmation?.title ?? 'Confirm action'} +>

{$confirmation?.message}

{#snippet actions()} - + {/snippet} - + + diff --git a/src/lib/ui/components/CopyValue.svelte b/src/lib/ui/components/CopyValue.svelte index b2b0457..2ed0e0f 100644 --- a/src/lib/ui/components/CopyValue.svelte +++ b/src/lib/ui/components/CopyValue.svelte @@ -2,12 +2,41 @@ import Button from './primitives/Button.svelte'; let { value, label = 'ID' }: { value: string; label?: string } = $props(); let copyStatus = $state(''); - $effect(() => { value; copyStatus = ''; }); + $effect(() => { + void value; + copyStatus = ''; + }); async function copy() { copyStatus = ''; - try { await navigator.clipboard.writeText(value); copyStatus = `${label} copied.`; } - catch { copyStatus = 'Copy unavailable. Select and copy the text directly.'; } + try { + await navigator.clipboard.writeText(value); + copyStatus = `${label} copied.`; + } catch { + copyStatus = 'Copy unavailable. Select and copy the text directly.'; + } } -
{value}{#if copyStatus}{copyStatus}{/if}
- + +
+ {value}{#if copyStatus}{copyStatus}{/if} +
+ + diff --git a/src/lib/ui/components/DeleteHostDialog.svelte b/src/lib/ui/components/DeleteHostDialog.svelte index 496d839..3858603 100644 --- a/src/lib/ui/components/DeleteHostDialog.svelte +++ b/src/lib/ui/components/DeleteHostDialog.svelte @@ -3,20 +3,35 @@ import Button from './primitives/Button.svelte'; import Dialog from './primitives/Dialog.svelte'; - let { open = $bindable(false), host, ondeleted }: - { open?: boolean; host: Pick | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise } = $props(); + let { + open = $bindable(false), + host, + ondeleted + }: { + open?: boolean; + host: Pick | null; + ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise; + } = $props(); let deletion = $state(null); - let checking = $state(false); let deleting = $state(false); let error = $state(''); + let checking = $state(false); + let deleting = $state(false); + let error = $state(''); async function inspect(id: string, signal?: AbortSignal) { - checking = true; deletion = null; error = ''; + checking = true; + deletion = null; + error = ''; try { const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { signal }); const body = await response.json(); - if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not check this host.'); + if (!response.ok) + throw new Error(body.error ?? body.message ?? 'Could not check this host.'); if (!signal?.aborted) deletion = body.deletion; - } catch (cause) { if (!signal?.aborted) error = (cause as Error).message; } - finally { if (!signal?.aborted) checking = false; } + } catch (cause) { + if (!signal?.aborted) error = (cause as Error).message; + } finally { + if (!signal?.aborted) checking = false; + } } $effect(() => { if (!open || !host) return; @@ -27,9 +42,12 @@ async function remove() { if (!host || deleting || checking || !deletion?.allowed) return; const id = host.id; - deleting = true; error = ''; + deleting = true; + error = ''; try { - const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { method: 'DELETE' }); + const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { + method: 'DELETE' + }); const body = await response.json(); if (!response.ok) { await inspect(id); @@ -37,30 +55,95 @@ } open = false; await ondeleted(body); - } catch (cause) { error = (cause as Error).message; } - finally { deleting = false; } + } catch (cause) { + error = (cause as Error).message; + } finally { + deleting = false; + } } - +
-

This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.

-

The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.

+

+ This deletes the host's game-library copies, server files, agent data, credentials, and + service installation. Electric Cherry's uploaded libraries on the controller and its + operation history are kept. +

+

+ The host remains visible until cleanup is confirmed. Offline hosts wait for + reconnection. Unsent commands will be cancelled. +

{#if checking}

Checking servers and unfinished operations…

- {:else if deletion && !deletion.allowed}
This host still has work to finish:
    {#each deletion.blockers as blocker}
  • {blocker}
    • {/each}

Review open = false}>operation history or open open = false}>host details to drain its servers.

- {:else if deletion?.allowed}

Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.

{/if} + {:else if deletion && !deletion.allowed}
+ This host still has work to finish: +
    + {#each deletion.blockers as blocker}
  • {blocker}
    • {/each} +
+

+ Review (open = false)} + >operation history + or open + (open = false)}>host details to + drain its servers. +

+
+ {:else if deletion?.allowed}

+ Cleanup cannot be undone once it starts. Adding this machine again requires + installing and enrolling the agent again. +

{/if} {#if error}{/if}
{#snippet actions()} - - {#if !deletion?.allowed}{/if} - + + {#if !deletion?.allowed}{/if} + {/snippet} diff --git a/src/lib/ui/components/DeployCollectionDialog.svelte b/src/lib/ui/components/DeployCollectionDialog.svelte index e51cdc4..9a1efe8 100644 --- a/src/lib/ui/components/DeployCollectionDialog.svelte +++ b/src/lib/ui/components/DeployCollectionDialog.svelte @@ -4,8 +4,15 @@ import Dialog from './primitives/Dialog.svelte'; import Button from './primitives/Button.svelte'; import NativeHostSelection from './NativeHostSelection.svelte'; - import { createCollectionLaunchController, type CollectionLaunchState } from '$ui/collection-launch.js'; - import { collectionMapGroups, collectionKeyCoverage, collectionCapacitySummary } from '$ui/collection-summary.js'; + import { + createCollectionLaunchController, + type CollectionLaunchState + } from '$ui/collection-launch.js'; + import { + collectionMapGroups, + collectionKeyCoverage, + collectionCapacitySummary + } from '$ui/collection-summary.js'; import { serverTitle } from '$ui/server-labels.js'; import { timeFormat } from '$ui/stores/time-zone.js'; import type { NativeHostSelectionStatus } from '$ui/native-game-files.js'; @@ -13,7 +20,8 @@ import type { ZombieGame } from '$lib/map-aliases.js'; import type { BlueprintSummary, CollectionMode } from '$lib/deployment-plan.js'; - let { open = $bindable(false), blueprintId = '' }: { open?: boolean; blueprintId?: string } = $props(); + let { open = $bindable(false), blueprintId = '' }: { open?: boolean; blueprintId?: string } = + $props(); const id = $props.id(); let mode = $state('fill'); let modeInput = $state(); @@ -21,11 +29,23 @@ let collectionName = $state('Launch collection'); let selection = $state({ libraryIds: {}, image: '' }); let hostReady = $state(false); - let hostStatus = $state({ loading: true, error: '', hasHosts: false, filesReady: false, missingGames: [], uploadedAt: null }); + let hostStatus = $state({ + loading: true, + error: '', + hasHosts: false, + filesReady: false, + missingGames: [], + uploadedAt: null + }); let confirmRemovals = $state(false); - let view = $state({ preview: null, checking: false, error: '', changeNotice: '' }); + let view = $state({ + preview: null, + checking: false, + error: '', + changeNotice: '' + }); let settledId: string | undefined; - const controller = createCollectionLaunchController(state => { + const controller = createCollectionLaunchController((state) => { if (state.checking || state.preview?.id !== settledId) confirmRemovals = false; settledId = state.preview?.id; view = state; @@ -45,68 +65,153 @@ const preview = $derived(view.preview); const mapGroups = $derived(preview ? collectionMapGroups(preview) : []); const keyCoverage = $derived(preview ? collectionKeyCoverage(preview) : []); - const missingKeys = $derived(keyCoverage.filter(pool => pool.available < pool.required - pool.unassigned)); - const hasChanges = $derived(!!preview && (preview.servers.length > 0 || preview.removals.length > 0)); + const missingKeys = $derived( + keyCoverage.filter((pool) => pool.available < pool.required - pool.unassigned) + ); + const hasChanges = $derived( + !!preview && (preview.servers.length > 0 || preview.removals.length > 0) + ); const expired = $derived(!!preview && Date.parse(preview.expiresAt) <= now); - const checking = $derived((!loaded && !loadingError) || (loaded && (hostStatus.loading || view.checking))); + const checking = $derived( + (!loaded && !loadingError) || (loaded && (hostStatus.loading || view.checking)) + ); const imageValid = $derived(/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(selection.image)); - const currentReady = $derived(!!preview && preview.mode === mode && hostReady && !hostStatus.loading && !view.checking && !expired); + const currentReady = $derived( + !!preview && + preview.mode === mode && + hostReady && + !hostStatus.loading && + !view.checking && + !expired + ); const noChanges = $derived(currentReady && preview?.ready && !hasChanges); - const canLaunch = $derived(!launching && !!preview && (unconfirmedLaunch || (currentReady && preview.ready && hasChanges - && preview.servers.every(server => server.name && server.hasSavedKey) && (!preview.removals.length || confirmRemovals)))); - const footerStatus = $derived(launching ? 'Queueing changes…' : unconfirmedLaunch ? 'Retry to confirm this launch.' - : loadingError ? 'Could not load collection.' : checking ? 'Checking fleet changes…' : noChanges ? 'Collection targets met' - : missingKeys.length ? 'More game keys needed.' : currentReady && preview?.ready && preview.removals.length && !confirmRemovals - ? 'Confirm the listed removals.' : canLaunch ? 'Ready to queue' : 'Launch unavailable'); - const actionLabel = $derived(unconfirmedLaunch ? 'Retry launch' : preview?.removals.length - ? `Create ${preview.servers.length} · Remove ${preview.removals.length}` - : `Queue ${preview?.servers.length ?? 0} server${preview?.servers.length === 1 ? '' : 's'}`); + const canLaunch = $derived( + !launching && + !!preview && + (unconfirmedLaunch || + (currentReady && + preview.ready && + hasChanges && + preview.servers.every((server) => server.name && server.hasSavedKey) && + (!preview.removals.length || confirmRemovals))) + ); + const footerStatus = $derived( + launching + ? 'Queueing changes…' + : unconfirmedLaunch + ? 'Retry to confirm this launch.' + : loadingError + ? 'Could not load collection.' + : checking + ? 'Checking fleet changes…' + : noChanges + ? 'Collection targets met' + : missingKeys.length + ? 'More game keys needed.' + : currentReady && + preview?.ready && + preview.removals.length && + !confirmRemovals + ? 'Confirm the listed removals.' + : canLaunch + ? 'Ready to queue' + : 'Launch unavailable' + ); + const actionLabel = $derived( + unconfirmedLaunch + ? 'Retry launch' + : preview?.removals.length + ? `Create ${preview.servers.length} · Remove ${preview.removals.length}` + : `Queue ${preview?.servers.length ?? 0} server${preview?.servers.length === 1 ? '' : 's'}` + ); $effect(() => { if (open && (!wasOpen || activeBlueprint !== blueprintId)) { - wasOpen = true; activeBlueprint = blueprintId; void initialize(); + wasOpen = true; + activeBlueprint = blueprintId; + void initialize(); } else if (!open && wasOpen) { - wasOpen = false; visit++; initialization?.abort(); controller.clear(); + wasOpen = false; + visit++; + initialization?.abort(); + controller.clear(); } }); $effect(() => { - const input = open && loaded && hostReady && !hostStatus.loading ? { - blueprintId, mode, image: selection.image, - libraryIds: Object.fromEntries(games.map(game => [game, selection.libraryIds[game]])) - } : null; + const input = + open && loaded && hostReady && !hostStatus.loading + ? { + blueprintId, + mode, + image: selection.image, + libraryIds: Object.fromEntries( + games.map((game) => [game, selection.libraryIds[game]]) + ) + } + : null; if (!launching && !unconfirmedLaunch) controller.select(input); }); async function initialize() { const current = ++visit; - initialization?.abort(); initialization = new AbortController(); + initialization?.abort(); + initialization = new AbortController(); const signal = initialization.signal; - controller.clear(); loaded = false; loadingError = ''; launchError = ''; launching = false; unconfirmedLaunch = false; - mode = 'fill'; games = []; collectionName = 'Launch collection'; confirmRemovals = false; - selection = { libraryIds: {}, image: '' }; hostReady = false; - hostStatus = { loading: true, error: '', hasHosts: false, filesReady: false, missingGames: [], uploadedAt: null }; + controller.clear(); + loaded = false; + loadingError = ''; + launchError = ''; + launching = false; + unconfirmedLaunch = false; + mode = 'fill'; + games = []; + collectionName = 'Launch collection'; + confirmRemovals = false; + selection = { libraryIds: {}, image: '' }; + hostReady = false; + hostStatus = { + loading: true, + error: '', + hasHosts: false, + filesReady: false, + missingGames: [], + uploadedAt: null + }; try { const response = await fetch('/api/blueprints', { signal }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load collection.'); if (current !== visit) return; - const blueprint = (body.blueprints as BlueprintSummary[]).find(item => item.id === blueprintId); + const blueprint = (body.blueprints as BlueprintSummary[]).find( + (item) => item.id === blueprintId + ); if (!blueprint) throw new Error('This collection is no longer available.'); - games = [...new Set(blueprint.entries.map(entry => entry.game))]; collectionName = blueprint.name; + games = [...new Set(blueprint.entries.map((entry) => entry.game))]; + collectionName = blueprint.name; loaded = true; await tick(); if (current === visit && open) modeInput?.focus(); } catch (error) { - if (current === visit && !signal.aborted) loadingError = error instanceof Error ? error.message : 'Could not load collection.'; + if (current === visit && !signal.aborted) + loadingError = + error instanceof Error ? error.message : 'Could not load collection.'; } } - function selectionChanged() { launchError = ''; confirmRemovals = false; } + function selectionChanged() { + launchError = ''; + confirmRemovals = false; + } function refreshChecks() { if (!open || launching || unconfirmedLaunch) return; - if (!loaded) { void initialize(); return; } - launchError = ''; confirmRemovals = false; - controller.select(null); refreshVersion++; + if (!loaded) { + void initialize(); + return; + } + launchError = ''; + confirmRemovals = false; + controller.select(null); + refreshVersion++; } onMount(() => { let returning: ReturnType; @@ -116,42 +221,63 @@ }; window.addEventListener('focus', refreshOnReturn); document.addEventListener('visibilitychange', refreshOnReturn); - const timer = setInterval(() => { now = Date.now(); }, 1000); + const timer = setInterval(() => { + now = Date.now(); + }, 1000); return () => { - clearTimeout(returning); clearInterval(timer); + clearTimeout(returning); + clearInterval(timer); window.removeEventListener('focus', refreshOnReturn); document.removeEventListener('visibilitychange', refreshOnReturn); }; }); - onDestroy(() => { visit++; initialization?.abort(); submission?.abort(); controller.dispose(); }); + onDestroy(() => { + visit++; + initialization?.abort(); + submission?.abort(); + controller.dispose(); + }); async function launch() { if (!canLaunch || !preview) return; const current = visit; const reviewed = preview; - controller.suspend(); launching = true; launchError = ''; + controller.suspend(); + launching = true; + launchError = ''; submission = new AbortController(); try { - const response = await fetch('/api/deployment-collections', { method: 'POST', signal: submission.signal, - headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ previewId: reviewed.id }) }); + const response = await fetch('/api/deployment-collections', { + method: 'POST', + signal: submission.signal, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ previewId: reviewed.id }) + }); const body = await response.json(); if (current !== visit) return; if (!response.ok) { unconfirmedLaunch = response.status >= 500; launchError = body.error ?? 'Could not queue this collection.'; - if (!unconfirmedLaunch) { controller.select(null); refreshVersion++; } + if (!unconfirmedLaunch) { + controller.select(null); + refreshVersion++; + } return; } if (!body.ok) throw new Error('The launch response could not be confirmed.'); open = false; - if (body.collectionId) await goto(`/collections/launches/${encodeURIComponent(body.collectionId)}`); + if (body.collectionId) + await goto(`/collections/launches/${encodeURIComponent(body.collectionId)}`); } catch { if (current === visit) { // The queue may already be committed. Keep its preview ID for an idempotent retry. unconfirmedLaunch = true; - launchError = 'Could not confirm the launch. Retry the same request, or check collection launches.'; + launchError = + 'Could not confirm the launch. Retry the same request, or check collection launches.'; } - } finally { if (current === visit) launching = false; } + } finally { + if (current === visit) launching = false; + } } @@ -166,174 +292,660 @@
Apply collection - - + +
-

{mode === 'exact' ? 'Match the entire fleet across both games. Excess lobbies and maps outside this collection will be removed.' : 'Create missing lobbies. All existing servers and their settings stay as they are.'}

+

+ {mode === 'exact' + ? 'Match the entire fleet across both games. Excess lobbies and maps outside this collection will be removed.' + : 'Create missing lobbies. All existing servers and their settings stay as they are.'} +

{#if preview} - {noChanges ? 'Collection targets met' : `${preview.servers.length} to create · ${preview.removals.length} to remove`} - {#if mode === 'exact' && preview.retainedServers.length}

{preview.retainedServers.length} existing or queued servers kept.

{/if} + {noChanges + ? 'Collection targets met' + : `${preview.servers.length} to create · ${preview.removals.length} to remove`} + {#if mode === 'exact' && preview.retainedServers.length}

+ {preview.retainedServers.length} existing or queued servers kept. +

{/if}
- {#each keyCoverage as pool}= pool.required} class:warning={pool.available < pool.required}>{pool.game.toUpperCase()} keys: {pool.unassigned ? 'waiting for lobby availability' : `${pool.available} of ${pool.required} required available`}{/each} - {#if preview.servers.length}{hostStatus.filesReady ? 'Game files ready' : 'Game files preparing'}{/if} + {#each keyCoverage as pool}= pool.required} + class:warning={pool.available < pool.required} + >{pool.game.toUpperCase()} keys: {pool.unassigned + ? 'waiting for lobby availability' + : `${pool.available} of ${pool.required} required available`}{/each} + {#if preview.servers.length}{hostStatus.filesReady + ? 'Game files ready' + : 'Game files preparing'}{/if}
- {#if preview.servers.length}

{collectionCapacitySummary(preview.availability)}

{/if} - {#if noChanges}

{mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet meets or exceeds every target.'} Existing servers keep their current settings.

{/if} - {:else}

{checking ? 'Checking fleet changes…' : 'Resolve the issues below to check this collection.'}

{/if} + {#if preview.servers.length}

+ {collectionCapacitySummary(preview.availability)} +

{/if} + {#if noChanges}

+ {mode === 'exact' + ? 'Your fleet already matches this collection.' + : 'Your fleet meets or exceeds every target.'} Existing servers keep their + current settings. +

{/if} + {:else}

+ {checking + ? 'Checking fleet changes…' + : 'Resolve the issues below to check this collection.'} +

{/if}
- {#if launchError}{/if} + {#if launchError}{/if} {#if view.changeNotice}

{view.changeNotice}

{/if} {#if hostStatus.error} - {:else if !hostStatus.loading && hostStatus.missingGames.length}

Upload {hostStatus.missingGames.map(game => game.toUpperCase()).join(' and ')} game files before launching.

Manage game files ↗
- {:else if !hostStatus.loading && !imageValid}

Enter a valid runtime image in Advanced settings.

+ {:else if !hostStatus.loading && hostStatus.missingGames.length}
+

+ Upload {hostStatus.missingGames + .map((game) => game.toUpperCase()) + .join(' and ')} game files before launching. +

+ Manage game files ↗ +
+ {:else if !hostStatus.loading && !imageValid}

+ Enter a valid runtime image in Advanced settings. +

{:else if view.error}{/if} {#if missingKeys.length}
- {#each missingKeys as pool}{@const shortfall = pool.required - pool.unassigned - pool.available}

{pool.game.toUpperCase()} needs {shortfall} more unused key{shortfall === 1 ? '' : 's'}. Manage keys ↗

{/each} - {#if preview?.removals.length}

Keys assigned to removals stay in use until cleanup finishes. Complete those removals first to reuse their keys.

{/if} -
{/if} - {#each preview?.blockers ?? [] as blocker}{/each} - {#if !checking && !unconfirmedLaunch && (view.error || hostStatus.error || missingKeys.length || preview?.blockers.length)}{/if} + {#each missingKeys as pool}{@const shortfall = + pool.required - pool.unassigned - pool.available} +

+ {pool.game.toUpperCase()} needs {shortfall} more unused key{shortfall === + 1 + ? '' + : 's'}. + Manage keys ↗ +

{/each} + {#if preview?.removals.length}

+ Keys assigned to removals stay in use until cleanup finishes. Complete + those removals first to reuse their keys. +

{/if} +
{/if} + {#each preview?.blockers ?? [] as blocker}{/each} + {#if !checking && !unconfirmedLaunch && (view.error || hostStatus.error || missingKeys.length || preview?.blockers.length)}{/if} {#if preview}
{#each mapGroups as group} - {#if mode === 'exact'}{/if} - {#each group.rows as row}{#if mode === 'exact'}{/if}{/each} + {#if mode === 'exact'}{/if} + {#each group.rows as row}{#if mode === 'exact'}{/if}{/each}
{group.label}
MapTargetExisting / queuedCreateRemove
{row.mapName}{row.target}{row.existing} 0}>{row.create} 0}>{row.remove}
MapTargetExisting / queuedCreateRemove
{row.mapName}{row.target}{row.existing} 0}>{row.create} 0} + >{row.remove}
{/each} -

Stopped and unhealthy servers also count toward targets. See Server details for health.{#if mode === 'exact'} Existing totals include the listed removals.{/if}

+

+ Stopped and unhealthy servers also count toward targets. See Server details + for health.{#if mode === 'exact'} + Existing totals include the listed removals.{/if} +

- {#if preview.removals.length}
-

Destroy {preview.removals.length} server{preview.removals.length === 1 ? '' : 's'}

-

Containers, files, and registrations will be removed. Occupied or unreachable servers wait for a fresh empty-player check; unstarted creations are cancelled.

-
    {#each preview.removals as server}
  • {serverTitle(server.name)}{server.name}{server.reason}
    • {/each}
- -
{/if} + {#if preview.removals.length}
+

+ Destroy {preview.removals.length} server{preview.removals.length === 1 + ? '' + : 's'} +

+

+ Containers, files, and registrations will be removed. Occupied or + unreachable servers wait for a fresh empty-player check; unstarted + creations are cancelled. +

+
    + {#each preview.removals as server}
  • + {serverTitle(server.name)}{server.name}{server.reason} +
    • {/each} +
+ +
{/if}
Server details
- {#if preview.servers.length}

Launch order

Hosts and ports are assigned when provisioning starts.

-
    {#each preview.servers as server}
  1. {server.name ? serverTitle(server.name) : preview.entries[server.entryIndex].recipe.mapName}{server.name ?? 'No available lobby'}
    2. {/each}
+ {#if preview.servers.length}

Launch order

+

+ Hosts and ports are assigned when provisioning starts. +

+
    + {#each preview.servers as server}
  1. + {server.name + ? serverTitle(server.name) + : preview.entries[server.entryIndex].recipe + .mapName}{server.name ?? 'No available lobby'} +
    2. {/each} +
{/if} - {#if preview.entries.some(entry => entry.existing)}

Existing and queued servers

-
    {#each preview.entries as entry}{#each entry.coverage ?? [] as server}
  • {serverTitle(server.name)} ↗{server.health === 'online' ? 'Healthy' : server.health}{#if mode === 'exact'} · {preview.retainedServers.includes(server.name) ? 'Keep' : 'Remove'}{/if}
    • {/each}{/each}
+ {#if preview.entries.some((entry) => entry.existing)}

+ Existing and queued servers +

+
    + {#each preview.entries as entry}{#each entry.coverage ?? [] as server}
  • + {serverTitle(server.name)} ↗{server.health === 'online' + ? 'Healthy' + : server.health}{#if mode === 'exact'} + · {preview.retainedServers.includes(server.name) + ? 'Keep' + : 'Remove'}{/if} +
    • {/each}{/each} +
{/if} - {#if !preview.servers.length && !preview.entries.some(entry => entry.existing)}

No new or matching servers.

{/if} + {#if !preview.servers.length && !preview.entries.some((entry) => entry.existing)}

+ No new or matching servers. +

{/if}
{#if preview.availability && preview.servers.length}
- Capacity details -
-
-
CPU available
{preview.availability.cpuCoresAvailable === null ? 'Unknown' : `${preview.availability.cpuCoresAvailable.toFixed(1)} cores`}
-
Memory available
{preview.availability.memoryBytesAvailable === null ? 'Unknown' : `${(preview.availability.memoryBytesAvailable / 1024 ** 3).toFixed(1)} GiB`}
-
Ports available
{preview.availability.portsAvailable ?? 'Unknown'}
-
Startup slots
{preview.availability.startupSlots ?? 'Unknown'}{#if preview.availability.startupLimit !== null} / {preview.availability.startupLimit}{/if}
-
- {#each preview.warnings ?? [] as warning}

{warning}

{/each} -

{preview.availability.updatedAt ? `Updated ${$timeFormat.dateTime(preview.availability.updatedAt)}.` : 'Waiting for host reports.'} Capacity is checked for each server when it starts.

- View capacity and history ↗ -
-
{/if} + Capacity details +
+
+
+
CPU available
+
+ {preview.availability.cpuCoresAvailable === null + ? 'Unknown' + : `${preview.availability.cpuCoresAvailable.toFixed(1)} cores`} +
+
+
+
Memory available
+
+ {preview.availability.memoryBytesAvailable === null + ? 'Unknown' + : `${(preview.availability.memoryBytesAvailable / 1024 ** 3).toFixed(1)} GiB`} +
+
+
+
Ports available
+
{preview.availability.portsAvailable ?? 'Unknown'}
+
+
+
Startup slots
+
+ {preview.availability.startupSlots ?? + 'Unknown'}{#if preview.availability.startupLimit !== null} + / {preview.availability.startupLimit}{/if} +
+
+
+ {#each preview.warnings ?? [] as warning}

+ {warning} +

{/each} +

+ {preview.availability.updatedAt + ? `Updated ${$timeFormat.dateTime(preview.availability.updatedAt)}.` + : 'Waiting for host reports.'} Capacity is checked for each server + when it starts. +

+ View capacity and history ↗ +
+ {/if} {/if} - {#if open}{/if} + {#if open}{/if} {#if preview}
- Setup details -
-

New servers use the current production setup when provisioning starts. Existing servers keep their files and configuration.

-
Runtime image
{selection.image}
- {#each games as game}{@const files = hostStatus.gameFiles?.[game]} - {#if files?.uploadedAt}
{game.toUpperCase()} game files uploaded
{$timeFormat.dateTime(files.uploadedAt)}
{/if} - {/each} -
- {#each preview.entries as entry}
- {entry.recipe.mapName} · {entry.recipe.game.toUpperCase()} -

{entry.recipe.configSource}

{entry.recipe.files.length} managed files

-
    {#each entry.recipe.files as file}
  • {file.name}{file.version === null ? 'Generated' : `v${file.version}`} · {file.kind}{file.path}
    • {/each}
-
-
{/each} -
-
{/if} + Setup details +
+

+ New servers use the current production setup when provisioning starts. + Existing servers keep their files and configuration. +

+
+
+
Runtime image
+
{selection.image}
+
+ {#each games as game}{@const files = hostStatus.gameFiles?.[game]} + {#if files?.uploadedAt}
+
{game.toUpperCase()} game files uploaded
+
{$timeFormat.dateTime(files.uploadedAt)}
+
{/if} + {/each} +
+ {#each preview.entries as entry}
+ {entry.recipe.mapName} · {entry.recipe.game.toUpperCase()} +
+

{entry.recipe.configSource}

+

{entry.recipe.files.length} managed files

+
    + {#each entry.recipe.files as file}
  • + {file.name}{file.version === null + ? 'Generated' + : `v${file.version}`} · {file.kind}{file.path} +
    • {/each} +
+
+
{/each} +
+ {/if} {/if}
{#snippet actions()}
-

{footerStatus}

-
{#if noChanges} - {:else}{/if} +

+ {footerStatus} +

+
+ {#if noChanges} + {:else}{/if}
{/snippet}
diff --git a/src/lib/ui/components/DeployServerDialog.svelte b/src/lib/ui/components/DeployServerDialog.svelte index 2ab71b4..fc56f33 100644 --- a/src/lib/ui/components/DeployServerDialog.svelte +++ b/src/lib/ui/components/DeployServerDialog.svelte @@ -11,17 +11,35 @@ import type { NativeDeploymentSelection } from '$lib/hosts.js'; import type { ZombieMapEntry, ZombieGame } from '$lib/map-aliases.js'; - let { open = $bindable(false), sourceDeploymentId = '', initialGame = 't5', initialMapSlug = '' }: - { open?: boolean; sourceDeploymentId?: string; initialGame?: ZombieGame; initialMapSlug?: string } = $props(); + let { + open = $bindable(false), + sourceDeploymentId = '', + initialGame = 't5', + initialMapSlug = '' + }: { + open?: boolean; + sourceDeploymentId?: string; + initialGame?: ZombieGame; + initialMapSlug?: string; + } = $props(); let game = $state('t5'); let mapSlug = $state(''); let catalogs = $state>({ t5: [], t6: [] }); let gameSelect = $state(); let selection = $state({ libraryIds: {}, image: '' }); let hostReady = $state(false); - let hostStatus = $state({ loading: true, error: '', hasHosts: false, filesReady: false, missingGames: [], uploadedAt: null }); + let hostStatus = $state({ + loading: true, + error: '', + hasHosts: false, + filesReady: false, + missingGames: [], + uploadedAt: null + }); let view = $state({ preview: null, checking: false, error: '' }); - const controller = createServerLaunchController(state => { view = state; }); + const controller = createServerLaunchController((state) => { + view = state; + }); let loaded = $state(false); let loadingError = $state(''); let launchError = $state(''); @@ -37,39 +55,89 @@ const preview = $derived(view.preview); const expired = $derived(!!preview && Date.parse(preview.expiresAt) <= now); const imageValid = $derived(/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(selection.image)); - const checksBlocked = $derived(preview?.checks.filter(check => check.state === 'blocked') ?? []); + const checksBlocked = $derived( + preview?.checks.filter((check) => check.state === 'blocked') ?? [] + ); const missingKey = $derived(!!preview?.name && !preview.hasSavedKey); - const canLaunch = $derived(!launching && !!preview?.name && (unconfirmedLaunch || - (hostReady && !hostStatus.loading && !view.checking && !expired && preview.ready && preview.hasSavedKey))); - const checking = $derived(!loaded && !loadingError || hostStatus.loading || view.checking); - const footerStatus = $derived(launching ? 'Queueing server…' : unconfirmedLaunch ? 'Retry to confirm this launch.' - : loadingError ? 'Could not load launch options.' : checking ? 'Checking availability…' - : missingKey ? 'No game key available.' : canLaunch ? 'Ready to launch' : 'Launch unavailable'); + const canLaunch = $derived( + !launching && + !!preview?.name && + (unconfirmedLaunch || + (hostReady && + !hostStatus.loading && + !view.checking && + !expired && + preview.ready && + preview.hasSavedKey)) + ); + const checking = $derived((!loaded && !loadingError) || hostStatus.loading || view.checking); + const footerStatus = $derived( + launching + ? 'Queueing server…' + : unconfirmedLaunch + ? 'Retry to confirm this launch.' + : loadingError + ? 'Could not load launch options.' + : checking + ? 'Checking availability…' + : missingKey + ? 'No game key available.' + : canLaunch + ? 'Ready to launch' + : 'Launch unavailable' + ); $effect(() => { - if (open && !wasOpen) { wasOpen = true; void initialize(); } - else if (!open && wasOpen) { - wasOpen = false; visit++; initialization?.abort(); controller.clear(); + if (open && !wasOpen) { + wasOpen = true; + void initialize(); + } else if (!open && wasOpen) { + wasOpen = false; + visit++; + initialization?.abort(); + controller.clear(); } }); $effect(() => { // Selection changes invalidate the old preview before another launch can use it. - const input = open && loaded && hostReady && !hostStatus.loading ? { - game, mapSlug, libraryId: selection.libraryIds[game]!, image: selection.image, - bindAddress: selection.bindAddress || undefined, publicAddress: selection.publicAddress || undefined, - publicPort: selection.publicPort - } : null; + const input = + open && loaded && hostReady && !hostStatus.loading + ? { + game, + mapSlug, + libraryId: selection.libraryIds[game]!, + image: selection.image, + bindAddress: selection.bindAddress || undefined, + publicAddress: selection.publicAddress || undefined, + publicPort: selection.publicPort + } + : null; if (!launching && !unconfirmedLaunch) controller.select(input); }); async function initialize() { const current = ++visit; - initialization?.abort(); initialization = new AbortController(); + initialization?.abort(); + initialization = new AbortController(); const signal = initialization.signal; - controller.clear(); loaded = false; loadingError = ''; launchError = ''; launching = false; unconfirmedLaunch = false; - game = initialGame; mapSlug = initialMapSlug; - selection = { libraryIds: {}, image: '' }; hostReady = false; - hostStatus = { loading: true, error: '', hasHosts: false, filesReady: false, missingGames: [], uploadedAt: null }; + controller.clear(); + loaded = false; + loadingError = ''; + launchError = ''; + launching = false; + unconfirmedLaunch = false; + game = initialGame; + mapSlug = initialMapSlug; + selection = { libraryIds: {}, image: '' }; + hostReady = false; + hostStatus = { + loading: true, + error: '', + hasHosts: false, + filesReady: false, + missingGames: [], + uploadedAt: null + }; try { const response = await fetch('/api/maps', { signal }); if (!response.ok) throw new Error('Could not load maps. Try again.'); @@ -77,75 +145,117 @@ if (current !== visit) return; catalogs = { t5: body.t5Zombie, t6: body.t6Zombie }; if (sourceDeploymentId) { - const sourceResponse = await fetch(`/api/provisioning/${encodeURIComponent(sourceDeploymentId)}`, { signal }); + const sourceResponse = await fetch( + `/api/provisioning/${encodeURIComponent(sourceDeploymentId)}`, + { signal } + ); const source = await sourceResponse.json(); if (current !== visit) return; - if (!sourceResponse.ok || !source.recipe || !['t5', 't6'].includes(source.recipe.game) - || !catalogs[source.recipe.game as ZombieGame].some(map => map.slug === source.recipe.mapSlug)) { + if ( + !sourceResponse.ok || + !source.recipe || + !['t5', 't6'].includes(source.recipe.game) || + !catalogs[source.recipe.game as ZombieGame].some( + (map) => map.slug === source.recipe.mapSlug + ) + ) { throw new Error('The original launch map is unavailable.'); } - game = source.recipe.game; mapSlug = source.recipe.mapSlug; + game = source.recipe.game; + mapSlug = source.recipe.mapSlug; } - if (!catalogs[game].some(map => map.slug === mapSlug)) mapSlug = catalogs[game][0]?.slug ?? ''; + if (!catalogs[game].some((map) => map.slug === mapSlug)) + mapSlug = catalogs[game][0]?.slug ?? ''; loaded = true; await tick(); if (current === visit && open) gameSelect?.focus(); } catch (error) { - if (current === visit && !signal.aborted) loadingError = error instanceof Error ? error.message : 'Could not load launch options.'; + if (current === visit && !signal.aborted) + loadingError = + error instanceof Error ? error.message : 'Could not load launch options.'; } } function refreshChecks() { if (!open || launching || unconfirmedLaunch) return; - if (!loaded) { void initialize(); return; } + if (!loaded) { + void initialize(); + return; + } launchError = ''; controller.clear(); refreshVersion++; } - function selectionChanged() { launchError = ''; } + function selectionChanged() { + launchError = ''; + } onMount(() => { - const refreshOnReturn = () => { if (document.visibilityState !== 'hidden') refreshChecks(); }; + const refreshOnReturn = () => { + if (document.visibilityState !== 'hidden') refreshChecks(); + }; window.addEventListener('focus', refreshOnReturn); document.addEventListener('visibilitychange', refreshOnReturn); - const timer = setInterval(() => { now = Date.now(); }, 1000); + const timer = setInterval(() => { + now = Date.now(); + }, 1000); return () => { window.removeEventListener('focus', refreshOnReturn); document.removeEventListener('visibilitychange', refreshOnReturn); clearInterval(timer); }; }); - onDestroy(() => { visit++; initialization?.abort(); submission?.abort(); controller.dispose(); }); + onDestroy(() => { + visit++; + initialization?.abort(); + submission?.abort(); + controller.dispose(); + }); async function launch() { if (!canLaunch || !preview) return; const current = visit; const reviewed = preview; - controller.suspend(); launching = true; launchError = ''; + controller.suspend(); + launching = true; + launchError = ''; submission = new AbortController(); try { - const response = await fetch('/api/servers', { method: 'POST', signal: submission.signal, - headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - game: reviewed.recipe.game, gameMode: 'zombie', mapSlug: reviewed.recipe.mapSlug, - expectedName: reviewed.name, previewId: reviewed.id - }) }); + const response = await fetch('/api/servers', { + method: 'POST', + signal: submission.signal, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + game: reviewed.recipe.game, + gameMode: 'zombie', + mapSlug: reviewed.recipe.mapSlug, + expectedName: reviewed.name, + previewId: reviewed.id + }) + }); const body = await response.json(); if (current !== visit) return; if (!response.ok || !body.ok) { unconfirmedLaunch = response.status >= 500; launchError = body.error ?? 'Could not queue this launch.'; - if (!unconfirmedLaunch) { controller.clear(); refreshVersion++; } + if (!unconfirmedLaunch) { + controller.clear(); + refreshVersion++; + } return; } open = false; await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`); - } catch (error) { + } catch { if (current === visit) { // Retain the preview ID: retrying a possibly accepted launch must be idempotent. unconfirmedLaunch = true; - launchError = 'Could not confirm the launch. Retry to check the same request, or view Provisioning.'; + launchError = + 'Could not confirm the launch. Retry to check the same request, or view Provisioning.'; } - } finally { if (current === visit) launching = false; } + } finally { + if (current === visit) launching = false; + } } @@ -158,92 +268,326 @@

Loading maps…

{:else}
- - + +
- {preview?.name ? serverTitle(preview.name) : maps.find(map => map.slug === mapSlug)?.friendly ?? 'Choose a map'} + {preview?.name + ? serverTitle(preview.name) + : (maps.find((map) => map.slug === mapSlug)?.friendly ?? + 'Choose a map')}

Uses the current production setup when the server starts.

{#if checking}Checking availability… {:else} - {hostStatus.missingGames.length ? 'Game files missing' : hostStatus.filesReady ? 'Game files ready' : 'Game files preparing'} + {hostStatus.missingGames.length + ? 'Game files missing' + : hostStatus.filesReady + ? 'Game files ready' + : 'Game files preparing'} {#if preview?.hasSavedKey}Key available{/if} Automatic host & port {/if}
- {#if launchError}{/if} - {#if missingKey}

No unused {game === 't5' ? 'Black Ops I' : 'Black Ops II'} keys are available.

Manage keys ↗
{/if} + {#if launchError}{/if} + {#if missingKey}
+

+ No unused {game === 't5' ? 'Black Ops I' : 'Black Ops II'} keys are available. +

+ Manage keys ↗ +
{/if} {#if hostStatus.error} - {:else if !hostStatus.loading && hostStatus.missingGames.length}

Upload game files before launching.

Manage game files ↗
- {:else if !hostStatus.loading && !imageValid}

Enter a valid runtime image in Advanced settings.

- {:else if view.error && hostStatus.hasHosts}{/if} - {#if !hostStatus.loading && !hostStatus.error && !hostStatus.hasHosts}

Add a host to run your server.

Manage hosts ↗
{/if} + {:else if !hostStatus.loading && hostStatus.missingGames.length}
+

Upload game files before launching.

+ Manage game files ↗ +
+ {:else if !hostStatus.loading && !imageValid}

+ Enter a valid runtime image in Advanced settings. +

+ {:else if view.error && hostStatus.hasHosts}{/if} + {#if !hostStatus.loading && !hostStatus.error && !hostStatus.hasHosts}
+

Add a host to run your server.

+ Manage hosts ↗ +
{/if} {#each checksBlocked as check}

{check.detail}

{/each} - {#if !checking && !unconfirmedLaunch && (view.error || hostStatus.error || missingKey || checksBlocked.length)}{/if} + {#if !checking && !unconfirmedLaunch && (view.error || hostStatus.error || missingKey || checksBlocked.length)}{/if} - {#if open}{/if} + {#if open}{/if} {#if preview}
- Setup details -
-
Server ID
{preview.name ?? 'No lobby available'}
-
Configuration
{preview.recipe.configSource}
-
Container image
{preview.recipe.image}
- {#if hostStatus.uploadedAt}
Game files uploaded
{$timeFormat.dateTime(hostStatus.uploadedAt)}
{/if} -
-

{preview.recipe.files.length} managed files

-
    {#each preview.recipe.files as file}
  • {file.name}{file.version === null ? 'Generated' : `v${file.version}`} · {file.kind}
    {file.path}
    • {/each}
-
{/if} + Setup details +
+
+
Server ID
+
{preview.name ?? 'No lobby available'}
+
+
+
Configuration
+
{preview.recipe.configSource}
+
+
+
Container image
+
{preview.recipe.image}
+
+ {#if hostStatus.uploadedAt}
+
Game files uploaded
+
{$timeFormat.dateTime(hostStatus.uploadedAt)}
+
{/if} +
+

{preview.recipe.files.length} managed files

+
    + {#each preview.recipe.files as file}
  • +
    + {file.name}{file.version === null ? 'Generated' : `v${file.version}`} · + {file.kind} +
    + {file.path} +
    • {/each} +
+ {/if} {/if}
{#snippet actions()}

{footerStatus}

-
+
+ +
{/snippet} diff --git a/src/lib/ui/components/DestroyServerModal.svelte b/src/lib/ui/components/DestroyServerModal.svelte index e776cf1..a3e793e 100644 --- a/src/lib/ui/components/DestroyServerModal.svelte +++ b/src/lib/ui/components/DestroyServerModal.svelte @@ -5,7 +5,12 @@ import Button from './primitives/Button.svelte'; import Dialog from './primitives/Dialog.svelte'; - let { open = $bindable(false), name, players, round = null }: { open?: boolean; name: string; players: number | null; round?: number | null } = $props(); + let { + open = $bindable(false), + name, + players, + round = null + }: { open?: boolean; name: string; players: number | null; round?: number | null } = $props(); let submitting = $state(false); let error = $state(''); $effect(() => { @@ -17,34 +22,66 @@ submitting = true; error = ''; try { - const res = await fetch(`/api/servers/${encodeURIComponent(name)}`, { method: 'DELETE' }); + const res = await fetch(`/api/servers/${encodeURIComponent(name)}`, { + method: 'DELETE' + }); const body = await res.json(); - if (!res.ok || !body.ok || !body.deploymentId) throw new Error(body.error ?? 'Could not queue server destruction. Please retry.'); + if (!res.ok || !body.ok || !body.deploymentId) + throw new Error(body.error ?? 'Could not queue server destruction. Please retry.'); await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`); open = false; - } catch (err) { error = (err as Error).message; } - finally { submitting = false; } + } catch (err) { + error = (err as Error).message; + } finally { + submitting = false; + } }

{lifecycleImpact('destroy', players, round)}

-

This stops and removes the container, its writable server files, and registration. Shared game libraries, recorded history, and the saved server key are retained.

+

+ This stops and removes the container, its writable server files, and registration. + Shared game libraries, recorded history, and the saved server key are retained. +

Follow progress or cancel the waiting request on the provisioning page.

-

You can cancel while waiting. Once destruction begins, it cannot be undone.

+

+ You can cancel while waiting. Once destruction begins, it cannot be undone. +

{#if error}{/if}
{#snippet actions()} - - + + {/snippet} diff --git a/src/lib/ui/components/DiffViewer.svelte b/src/lib/ui/components/DiffViewer.svelte index a927ee2..bef34fb 100644 --- a/src/lib/ui/components/DiffViewer.svelte +++ b/src/lib/ui/components/DiffViewer.svelte @@ -31,9 +31,15 @@ ) j++; const prefix = alines.slice(0, i).map((t) => ({ kind: 'ctx' as const, text: t })); - const suffix = alines.slice(alines.length - j).map((t) => ({ kind: 'ctx' as const, text: t })); - const midA = alines.slice(i, alines.length - j).map((t) => ({ kind: 'del' as const, text: t })); - const midB = blines.slice(i, blines.length - j).map((t) => ({ kind: 'add' as const, text: t })); + const suffix = alines + .slice(alines.length - j) + .map((t) => ({ kind: 'ctx' as const, text: t })); + const midA = alines + .slice(i, alines.length - j) + .map((t) => ({ kind: 'del' as const, text: t })); + const midB = blines + .slice(i, blines.length - j) + .map((t) => ({ kind: 'add' as const, text: t })); return [...prefix, ...midA, ...midB, ...suffix]; } @@ -44,8 +50,8 @@ {relPath} 
-{#each rows as r, idx (idx)}{r.kind === 'add' ? '+' : r.kind === 'del' ? '-' : ' '} {r.text}
+{#each rows as r, idx (idx)}{r.kind === 'add' ? '+' : r.kind === 'del' ? '-' : ' '} {r.text}
 {/each}
@@ -76,12 +82,12 @@ border: 1px solid transparent; } .verb-added { - background: rgba(124, 212, 255, 0.10); + background: rgba(124, 212, 255, 0.1); border-color: rgba(124, 212, 255, 0.3); color: var(--state-online); } .verb-updated { - background: rgba(124, 192, 245, 0.10); + background: rgba(124, 192, 245, 0.1); border-color: rgba(124, 192, 245, 0.28); color: var(--text-primary); } @@ -95,7 +101,11 @@ border-color: rgba(255, 176, 32, 0.3); color: var(--state-warning); } - .path { color: var(--text-primary); font-family: var(--font-mono); font-size: var(--text-sm); } + .path { + color: var(--text-primary); + font-family: var(--font-mono); + font-size: var(--text-sm); + } pre { margin: 0; padding: 14px 16px; @@ -106,7 +116,15 @@ max-height: 540px; overflow: auto; } - .ln { display: block; } - .ln-add { background: color-mix(in srgb, var(--state-online) 8%, transparent); color: var(--state-online); } - .ln-del { background: rgba(255, 48, 80, 0.08); color: #ff8a9a; } + .ln { + display: block; + } + .ln-add { + background: color-mix(in srgb, var(--state-online) 8%, transparent); + color: var(--state-online); + } + .ln-del { + background: rgba(255, 48, 80, 0.08); + color: #ff8a9a; + } diff --git a/src/lib/ui/components/FleetActionsDialog.svelte b/src/lib/ui/components/FleetActionsDialog.svelte index e7be900..e38e611 100644 --- a/src/lib/ui/components/FleetActionsDialog.svelte +++ b/src/lib/ui/components/FleetActionsDialog.svelte @@ -2,7 +2,13 @@ import { untrack } from 'svelte'; import { currentRound, freshServer, type FleetServer } from '$lib/fleet-overview.js'; import { actionLabels } from '$lib/provisioning.js'; - import { fleetActions, fleetActionAvailable, queueFleetAction, type FleetAction, type FleetActionFailure } from '$ui/fleet-actions.js'; + import { + fleetActions, + fleetActionAvailable, + queueFleetAction, + type FleetAction, + type FleetActionFailure + } from '$ui/fleet-actions.js'; import { lifecycleImpact } from '$ui/lifecycle-actions.js'; import { serverHealth } from '$ui/server-health.js'; import { serverTitle } from '$ui/server-labels.js'; @@ -10,7 +16,10 @@ import Button from './primitives/Button.svelte'; import Dialog from './primitives/Dialog.svelte'; - let { open = $bindable(false), onqueued }: { open?: boolean; onqueued: (count: number, action: FleetAction) => void } = $props(); + let { + open = $bindable(false), + onqueued + }: { open?: boolean; onqueued: (count: number, action: FleetAction) => void } = $props(); let servers = $state([]); let action = $state('restart'); let selected = $state([]); @@ -25,71 +34,121 @@ start: 'Start the selected servers and verify their health.', restart: 'Restart the selected servers once they are empty, then verify their health.', stop: 'Stop the selected servers once they are empty. Their files, keys, and registrations are kept.', - rebuild: 'Rebuild the selected servers from the latest promoted setup once they are empty. Their writable files are permanently deleted.', - destroy: 'Destroy the selected servers once they are empty. Their containers, writable files, and registrations are permanently deleted.' + rebuild: + 'Rebuild the selected servers from the latest promoted setup once they are empty. Their writable files are permanently deleted.', + destroy: + 'Destroy the selected servers once they are empty. Their containers, writable files, and registrations are permanently deleted.' }; const destructive = $derived(action === 'rebuild' || action === 'destroy'); - const eligible = $derived(servers.filter(server => !queued.includes(server.name) && fleetActionAvailable(server, action, $uiNow))); - const targets = $derived(eligible.filter(server => selected.includes(server.name))); + const eligible = $derived( + servers.filter( + (server) => + !queued.includes(server.name) && fleetActionAvailable(server, action, $uiNow) + ) + ); + const targets = $derived(eligible.filter((server) => selected.includes(server.name))); const allSelected = $derived(eligible.length > 0 && targets.length === eligible.length); $effect(() => { if (!open) return; - untrack(() => { action = 'restart'; selected = []; queued = []; failures = []; confirmed = false; }); + untrack(() => { + action = 'restart'; + selected = []; + queued = []; + failures = []; + confirmed = false; + }); }); $effect(() => { if (!open) return; - refresh; + void refresh; const controller = new AbortController(); - untrack(() => { void load(controller.signal); }); + untrack(() => { + void load(controller.signal); + }); return () => controller.abort(); }); async function load(signal: AbortSignal) { - loading = true; error = ''; + loading = true; + error = ''; try { const response = await fetch('/api/servers', { signal, cache: 'no-store' }); if (!response.ok) throw new Error('Could not load servers.'); const body = await response.json(); if (signal.aborted) return; - servers = body.servers.sort((a: FleetServer, b: FleetServer) => serverTitle(a.name).localeCompare(serverTitle(b.name))); + servers = body.servers.sort((a: FleetServer, b: FleetServer) => + serverTitle(a.name).localeCompare(serverTitle(b.name)) + ); syncServerTime(body.serverNow); - } catch (cause) { if (!signal.aborted) error = cause instanceof Error ? cause.message : 'Could not load servers.'; } - finally { if (!signal.aborted) loading = false; } + } catch (cause) { + if (!signal.aborted) + error = cause instanceof Error ? cause.message : 'Could not load servers.'; + } finally { + if (!signal.aborted) loading = false; + } + } + function chooseAction() { + selected = []; + confirmed = false; + failures = []; } - function chooseAction() { selected = []; confirmed = false; failures = []; } function toggle(name: string, checked: boolean) { - selected = checked ? [...selected, name] : selected.filter(value => value !== name); + selected = checked ? [...selected, name] : selected.filter((value) => value !== name); + confirmed = false; + } + function selectAll(checked: boolean) { + selected = checked ? eligible.map((server) => server.name) : []; confirmed = false; } - function selectAll(checked: boolean) { selected = checked ? eligible.map(server => server.name) : []; confirmed = false; } async function submit() { if (busy || loading || error || (destructive && !confirmed)) return; - const names = targets.filter(server => fleetActionAvailable(server, action, serverTimeNow())).map(server => server.name); + const names = targets + .filter((server) => fleetActionAvailable(server, action, serverTimeNow())) + .map((server) => server.name); if (!names.length) return; - busy = true; failures = []; + busy = true; + failures = []; try { const result = await queueFleetAction(action, names); - queued = [...queued, ...result.queued.map(item => item.server)]; - selected = selected.filter(name => !queued.includes(name)); + queued = [...queued, ...result.queued.map((item) => item.server)]; + selected = selected.filter((name) => !queued.includes(name)); failures = result.failed; confirmed = false; if (result.queued.length) onqueued(result.queued.length, action); if (!result.failed.length) open = false; - } finally { busy = false; } + } finally { + busy = false; + } } - +
- +

{descriptions[action]}

{#if error}{/if} {#if loading}

Loading the latest server status…

{:else}
- + {targets.length} selected · {eligible.length} eligible
@@ -97,46 +156,165 @@
Servers {#each servers as server (server.name)} - {@const available = eligible.some(item => item.name === server.name)} + {@const available = eligible.some((item) => item.name === server.name)} {@const health = serverHealth(server, $uiNow)} {/each}
{:else if !error}

No servers are available.

{/if} {/if} - {#if action !== 'start'}

Each server waits for a live check confirming it is empty. Follow progress or cancel waiting jobs in Provisioning.

{/if} - {#if destructive}

Shared game libraries, recorded history, saved keys, and collections are retained.

{/if} - {#if destructive}{/if} - {#if failures.length}

Some requests could not be queued:

    {#each failures as failure}
  • {serverTitle(failure.server)}: {failure.error}
    • {/each}
{/if} - {#if busy}

Queuing {actionLabels[action].toLowerCase()} for the selected servers…

{/if} + {#if action !== 'start'}

+ Each server waits for a live check confirming it is empty. Follow progress or cancel + waiting jobs in Provisioning. +

{/if} + {#if destructive}

+ Shared game libraries, recorded history, saved keys, and collections are retained. +

{/if} + {#if destructive}{/if} + {#if failures.length}
+

Some requests could not be queued:

+
    + {#each failures as failure}
  • + {serverTitle(failure.server)}: + {failure.error} +
    • {/each} +
+
{/if} + {#if busy}

+ Queuing {actionLabels[action].toLowerCase()} for the selected servers… +

{/if}
{#snippet actions()} - - + + {/snippet} diff --git a/src/lib/ui/components/Freshness.svelte b/src/lib/ui/components/Freshness.svelte index ddf02df..119953b 100644 --- a/src/lib/ui/components/Freshness.svelte +++ b/src/lib/ui/components/Freshness.svelte @@ -1,20 +1,70 @@ +
- {#if error}{error}{#if onRetry}{/if} - {:else}{paused ? 'Paused' : known ? 'Live' : 'Waiting for update'}{#if known}{' · updated '}{/if}{!paused && intervalLabel ? ` · ${intervalLabel}` : ''}{/if} + {#if error}{error}{#if onRetry}{/if} + {:else}{paused + ? 'Paused' + : known + ? 'Live' + : 'Waiting for update'}{#if known}{' · updated '}{/if}{!paused && intervalLabel ? ` · ${intervalLabel}` : ''}{/if}
+ diff --git a/src/lib/ui/components/GameFileHosts.svelte b/src/lib/ui/components/GameFileHosts.svelte index 46fab67..04f675f 100644 --- a/src/lib/ui/components/GameFileHosts.svelte +++ b/src/lib/ui/components/GameFileHosts.svelte @@ -3,49 +3,137 @@ import type { Host } from '$lib/hosts.js'; import GameFileTransferProgress from './GameFileTransferProgress.svelte'; import Badge from './primitives/Badge.svelte'; - let { caches, hosts, latest = false }: { caches: HostLibraryCache[]; hosts: Host[]; latest?: boolean } = $props(); - const healthy = $derived(caches.filter(cache => cache.status === 'ready' && hosts.some(host => host.id === cache.hostId && host.connected) && !cache.error)); - const attention = $derived(caches.filter(cache => !healthy.includes(cache))); + let { + caches, + hosts, + latest = false + }: { caches: HostLibraryCache[]; hosts: Host[]; latest?: boolean } = $props(); + const healthy = $derived( + caches.filter( + (cache) => + cache.status === 'ready' && + hosts.some((host) => host.id === cache.hostId && host.connected) && + !cache.error + ) + ); + const attention = $derived(caches.filter((cache) => !healthy.includes(cache))); function automaticDownloadDetail(host: Host | undefined): string { if (host?.decommission) return 'Automatic setup is paused during host deletion.'; if (!host?.enrolled) return 'Waiting for host enrollment.'; if (host.mode !== 'active') return 'Automatic setup resumes when the host is active.'; - if (!host.credentialActive || !host.connected) return 'Automatic setup resumes when the host reconnects.'; - if (host.readiness === 'degraded') return host.readinessReasons[0] ?? 'The host needs attention before automatic setup can continue.'; + if (!host.credentialActive || !host.connected) + return 'Automatic setup resumes when the host reconnects.'; + if (host.readiness === 'degraded') + return ( + host.readinessReasons[0] ?? + 'The host needs attention before automatic setup can continue.' + ); return 'Download starts automatically after runtime preparation and earlier work finish.'; } - -{#if !hosts.length}

Add a host to prepare these files automatically.

{/if} +{#if !hosts.length}

+ Add a host to prepare these files automatically. +

{/if} {#if attention.length}{@render hostTable(attention)}{/if} -{#if healthy.length}
Ready hosts ({healthy.length}){@render hostTable(healthy)}
{/if} +{#if healthy.length}
+ Ready hosts ({healthy.length}){@render hostTable(healthy)} +
{/if} {#snippet hostTable(rows: HostLibraryCache[])} -
- {#each rows as cache (cache.hostId)} - {@const host = hosts.find(host => host.id === cache.hostId)} - {@const waiting = latest && cache.status === 'missing'} - - - - - - {/each} -
HostFile statusDetails
{host?.name ?? cache.hostId}{#if !host?.connected}{' · Offline'}{/if}{waiting ? 'Waiting' : cache.status[0].toUpperCase() + cache.status.slice(1)}{#if cache.status === 'ready'}
Transfer details
- {:else if cache.status !== 'missing'}{/if} - {#if cache.error}

{cache.error}

{:else if waiting}

{automaticDownloadDetail(host)}

{/if} - {#if waiting || cache.status === 'failed'}View host setup{/if} -
+
+ + + {#each rows as cache (cache.hostId)} + {@const host = hosts.find((host) => host.id === cache.hostId)} + {@const waiting = latest && cache.status === 'missing'} + + + + + + {/each} + +
HostFile statusDetails
{host?.name ?? cache.hostId}{#if !host?.connected}{' · Offline'}{/if}{waiting + ? 'Waiting' + : cache.status[0].toUpperCase() + cache.status.slice(1)}{#if cache.status === 'ready'}
+ Transfer details +
+ {:else if cache.status !== 'missing'}{/if} + {#if cache.error}

{cache.error}

{:else if waiting}

+ {automaticDownloadDetail(host)} +

{/if} + {#if waiting || cache.status === 'failed'}View host setup{/if} +
+
{/snippet} diff --git a/src/lib/ui/components/GameFileTransferProgress.svelte b/src/lib/ui/components/GameFileTransferProgress.svelte index cc95557..ed79eaa 100644 --- a/src/lib/ui/components/GameFileTransferProgress.svelte +++ b/src/lib/ui/components/GameFileTransferProgress.svelte @@ -14,7 +14,12 @@ onMount(() => { const timer = setInterval(() => { - if (cache.transfer && connected && (cache.status === 'downloading' || cache.status === 'verifying')) now = Date.now(); + if ( + cache.transfer && + connected && + (cache.status === 'downloading' || cache.status === 'verifying') + ) + now = Date.now(); }, 1000); return () => clearInterval(timer); }); @@ -22,31 +27,92 @@
    -
  1. 1. {downloaded ? 'Downloaded' : downloading ? 'Downloading' : 'Download'}
    2. -
  2. 2. {progress.complete ? 'Verified' : verifying ? 'Verifying' : 'Verify'}
    3. +
  3. + 1. {downloaded ? 'Downloaded' : downloading ? 'Downloading' : 'Download'} +
    4. +
  4. + 2. {progress.complete ? 'Verified' : verifying ? 'Verifying' : 'Verify'} +
{#if progress.phase && !progress.complete} - - {#if progress.phaseBytes !== null}{formatGameBytes(progress.phaseBytes)} / {formatGameBytes(progress.total)} {progress.phase === 'verifying' ? 'checked' : 'downloaded'}{:else}Checking every file…{/if} + + {#if progress.phaseBytes !== null}{formatGameBytes(progress.phaseBytes)} / {formatGameBytes( + progress.total + )} + {progress.phase === 'verifying' ? 'checked' : 'downloaded'}{:else}Checking every + file…{/if} {/if} {#if progress.elapsedSeconds !== null}
Elapsed {formatDuration(progress.elapsedSeconds * 1000)} - {#if progress.bytesPerSecond !== null}{formatGameBytes(progress.bytesPerSecond)}/s{/if} - {#if progress.etaSeconds !== null}About {formatDuration(Math.ceil(progress.etaSeconds) * 1000)} {progress.phase === 'verifying' ? 'verification' : 'download'} remaining - {:else if progress.live && progress.phaseBytes !== progress.total}Estimating {progress.phase === 'verifying' ? 'verification' : 'download'} time remaining…{/if} + {#if progress.bytesPerSecond !== null}{formatGameBytes(progress.bytesPerSecond)}/s{/if} + {#if progress.etaSeconds !== null}About {formatDuration(Math.ceil(progress.etaSeconds) * 1000)} + {progress.phase === 'verifying' ? 'verification' : 'download'} remaining + {:else if progress.live && progress.phaseBytes !== progress.total}Estimating {progress.phase === 'verifying' ? 'verification' : 'download'} time remaining…{/if}
{/if} - {#if progress.active && !connected}Host offline · waiting to resume updates + {#if progress.active && !connected}Host offline · waiting to resume updates {:else if progress.stale}Waiting for a fresh host report - {:else if downloading}All files download before verification starts.{/if} + {:else if downloading}All files download before verification starts.{/if}
diff --git a/src/lib/ui/components/HistoricalServerIdentity.svelte b/src/lib/ui/components/HistoricalServerIdentity.svelte index 14f9b28..60d23ad 100644 --- a/src/lib/ui/components/HistoricalServerIdentity.svelte +++ b/src/lib/ui/components/HistoricalServerIdentity.svelte @@ -2,7 +2,8 @@ import ServerIdentity from './ServerIdentity.svelte'; import Badge from './primitives/Badge.svelte'; - let { name, deletedAt, href }: { name: string; deletedAt?: string | null; href?: string } = $props(); + let { name, deletedAt, href }: { name: string; deletedAt?: string | null; href?: string } = + $props(); @@ -12,6 +13,14 @@ diff --git a/src/lib/ui/components/HostList.svelte b/src/lib/ui/components/HostList.svelte index fcaa7f6..d952f31 100644 --- a/src/lib/ui/components/HostList.svelte +++ b/src/lib/ui/components/HostList.svelte @@ -1,42 +1,164 @@ -
- - {#each hosts as host (host.id)} - {@const status = hostListStatus(host, now)} - {@const stale = hostReadingsStale(host, now)} - - - - - - - - - - {/each} -
HostStatusServersPlayersCPUMemory used / totalDisk used / total
{host.name}{#if host.hostname && host.hostname !== host.name}{host.hostname}{/if} - {#if stale && host.lastHeartbeat}Resource readings stale{/if}{status.label}{#if status.detail}{status.detail}{/if}{host.serverCount}{host.unreportedInstances === host.serverCount && host.serverCount > 0 ? '—' : `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}{hostCpuUsage(host.inventory.cpuUsagePercent)}{hostUsedSpace(host.inventory.memoryTotal, host.inventory.memoryAvailable)}{hostUsedSpace(host.inventory.storageTotal, host.inventory.storageAvailable)}
+
+ + + {#each hosts as host (host.id)} + {@const status = hostListStatus(host, now)} + {@const stale = hostReadingsStale(host, now)} + + + + + + + + + + {/each} +
HostStatusServersPlayersCPUMemory used / totalDisk used / total
{host.name}{#if host.hostname && host.hostname !== host.name}{host.hostname}{/if} + {#if stale && host.lastHeartbeat}Resource readings stale{/if}{status.label}{#if status.detail}{status.detail}{/if}{host.serverCount}{host.unreportedInstances === host.serverCount && host.serverCount > 0 + ? '—' + : `${host.playerCount}${host.unreportedInstances ? '+' : ''}`}{hostCpuUsage(host.inventory.cpuUsagePercent)}{hostUsedSpace( + host.inventory.memoryTotal, + host.inventory.memoryAvailable + )}{hostUsedSpace( + host.inventory.storageTotal, + host.inventory.storageAvailable + )}
+
diff --git a/src/lib/ui/components/HostPreparation.svelte b/src/lib/ui/components/HostPreparation.svelte index c5623f8..3cb3e2a 100644 --- a/src/lib/ui/components/HostPreparation.svelte +++ b/src/lib/ui/components/HostPreparation.svelte @@ -6,20 +6,53 @@
-

The default runtime and the latest available Black Ops I and Black Ops II game files are prepared automatically. Re-enabling a host requires a fresh runtime pull and verification of both game libraries before it accepts deployments. Running servers keep their current files.

-

Supported hosts also cache Plutonium files to speed up new deployments. This optional cache refreshes when a new production release is detected; deployments can proceed while it is unavailable.

+

+ The default runtime and the latest available Black Ops I and Black Ops II game files are + prepared automatically. Re-enabling a host requires a fresh runtime pull and verification of + both game libraries before it accepts deployments. Running servers keep their current files. +

+

+ Supported hosts also cache Plutonium files to speed up new deployments. This optional cache + refreshes when a new production release is detected; deployments can proceed while it is + unavailable. +

{#if host.preparation} -

Automatic setup: {host.preparation.state === 'ready' ? 'Ready' : host.preparation.state === 'preparing' ? 'In progress' : host.preparation.state === 'blocked' ? 'Needs attention' : 'Waiting'}

+

+ Automatic setup: {host.preparation.state === 'ready' + ? 'Ready' + : host.preparation.state === 'preparing' + ? 'In progress' + : host.preparation.state === 'blocked' + ? 'Needs attention' + : 'Waiting'} +

    {#each host.preparation.items as item}
  • -
    {item.label}{item.state === 'blocked' ? 'Needs attention' : item.state}
    +
    + {item.label}{item.state === 'blocked' ? 'Needs attention' : item.state} +

    {item.detail}

    {#if item.cache && item.cache.status !== 'missing'} -
    +
    + +
    {:else if item.bytesTotal && item.bytesDownloaded !== undefined && item.state === 'preparing'} - - {formatGameBytes(item.bytesDownloaded)} / {formatGameBytes(item.bytesTotal)} + + {formatGameBytes(item.bytesDownloaded)} / {formatGameBytes( + item.bytesTotal + )} {/if}
    • {/each} @@ -29,8 +62,62 @@
diff --git a/src/lib/ui/components/LaunchRecipe.svelte b/src/lib/ui/components/LaunchRecipe.svelte index bbc12a4..b0d4da3 100644 --- a/src/lib/ui/components/LaunchRecipe.svelte +++ b/src/lib/ui/components/LaunchRecipe.svelte @@ -1,34 +1,156 @@
-
{#if !historical}Current promoted setup{/if}

{recipe.mapName}

{recipe.game === 't6' ? 'Black Ops II' : 'Black Ops I'}
+
+
+ {#if !historical}Current promoted setup{/if} +

{recipe.mapName}

+
+ {recipe.game === 't6' ? 'Black Ops II' : 'Black Ops I'} +
-
Configuration
{recipe.configSource}
-
Container image
{recipe.image}
+
+
Configuration
+
{recipe.configSource}
+
+
+
Container image
+
{recipe.image}
+
- {#if !historical}

Each server uses the latest promoted scripts, plugins, and configuration when provisioning begins, including promotions made after this preview or while queued.

{/if} + {#if !historical}

+ Each server uses the latest promoted scripts, plugins, and configuration when + provisioning begins, including promotions made after this preview or while queued. +

{/if}
- {recipe.files.length} managed file{recipe.files.length === 1 ? '' : 's'} {historical ? 'recorded for this deployment' : 'in the current setup'} -
    {#each recipe.files as file}
  • {file.name}{file.version !== null ? `v${file.version}` : 'Generated'} · {file.kind} · {file.size.toLocaleString()} bytes
    {file.path}SHA-256 {file.sha256.slice(0, 12)}
    • {/each}
- {#if !historical}

Previewed . This preview shows the promoted setup at the time above. The deployment history records the files selected when provisioning begins.

{/if} - Recipe {recipe.fingerprint.slice(0, 16)} + {recipe.files.length} managed file{recipe.files.length === 1 ? '' : 's'} + {historical ? 'recorded for this deployment' : 'in the current setup'} +
    + {#each recipe.files as file}
  • +
    + {file.name}{file.version !== null ? `v${file.version}` : 'Generated'} · {file.kind} + · {file.size.toLocaleString()} bytes +
    + {file.path}SHA-256 {file.sha256.slice(0, 12)} +
    • {/each} +
+ {#if !historical}

+ Previewed . This preview shows the + promoted setup at the time above. The deployment history records the files selected + when provisioning begins. +

{/if} + Recipe {recipe.fingerprint.slice(0, 16)}
diff --git a/src/lib/ui/components/LibraryVersions.svelte b/src/lib/ui/components/LibraryVersions.svelte index 2a67378..a0f1e6e 100644 --- a/src/lib/ui/components/LibraryVersions.svelte +++ b/src/lib/ui/components/LibraryVersions.svelte @@ -1,23 +1,72 @@
- {#if newer}
{label(newer)}{state}
{/if} - {#if current}
{label(current)}{#if newer}Promoted{:else}Promoted{/if}
+ {#if newer}
+ {label(newer)}{state} +
{/if} + {#if current}
+ {label(current)}{#if newer}Promoted{:else}Promoted{/if} +
{:else}{latest ? 'Never promoted' : 'No versions'}{/if} {#if betaId !== null && !newer}Beta active{/if}
diff --git a/src/lib/ui/components/LifecycleDialog.svelte b/src/lib/ui/components/LifecycleDialog.svelte index 6517aba..21529a9 100644 --- a/src/lib/ui/components/LifecycleDialog.svelte +++ b/src/lib/ui/components/LifecycleDialog.svelte @@ -10,8 +10,21 @@ import Dialog from './primitives/Dialog.svelte'; import Button from './primitives/Button.svelte'; - let { open = $bindable(false), initialServer = '', initialAction = 'start', serverContext = null, players = null, round = null }: - { open?: boolean; initialServer?: string; initialAction?: RuntimeAction; serverContext?: FleetServer | null; players?: number | null; round?: number | null } = $props(); + let { + open = $bindable(false), + initialServer = '', + initialAction = 'start', + serverContext = null, + players = null, + round = null + }: { + open?: boolean; + initialServer?: string; + initialAction?: RuntimeAction; + serverContext?: FleetServer | null; + players?: number | null; + round?: number | null; + } = $props(); let servers = $state([]); let server = $state(untrack(() => initialServer)); let action = $state(untrack(() => initialAction)); @@ -19,15 +32,35 @@ let loading = $state(false); let error = $state(''); let confirmed = $state(false); - const selected = $derived(serverContext?.name === server ? serverContext : servers.find(item => item.name === server)); + const selected = $derived( + serverContext?.name === server + ? serverContext + : servers.find((item) => item.name === server) + ); const available = $derived(selected ? applicableRuntimeActions(selected, $uiNow) : []); const applicable = $derived(action === 'destroy' || available.includes(action)); const destructive = $derived(action === 'rebuild' || action === 'destroy'); const health = $derived(selected ? serverHealth(selected, $uiNow) : null); - const impactPlayers = $derived(initialServer ? players : selected && health?.state === 'online' && freshServer(selected, $uiNow) ? selected.last_players : null); - const impactRound = $derived(initialServer ? round : selected ? currentRound(selected, selected.telemetry, $uiNow) : null); - const title = $derived(initialServer ? `${actionLabels[action]} ${serverTitle(initialServer)}?` : 'Manage server'); - const submitLabel = $derived(action === 'start' ? 'Start server' : action === 'destroy' ? 'Destroy server when empty' : `${actionLabels[action]} when empty`); + const impactPlayers = $derived( + initialServer + ? players + : selected && health?.state === 'online' && freshServer(selected, $uiNow) + ? selected.last_players + : null + ); + const impactRound = $derived( + initialServer ? round : selected ? currentRound(selected, selected.telemetry, $uiNow) : null + ); + const title = $derived( + initialServer ? `${actionLabels[action]} ${serverTitle(initialServer)}?` : 'Manage server' + ); + const submitLabel = $derived( + action === 'start' + ? 'Start server' + : action === 'destroy' + ? 'Destroy server when empty' + : `${actionLabels[action]} when empty` + ); $effect(() => { if (!open) return; const fixedServer = initialServer; @@ -36,86 +69,181 @@ // Live context and match reports update the copy and eligibility without // reopening the dialog or clearing a deliberate confirmation. untrack(() => { - server = fixedServer; action = requestedAction; confirmed = false; error = ''; servers = []; + server = fixedServer; + action = requestedAction; + confirmed = false; + error = ''; + servers = []; loading = false; - if (!fixedServer || serverContext?.name !== fixedServer) void load(fixedServer, controller.signal); + if (!fixedServer || serverContext?.name !== fixedServer) + void load(fixedServer, controller.signal); }); return () => controller.abort(); }); async function load(fixedServer: string, signal: AbortSignal) { loading = true; try { - const response = await fetch(fixedServer ? `/api/servers/${encodeURIComponent(fixedServer)}` : '/api/servers', { signal }); + const response = await fetch( + fixedServer ? `/api/servers/${encodeURIComponent(fixedServer)}` : '/api/servers', + { signal } + ); if (!response.ok) throw new Error('Could not load servers.'); const body = await response.json(); if (signal.aborted) return; servers = body.servers; syncServerTime(body.serverNow); - } catch (err) { if (!signal.aborted) error = (err as Error).message; } - finally { if (!signal.aborted) loading = false; } + } catch (err) { + if (!signal.aborted) error = (err as Error).message; + } finally { + if (!signal.aborted) loading = false; + } } function chooseServer(event: Event & { currentTarget: EventTarget & HTMLSelectElement }) { server = event.currentTarget.value; - confirmed = false; error = ''; - const item = servers.find(item => item.name === server); + confirmed = false; + error = ''; + const item = servers.find((item) => item.name === server); const actions = item ? applicableRuntimeActions(item, serverTimeNow()) : []; action = actions[0] ?? 'destroy'; } async function submit() { if (busy || !server || (destructive && !confirmed)) return; - if (action !== 'destroy' && (!selected || !applicableRuntimeActions(selected, serverTimeNow()).includes(action))) { - error = 'The server status has changed. Close this dialog and choose an action from its current status.'; + if ( + action !== 'destroy' && + (!selected || !applicableRuntimeActions(selected, serverTimeNow()).includes(action)) + ) { + error = + 'The server status has changed. Close this dialog and choose an action from its current status.'; return; } - busy = true; error = ''; + busy = true; + error = ''; try { - const response = action === 'destroy' - ? await fetch(`/api/servers/${encodeURIComponent(server)}`, { method: 'DELETE' }) - : await fetch('/api/provisioning', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, action }) }); + const response = + action === 'destroy' + ? await fetch(`/api/servers/${encodeURIComponent(server)}`, { + method: 'DELETE' + }) + : await fetch('/api/provisioning', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ server, action }) + }); const body = await response.json(); - if (!response.ok || !body.deploymentId) throw new Error(body.error ?? 'Could not queue this operation.'); + if (!response.ok || !body.deploymentId) + throw new Error(body.error ?? 'Could not queue this operation.'); await goto(`/provisioning/${encodeURIComponent(body.deploymentId)}`); open = false; - } catch (err) { error = (err as Error).message; } - finally { busy = false; } + } catch (err) { + error = (err as Error).message; + } finally { + busy = false; + } }
{#if !initialServer} - - + + {/if} {#if loading}

Checking the latest server status…

{:else if server} {#if applicable}

{lifecycleImpact(action, impactPlayers, impactRound)}

-

{action === 'destroy' ? 'Remove the container, its writable server files, and registration after the server is empty. Shared game libraries, recorded history, and the key saved separately in Keys are retained.' : runtimeDescriptions[action]}

- {#if action !== 'start'}

Follow progress or cancel the waiting request on the provisioning page.

{/if} - {:else}

{actionLabels[action]} is unavailable. {health?.reason ?? 'The current server status could not be verified.'} Close this dialog and choose an action from its current status.

{/if} - {#if applicable && destructive}{/if} +

+ {action === 'destroy' + ? 'Remove the container, its writable server files, and registration after the server is empty. Shared game libraries, recorded history, and the key saved separately in Keys are retained.' + : runtimeDescriptions[action]} +

+ {#if action !== 'start'}

+ Follow progress or cancel the waiting request on the provisioning page. +

{/if} + {:else}

+ {actionLabels[action]} is unavailable. {health?.reason ?? + 'The current server status could not be verified.'} Close this dialog and choose + an action from its current status. +

{/if} + {#if applicable && destructive}{/if} {/if} {#if error}{/if}
{#snippet actions()} - - + + {/snippet} diff --git a/src/lib/ui/components/LogPanel.svelte b/src/lib/ui/components/LogPanel.svelte index cc0c402..641b8a3 100644 --- a/src/lib/ui/components/LogPanel.svelte +++ b/src/lib/ui/components/LogPanel.svelte @@ -11,8 +11,8 @@ const uid = $props.id(); interface TabDef { - key: string; // matches the log key in KNOWN_LOGS - label: string; // human-facing tab name + key: string; // matches the log key in KNOWN_LOGS + label: string; // human-facing tab name } interface Props { @@ -23,16 +23,21 @@ } let { server, tabs, lines = 1000, pollMs = 5000 }: Props = $props(); - const active = $derived(tabs.find(tab => tab.key === $page.url.searchParams.get('log'))?.key ?? tabs[0]?.key ?? ''); + const active = $derived( + tabs.find((tab) => tab.key === $page.url.searchParams.get('log'))?.key ?? tabs[0]?.key ?? '' + ); let data = $state>({}); let checkedAt = $state>({}); function chooseLog(key: string) { - const url = new URL($page.url); url.searchParams.set('log', key); + const url = new URL($page.url); + url.searchParams.set('log', key); void goto(url, { noScroll: true, keepFocus: true, replaceState: true }); } let query = $state(''); - onMount(() => { query = $page.url.searchParams.get('logQuery') ?? ''; }); + onMount(() => { + query = $page.url.searchParams.get('logQuery') ?? ''; + }); let loading = $state(false); let autoRefresh = $state(true); let refreshError = $state(''); @@ -41,15 +46,25 @@ let stickToBottom = $state(true); const refresh = createLogRefresh({ - onReset: () => { data = {}; checkedAt = {}; refreshError = ''; copyStatus = ''; }, - onLoading: value => { loading = value; }, - onError: message => { refreshError = message; }, + onReset: () => { + data = {}; + checkedAt = {}; + refreshError = ''; + copyStatus = ''; + }, + onLoading: (value) => { + loading = value; + }, + onError: (message) => { + refreshError = message; + }, onData: (key, body) => { data = { ...data, [key]: body }; checkedAt = { ...checkedAt, [key]: Date.now() }; const currentServer = server; void tick().then(() => { - if (server === currentServer && active === key && stickToBottom && !query && bodyEl) bodyEl.scrollTop = bodyEl.scrollHeight; + if (server === currentServer && active === key && stickToBottom && !query && bodyEl) + bodyEl.scrollTop = bodyEl.scrollHeight; }); } }); @@ -68,32 +83,51 @@ refresh.configure({ server, key: active, lines }, autoRefresh, pollMs); } async function jumpToLatest() { - query = ''; stickToBottom = true; - await tick(); if (bodyEl) bodyEl.scrollTop = bodyEl.scrollHeight; + query = ''; + stickToBottom = true; + await tick(); + if (bodyEl) bodyEl.scrollTop = bodyEl.scrollHeight; } async function copy(value: string, message: string) { copyStatus = ''; - try { await navigator.clipboard.writeText(value); copyStatus = message; } - catch { copyStatus = 'Could not copy. Select the log text or browser address to copy manually.'; } + try { + await navigator.clipboard.writeText(value); + copyStatus = message; + } catch { + copyStatus = 'Could not copy. Select the log text or browser address to copy manually.'; + } } function copyLink() { - const url = new URL(window.location.href); url.searchParams.set('tab', 'logs'); url.searchParams.set('log', active); - if (query) url.searchParams.set('logQuery', query); else url.searchParams.delete('logQuery'); + const url = new URL(window.location.href); + url.searchParams.set('tab', 'logs'); + url.searchParams.set('log', active); + if (query) url.searchParams.set('logQuery', query); + else url.searchParams.delete('logQuery'); void copy(url.href, 'Log link copied.'); } function download() { const blob = new Blob([(current?.lines ?? []).join('\n')], { type: 'text/plain' }); const url = URL.createObjectURL(blob); - const link = document.createElement('a'); link.href = url; link.download = `${server}-${active}.log`; link.click(); + const link = document.createElement('a'); + link.href = url; + link.download = `${server}-${active}.log`; + link.click(); setTimeout(() => URL.revokeObjectURL(url), 1000); } function highlight(line: string) { if (!query) return [{ text: line, match: false }]; - const parts: { text: string; match: boolean }[] = []; let start = 0; let index: number; + const parts: { text: string; match: boolean }[] = []; + let start = 0; + let index: number; while ((index = line.toLowerCase().indexOf(query.toLowerCase(), start)) !== -1) { - parts.push({ text: line.slice(start, index), match: false }, { text: line.slice(index, index + query.length), match: true }); start = index + query.length; + parts.push( + { text: line.slice(start, index), match: false }, + { text: line.slice(index, index + query.length), match: true } + ); + start = index + query.length; } - parts.push({ text: line.slice(start), match: false }); return parts; + parts.push({ text: line.slice(start), match: false }); + return parts; } $effect(() => { const target = { server, key: active, lines }; @@ -111,67 +145,193 @@ } let current = $derived(data[active] ?? null); - const visibleLines = $derived((current?.lines ?? []).map(stripGameColorCodes).filter(line => !query || line.toLowerCase().includes(query.toLowerCase()))); + const visibleLines = $derived( + (current?.lines ?? []) + .map(stripGameColorCodes) + .filter((line) => !query || line.toLowerCase().includes(query.toLowerCase())) + );
- ({id:t.key,label:t.label}))} value={active} onchange={chooseLog} idPrefix={uid} panelId={uid + '-panel'} ariaLabel="Log files" size="sm" /> + ({ id: t.key, label: t.label }))} + value={active} + onchange={chooseLog} + idPrefix={uid} + panelId={uid + '-panel'} + ariaLabel="Log files" + size="sm" + />
{#if current?.mtime} - {fmtBytes(current.size)} · File changed + {fmtBytes(current.size)} · File changed {/if} - {#if !autoRefresh}{/if} + {#if !autoRefresh}{/if}
- void refresh.refresh()} retryDisabled={loading} /> -
More
+ void refresh.refresh()} + retryDisabled={loading} + /> +
+ +
+ More +
+ +
+
+
{#if copyStatus}

{copyStatus}

{/if} - +
- {#if refreshError}{/if} - {#if !current && !refreshError} -
{loading ? 'Loading log…' : autoRefresh ? 'Waiting for the first log update.' : 'No log loaded. Select Refresh to load it.'}
- {:else if current && !current.ok} -
error: {current.reason ?? 'unknown'}
- {:else if current?.missing} -
- log file not written yet - {#if current.path}
{current.path}{/if} -
- {:else if current && (!current.lines || current.lines.length === 0)} -
log is empty
- {:else if current?.lines} - - 
{#each visibleLines as line}{#each highlight(line) as part}{#if part.match}{part.text}{:else}{part.text}{/if}{/each}{'\n'}{/each}
- {#if query && !visibleLines.length}

No matches in the loaded tail. Try another search or download the tail.

{/if} - {#if current.truncated} -
showing last {current.lines.length} lines (file is larger)
- {:else} -
{current.lines.length} line{current.lines.length === 1 ? '' : 's'}
+ {#if refreshError}{/if} + {#if !current && !refreshError} +
+ {loading + ? 'Loading log…' + : autoRefresh + ? 'Waiting for the first log update.' + : 'No log loaded. Select Refresh to load it.'} +
+ {:else if current && !current.ok} +
error: {current.reason ?? 'unknown'}
+ {:else if current?.missing} +
+ log file not written yet + {#if current.path}
{current.path}{/if} +
+ {:else if current && (!current.lines || current.lines.length === 0)} +
log is empty
+ {:else if current?.lines} + + 
{#each visibleLines as line}{#each highlight(line) as part}{#if part.match}{part.text}{:else}{part.text}{/if}{/each}{'\n'}{/each}
+ {#if query && !visibleLines.length}

+ No matches in the loaded tail. Try another search or download the tail. +

{/if} + {#if current.truncated} +
showing last {current.lines.length} lines (file is larger)
+ {:else} +
+ {current.lines.length} line{current.lines.length === 1 ? '' : 's'} +
+ {/if} {/if} - {/if}
diff --git a/src/lib/ui/components/MatchHistory.svelte b/src/lib/ui/components/MatchHistory.svelte index 4eb4d04..43a09e3 100644 --- a/src/lib/ui/components/MatchHistory.svelte +++ b/src/lib/ui/components/MatchHistory.svelte @@ -6,7 +6,14 @@ import { goto } from '$app/navigation'; import { timeFormat, timeZone } from '$ui/stores/time-zone.js'; import type { MatchHistoryResponse } from '$lib/match-history.js'; - import { matchApiParams, matchHistoryHref, matchPageSize, readMatchQuery, type MatchHistoryQuery, type MatchSort } from '$lib/match-history-query.js'; + import { + matchApiParams, + matchHistoryHref, + matchPageSize, + readMatchQuery, + type MatchHistoryQuery, + type MatchSort + } from '$lib/match-history-query.js'; import { matchInstanceLabel } from '$ui/match-history.js'; import { pollVisible } from '$ui/poll.js'; import { formatDuration } from '$ui/duration.js'; @@ -22,7 +29,7 @@ let search = $state(untrack(() => matchApiParams($page.url).get('q') ?? '')); let retry = $state(0); let filtersOpen = $state(false); - let checkedAt = $state(untrack(() => initial ? Date.now() : null)); + let checkedAt = $state(untrack(() => (initial ? Date.now() : null))); let tableRegion = $state(); let pendingUrl: URL | undefined; let navigationId = 0; @@ -30,32 +37,60 @@ const apiUrl = $derived.by(() => { const params = matchApiParams($page.url); params.set('limit', String(matchPageSize)); - const endpoint = server === undefined ? '/api/matches' : `/api/servers/${encodeURIComponent(server)}/matches`; + const endpoint = + server === undefined + ? '/api/matches' + : `/api/servers/${encodeURIComponent(server)}/matches`; return `${endpoint}?${params}`; }); - let loadedUrl = $state(untrack(() => initial ? apiUrl : '')); - const filterCount = $derived([filters.game, filters.map, filters.q, filters.days, filters.status].filter(Boolean).length); - const hasFilters = $derived(!!(filters.game || filters.map || filters.q || filters.days || filters.status)); - const sorts: Array<{ key: MatchSort; label: string }> = [{ key: 'started', label: 'Started' }, { key: 'players', label: 'Players' }, { key: 'duration', label: 'Duration' }, { key: 'round', label: 'Round' }]; + let loadedUrl = $state(untrack(() => (initial ? apiUrl : ''))); + const filterCount = $derived( + [filters.game, filters.map, filters.q, filters.days, filters.status].filter(Boolean).length + ); + const hasFilters = $derived( + !!(filters.game || filters.map || filters.q || filters.days || filters.status) + ); + const sorts: Array<{ key: MatchSort; label: string }> = [ + { key: 'started', label: 'Started' }, + { key: 'players', label: 'Players' }, + { key: 'duration', label: 'Duration' }, + { key: 'round', label: 'Round' } + ]; - $effect(() => { search = filters.q; }); + $effect(() => { + search = filters.q; + }); $effect(() => { const report = initial; - if (report) untrack(() => { data = report; loadedUrl = apiUrl; loading = false; error = ''; checkedAt = Date.now(); }); + if (report) + untrack(() => { + data = report; + loadedUrl = apiUrl; + loading = false; + error = ''; + checkedAt = Date.now(); + }); }); $effect(() => { const url = apiUrl; - retry; - untrack(() => { if (loadedUrl !== url) loading = true; error = ''; }); - return pollVisible(async signal => { + void retry; + untrack(() => { + if (loadedUrl !== url) loading = true; + error = ''; + }); + return pollVisible(async (signal) => { try { const response = await fetch(url, { signal, cache: 'no-store' }); if (!response.ok) throw new Error('Could not load match history.'); const body: MatchHistoryResponse = await response.json(); if (signal.aborted || url !== apiUrl) return; // Do not move keyboard users to another match while they are following a link. - if (loadedUrl !== url || !tableRegion?.contains(document.activeElement)) data = body; - loadedUrl = url; loading = false; error = ''; checkedAt = Date.now(); + if (loadedUrl !== url || !tableRegion?.contains(document.activeElement)) + data = body; + loadedUrl = url; + loading = false; + error = ''; + checkedAt = Date.now(); } catch (cause) { if (signal.aborted || url !== apiUrl) return; error = cause instanceof Error ? cause.message : 'Could not load match history.'; @@ -69,130 +104,547 @@ pendingUrl = new URL(href, $page.url); const id = ++navigationId; void goto(href, { noScroll: true, keepFocus: true }) - .catch(() => { error = 'Could not update the match filters. Please try again.'; }) - .finally(() => { if (id === navigationId) pendingUrl = undefined; }); + .catch(() => { + error = 'Could not update the match filters. Please try again.'; + }) + .finally(() => { + if (id === navigationId) pendingUrl = undefined; + }); + } + function sort(key: MatchSort) { + update({ + sort: key, + direction: filters.sort === key && filters.direction === 'desc' ? 'asc' : 'desc' + }); + } + function reset() { + update({ game: '', map: '', q: '', days: '', status: '' }); } - function sort(key: MatchSort) { update({ sort: key, direction: filters.sort === key && filters.direction === 'desc' ? 'asc' : 'desc' }); } - function reset() { update({ game: '', map: '', q: '', days: '', status: '' }); }
{#snippet filterFields(suffix: string)}
- {#if server === undefined}{/if} - - {#if server === undefined}
{ event.preventDefault(); update({ q: search.trim() }); if (suffix) filtersOpen = false; }}> - - Press Enter to search by server name. -
{/if} - - + {#if server === undefined}{/if} + + {#if server === undefined}
{ + event.preventDefault(); + update({ q: search.trim() }); + if (suffix) filtersOpen = false; + }} + > + + Press Enter to search by server name. +
{/if} + + {#if suffix} - - + + {/if}
{/snippet}
{@render filterFields('')}
-
{data?.total ?? '—'} matches{filters.map ? ` · ${data?.maps.find(map => map.key === filters.map)?.name ?? 'Selected map'}` : ''}
- +
+ {data?.total ?? '—'} matches{filters.map + ? ` · ${data?.maps.find((map) => map.key === filters.map)?.name ?? 'Selected map'}` + : ''} +
+ {@render filterFields('-mobile')} - {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet} -
Times in {$timeZone}{filters.days ? ` · Last ${filters.days} days` : ''}{#if hasFilters}{/if}{#if !error}{/if}
- {#if error}{/if} +
+ Times in {$timeZone}{filters.days ? ` · Last ${filters.days} days` : ''}{#if hasFilters}{/if}{#if !error}{/if} +
+ {#if error}{/if} {#if loading} - {:else if loadedUrl !== apiUrl && error} + {:else if loadedUrl !== apiUrl && error} {:else if data?.matches.length} -
+
- - - - - {#each sorts.filter(item => item.key !== 'started') as item} - - {/each} - - {#each data.matches as match (match.id)} - - - - - - - - - {/each} + + + + + {#each sorts.filter((item) => item.key !== 'started') as item} + + {/each} + + {#each data.matches as match (match.id)} + + + + + + + + + {/each}
Matches sorted by {sorts.find(item => item.key === filters.sort)?.label.toLowerCase()}, {filters.direction === 'desc' ? 'highest or newest' : 'lowest or oldest'} first
MatchStatus
- {#if match.status === 'completed'}{match.map} · Match #{match.id} - {:else}{match.map}{/if} - · {match.status === 'active' ? 'In progress' : match.status === 'completed' ? 'Completed' : 'Interrupted'}{match.durationMs !== null ? ` · ${formatDuration(match.durationMs)}` : ''} - {#if server === undefined}
- {matchInstanceLabel(match)}{#if match.server_deleted_at} · Historical instance{/if} -
{/if} - 		{match.status === 'active' ? 'In progress' : match.status === 'completed' ? 'Completed' : 'Interrupted'}{match.uniquePlayers}{match.durationMs === null ? '—' : formatDuration(match.durationMs)}{match.status === 'completed' ? match.finalRound ?? '—' : match.observedRound ?? '—'}{#if match.status !== 'completed'}Highest observed{/if}
Matches sorted by {sorts + .find((item) => item.key === filters.sort) + ?.label.toLowerCase()}, {filters.direction === 'desc' + ? 'highest or newest' + : 'lowest or oldest'} first
MatchStatus
+ {#if match.status === 'completed'}{match.map} · Match #{match.id} + {:else}{match.map}{/if} + · {match.status === + 'active' + ? 'In progress' + : match.status === 'completed' + ? 'Completed' + : 'Interrupted'}{match.durationMs !== null + ? ` · ${formatDuration(match.durationMs)}` + : ''} + {#if server === undefined}
+ {matchInstanceLabel(match)}{#if match.server_deleted_at} + · Historical instance{/if} +
{/if} + 		{match.status === 'active' + ? 'In progress' + : match.status === 'completed' + ? 'Completed' + : 'Interrupted'}{match.uniquePlayers}{match.durationMs === null + ? '—' + : formatDuration(match.durationMs)}{match.status === 'completed' + ? (match.finalRound ?? '—') + : (match.observedRound ?? + '—')}{#if match.status !== 'completed'}Highest observed{/if}
-
diff --git a/src/lib/ui/components/NativeHostSelection.svelte b/src/lib/ui/components/NativeHostSelection.svelte index d387e00..1a9e7c7 100644 --- a/src/lib/ui/components/NativeHostSelection.svelte +++ b/src/lib/ui/components/NativeHostSelection.svelte @@ -5,31 +5,90 @@ import type { GameLibrary } from '$lib/game-files.js'; import type { ZombieGame } from '$lib/map-aliases.js'; import { pollVisible } from '$ui/poll.js'; - import { latestGameFiles, selectLatestGameFiles, type NativeHostSelectionStatus } from '$ui/native-game-files.js'; - let { games, selection = $bindable(), ready = $bindable(false), status = $bindable(), disabled = false, network = false, compact = false, refreshVersion = 0, onchange }: - { games: ZombieGame[]; selection: NativeDeploymentSelection; ready?: boolean; status?: NativeHostSelectionStatus; disabled?: boolean; network?: boolean; compact?: boolean; refreshVersion?: number; onchange?: () => void } = $props(); - let hosts = $state([]); let libraries = $state([]); let loading = $state(true); let error = $state(''); + import { + latestGameFiles, + selectLatestGameFiles, + type NativeHostSelectionStatus + } from '$ui/native-game-files.js'; + let { + games, + selection = $bindable(), + // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-useless-assignment -- Written here and read through parent bindings. + ready = $bindable(false), + // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-useless-assignment -- Written here and read through parent bindings. + status = $bindable(), + disabled = false, + network = false, + compact = false, + refreshVersion = 0, + onchange + }: { + games: ZombieGame[]; + selection: NativeDeploymentSelection; + ready?: boolean; + status?: NativeHostSelectionStatus; + disabled?: boolean; + network?: boolean; + compact?: boolean; + refreshVersion?: number; + onchange?: () => void; + } = $props(); + let hosts = $state([]); + let libraries = $state([]); + let loading = $state(true); + let error = $state(''); let defaultRuntimeImage = $state(''); let runtimeSelection: NativeDeploymentSelection | null = null; let requestVersion = 0; let activeController: AbortController | undefined; - const eligible = $derived(hosts.filter(host => host.readiness === 'ready' && host.inventory.runtimeImages?.includes(selection.image))); + const eligible = $derived( + hosts.filter( + (host) => + host.readiness === 'ready' && + host.inventory.runtimeImages?.includes(selection.image) + ) + ); const latest = $derived(latestGameFiles(libraries)); - function cacheReady(game: ZombieGame) { return !!latest[game]?.caches.some(cache => eligible.some(host => host.id === cache.hostId) && ['ready', 'outdated'].includes(cache.status)); } - $effect(() => { ready = !loading && !error && /^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(selection.image) - && games.length > 0 && games.every(game => !!latest[game]?.id && selection.libraryIds[game] === latest[game]?.id); }); + function cacheReady(game: ZombieGame) { + return !!latest[game]?.caches.some( + (cache) => + eligible.some((host) => host.id === cache.hostId) && + ['ready', 'outdated'].includes(cache.status) + ); + } + $effect(() => { + ready = + !loading && + !error && + /^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(selection.image) && + games.length > 0 && + games.every( + (game) => !!latest[game]?.id && selection.libraryIds[game] === latest[game]?.id + ); + }); $effect(() => { status = { loading, error, hasHosts: hosts.length > 0, filesReady: games.length > 0 && games.every(cacheReady), - missingGames: games.filter(game => !latest[game]?.id), - uploadedAt: games.map(game => latest[game]?.publishedAt ?? latest[game]?.createdAt) - .filter((value): value is string => !!value).sort().at(-1) ?? null, - gameFiles: Object.fromEntries(games.map(game => [game, { ready: cacheReady(game), - uploadedAt: latest[game]?.publishedAt ?? latest[game]?.createdAt ?? null }])), + missingGames: games.filter((game) => !latest[game]?.id), + uploadedAt: + games + .map((game) => latest[game]?.publishedAt ?? latest[game]?.createdAt) + .filter((value): value is string => !!value) + .sort() + .at(-1) ?? null, + gameFiles: Object.fromEntries( + games.map((game) => [ + game, + { + ready: cacheReady(game), + uploadedAt: latest[game]?.publishedAt ?? latest[game]?.createdAt ?? null + } + ]) + ) }; }); $effect(() => { @@ -52,14 +111,24 @@ if (signal?.aborted) controller.abort(); else signal?.addEventListener('abort', abort, { once: true }); try { - const [hostResponse, libraryResponse] = await Promise.all([fetch('/api/hosts', { signal: controller.signal }), fetch('/api/game-files', { signal: controller.signal })]); - if (!hostResponse.ok || !libraryResponse.ok) throw new Error('Could not load hosts and game files.'); - const [hostBody, libraryBody] = await Promise.all([hostResponse.json(), libraryResponse.json()]); + const [hostResponse, libraryResponse] = await Promise.all([ + fetch('/api/hosts', { signal: controller.signal }), + fetch('/api/game-files', { signal: controller.signal }) + ]); + if (!hostResponse.ok || !libraryResponse.ok) + throw new Error('Could not load hosts and game files.'); + const [hostBody, libraryBody] = await Promise.all([ + hostResponse.json(), + libraryResponse.json() + ]); if (version !== requestVersion || controller.signal.aborted) return; - hosts = hostBody.hosts; libraries = libraryBody.libraries; error = ''; + hosts = hostBody.hosts; + libraries = libraryBody.libraries; + error = ''; defaultRuntimeImage = hostBody.defaultRuntimeImage ?? ''; } catch (cause) { - if (version === requestVersion && !controller.signal.aborted) error = (cause as Error).message; + if (version === requestVersion && !controller.signal.aborted) + error = (cause as Error).message; } finally { signal?.removeEventListener('abort', abort); if (version === requestVersion && !controller.signal.aborted) loading = false; @@ -67,7 +136,7 @@ } $effect(() => { // Restart the poll for a parent refresh, sharing one initial request. - refreshVersion; + void refreshVersion; loading = true; const stop = pollVisible(load, 5000); return () => { @@ -79,14 +148,45 @@ {#snippet runtimeFields()} - + {#if compact}

Custom images must be prepared on a host before it can run this server.

{/if} {/snippet} {#snippet networkFields()} - - - + + +

Public mapping does not change the local Docker port reservation.

{/snippet} @@ -95,39 +195,175 @@ Advanced settings
{@render runtimeFields()} - {#if network}
Network overrides{@render networkFields()}
{/if} + {#if network}
+ Network overrides{@render networkFields()} +
{/if}
{:else} -
-

Host and game files

- {#if error}{/if} - {#if loading}

Loading host readiness and game files…

{/if} -

Hosts are assigned when provisioning starts. You can queue launches while hosts are busy or preparing; each server waits for an eligible host and available capacity.

- {#if !loading && !hosts.length}

Add a host. Its runtime and latest Black Ops I and Black Ops II files will be prepared automatically.

{/if} - {#if !loading}{#each games as game} -
- {game === 't5' ? 'Black Ops I' : 'Black Ops II'} files - {#if latest[game]} -

Latest upload ·

-

{cacheReady(game) ? 'Latest files ready on an eligible host.' : 'Launches can be queued now and will wait for a host to prepare the runtime and latest files.'}

- {:else} -

No files uploaded yet. Upload {gameTitle(game, true)} files to prepare your hosts.

- {/if} +
+
+

Host and game files

+
- {/each}{/if} -

Hosts keep the latest Black Ops I and Black Ops II files ready automatically. Upload game files when they change. Existing servers keep their current files.

-
Advanced: runtime override

The configured default runtime is prepared automatically. Custom images must first be prepared by an admin from the destination host page.

{@render runtimeFields()}
- {#if network}
Network overrides{@render networkFields()}
{/if} -
+ {#if error}{/if} + {#if loading}

Loading host readiness and game files…

{/if} +

+ Hosts are assigned when provisioning starts. You can queue launches while hosts are busy + or preparing; each server waits for an eligible host and available capacity. +

+ {#if !loading && !hosts.length}

+ Add a host. Its runtime and + latest Black Ops I and Black Ops II files will be prepared automatically. +

{/if} + {#if !loading}{#each games as game} +
+ {game === 't5' ? 'Black Ops I' : 'Black Ops II'} files + {#if latest[game]} +

+ Latest upload · +

+

+ {cacheReady(game) + ? 'Latest files ready on an eligible host.' + : 'Launches can be queued now and will wait for a host to prepare the runtime and latest files.'} +

+ {:else} +

+ No files uploaded yet. Upload {gameTitle(game, true)} files to prepare your hosts. +

+ {/if} +
+ {/each}{/if} +

+ Hosts keep the latest Black Ops I and Black Ops II files ready automatically. Upload game files when they change. Existing servers keep their current files. +

+
+ Advanced: runtime override +

+ The configured default runtime is prepared automatically. Custom images must first + be prepared by an admin from the destination host page. +

+ {@render runtimeFields()} +
+ {#if network}
+ Network overrides{@render networkFields()} +
{/if} +
{/if} diff --git a/src/lib/ui/components/Navigation.svelte b/src/lib/ui/components/Navigation.svelte index a37d9c1..9350977 100644 --- a/src/lib/ui/components/Navigation.svelte +++ b/src/lib/ui/components/Navigation.svelte @@ -11,12 +11,32 @@ const groups = $derived(visibleNavigation(role)); const attention = $derived.by(() => { if (!$fleet.receivedAt) return null; - const statuses = $fleetServers.map(server => ({ server, state: serverHealth(server, $uiNow).state })); - const servers = statuses.filter(({ server, state }) => !['online', 'starting', 'provisioning'].includes(state) - || $fleet.operations.some(operation => operation.server === server.name && operation.state === 'attention')).length; - const hosts = $fleet.hostAttention ?? new Set(statuses.filter(({ server, state }) => server.host && state === 'host-offline').map(({ server }) => server.host!.id)).size; - const maintenance = $fleet.operations.filter(operation => operation.kind === 'maintenance' && operation.state === 'attention').length; - return { '/': servers, '/hosts': hosts, '/maintenance': maintenance } as Record; + const statuses = $fleetServers.map((server) => ({ + server, + state: serverHealth(server, $uiNow).state + })); + const servers = statuses.filter( + ({ server, state }) => + !['online', 'starting', 'provisioning'].includes(state) || + $fleet.operations.some( + (operation) => + operation.server === server.name && operation.state === 'attention' + ) + ).length; + const hosts = + $fleet.hostAttention ?? + new Set( + statuses + .filter(({ server, state }) => server.host && state === 'host-offline') + .map(({ server }) => server.host!.id) + ).size; + const maintenance = $fleet.operations.filter( + (operation) => operation.kind === 'maintenance' && operation.state === 'attention' + ).length; + return { '/': servers, '/hosts': hosts, '/maintenance': maintenance } as Record< + string, + number + >; }); @@ -24,25 +44,86 @@ {#each groups as group (group.id)}

{group.label}

- +
{/each} diff --git a/src/lib/ui/components/OperationHistory.svelte b/src/lib/ui/components/OperationHistory.svelte index 35f71a2..8836bcc 100644 --- a/src/lib/ui/components/OperationHistory.svelte +++ b/src/lib/ui/components/OperationHistory.svelte @@ -3,72 +3,228 @@ import type { ActivityEvent, OperationStep } from '$lib/operation-flow.js'; import Button from './primitives/Button.svelte'; import type { Snippet } from 'svelte'; - let { steps = [], events, active = false, showOverview = true, footer }: { steps?: OperationStep[]; events: ActivityEvent[]; active?: boolean; showOverview?: boolean; footer?: Snippet } = $props(); + let { + steps = [], + events, + active = false, + showOverview = true, + footer + }: { + steps?: OperationStep[]; + events: ActivityEvent[]; + active?: boolean; + showOverview?: boolean; + footer?: Snippet; + } = $props(); let limit = $state(25); let query = $state(''); let newestFirst = $state(true); const latest = $derived(events.at(-1)); - const matching = $derived(events.filter(event => event.message.toLowerCase().includes(query.toLowerCase()))); - const visible = $derived(newestFirst ? matching.slice(-limit).reverse() : matching.slice(0, limit)); + const matching = $derived( + events.filter((event) => event.message.toLowerCase().includes(query.toLowerCase())) + ); + const visible = $derived( + newestFirst ? matching.slice(-limit).reverse() : matching.slice(0, limit) + ); {#if showOverview} -
-

Operation flow

-
    - {#each steps as step, index} -
  1. - {index + 1}{step.label} - ({step.state}) -
    2. - {/each} -
-
-{/if} -
- {#if showOverview && latest}

Latest update {latest.message}

{/if} -
- Activity history · {events.length} events{active ? ' · updates automatically' : ''} - {#if events.length} -
-
    - {#each visible as event (event.id)} -

  1. {event.message}

    2. +
    +

    Operation flow

    +
      + {#each steps as step, index} +
    1. + {index + 1}{step.label} + ({step.state}) +
      2. {/each}
    - {#if !matching.length}

    No matching activity.

    {/if} - {#if matching.length > limit}{/if} - {:else} -

    This operation predates detailed activity tracking. Its recorded status and timestamps are shown on this page.

    - {/if} - {@render footer?.()} +
    +{/if} +
    + {#if showOverview && latest}

    + Latest update + {latest.message} +

    {/if} +
    + Activity history · {events.length} events{active + ? ' · updates automatically' + : ''} + {#if events.length} +
    + +
    +
      + {#each visible as event (event.id)} +
    1. + +

      {event.message}

      +
      2. + {/each} +
    + {#if !matching.length}

    No matching activity.

    {/if} + {#if matching.length > limit}{/if} + {:else} +

    + This operation predates detailed activity tracking. Its recorded status and + timestamps are shown on this page. +

    + {/if} + {@render footer?.()}
    diff --git a/src/lib/ui/components/OperationTime.svelte b/src/lib/ui/components/OperationTime.svelte index f1c9840..92266cd 100644 --- a/src/lib/ui/components/OperationTime.svelte +++ b/src/lib/ui/components/OperationTime.svelte @@ -3,14 +3,25 @@ import { formatTimeOffset } from '$ui/duration.js'; import { timeFormat } from '$ui/stores/time-zone.js'; - let { value, start, missing = 'Not recorded' }: { value: Timestamp; start: Timestamp; missing?: string } = $props(); + let { + value, + start, + missing = 'Not recorded' + }: { value: Timestamp; start: Timestamp; missing?: string } = $props(); const iso = $derived(timestampISO(value)); const offset = $derived(formatTimeOffset(start, value)); const absolute = $derived($timeFormat.dateTime(value)); -{#if iso}{:else}{missing}{/if} +{#if iso}{:else}{missing}{/if} diff --git a/src/lib/ui/components/PipelineTimeline.svelte b/src/lib/ui/components/PipelineTimeline.svelte index be716f8..4b37ead 100644 --- a/src/lib/ui/components/PipelineTimeline.svelte +++ b/src/lib/ui/components/PipelineTimeline.svelte @@ -25,15 +25,20 @@
    {dep.action || dep.id} - {dep.servers.size} server{dep.servers.size === 1 ? '' : 's'} + {dep.servers.size} server{dep.servers.size === 1 ? '' : 's'}
    {#each [...dep.servers.entries()] as [srv, meta] (srv)} - {srv}{meta.players !== null && meta.players > 0 ? ` ${meta.players}p` : ''} + {srv}{meta.players !== null && meta.players > 0 + ? ` ${meta.players}p` + : ''} {/each}
    @@ -72,7 +77,10 @@ margin: 0; padding: 14px 0; } - .rows { display: grid; gap: 10px; } + .rows { + display: grid; + gap: 10px; + } .row { display: block; color: var(--text-primary); @@ -83,16 +91,31 @@ border: 1px solid var(--border-subtle); transition: border-color 0.15s; } - .row:hover { border-color: var(--accent-primary); } + .row:hover { + border-color: var(--accent-primary); + } .row-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 8px; } - .action { font-family: var(--font-body); font-size: var(--text-sm); color: var(--text-primary); } - .count { color: var(--text-tertiary); font-size: var(--text-xs); font-family: var(--font-numeric); font-variant-numeric: tabular-nums; } - .chips { display: flex; flex-wrap: wrap; gap: 6px; } + .action { + font-family: var(--font-body); + font-size: var(--text-sm); + color: var(--text-primary); + } + .count { + color: var(--text-tertiary); + font-size: var(--text-xs); + font-family: var(--font-numeric); + font-variant-numeric: tabular-nums; + } + .chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + } .chip { padding: 3px 9px; border-radius: var(--radius-sm); @@ -116,7 +139,7 @@ color: var(--state-warning); } .chip-held { - background: rgba(124, 192, 245, 0.10); + background: rgba(124, 192, 245, 0.1); border-color: rgba(124, 192, 245, 0.28); color: var(--text-primary); } diff --git a/src/lib/ui/components/PlayerMapCards.svelte b/src/lib/ui/components/PlayerMapCards.svelte index eb342c4..5040dcd 100644 --- a/src/lib/ui/components/PlayerMapCards.svelte +++ b/src/lib/ui/components/PlayerMapCards.svelte @@ -2,27 +2,111 @@ import { type PlayerMapRecord, type LeaderboardMetric } from '$lib/leaderboard.js'; import { contextHref } from '$lib/navigation-context.js'; import { timeFormat } from '$ui/stores/time-zone.js'; - let { maps, metric = 'highestRound', returnTo }: { maps: PlayerMapRecord[]; metric?: LeaderboardMetric; returnTo?: string } = $props(); + let { + maps, + metric = 'highestRound', + returnTo + }: { maps: PlayerMapRecord[]; metric?: LeaderboardMetric; returnTo?: string } = $props(); const number = new Intl.NumberFormat('en-US'); - const records = [{ key: 'highestRound', label: 'Best round' }, { key: 'bestKills', label: 'Best kills' }, { key: 'bestScore', label: 'Best score' }] as const; + const records = [ + { key: 'highestRound', label: 'Best round' }, + { key: 'bestKills', label: 'Best kills' }, + { key: 'bestScore', label: 'Best score' } + ] as const; +
    {#each maps as map (`${map.game}:${map.key}`)}
    -

    {map.name}

    {map.game === 't5' ? 'Black Ops I' : 'Black Ops II'} · {number.format(map.gamesPlayed)} {map.gamesPlayed === 1 ? 'game' : 'games'}
    -
    {#each records as record}
    {record.label}
    {number.format(map[record.key])}Rank #{number.format(map.ranks[record.key])}
    {/each}
    -

    {number.format(map.kills)} total kills · {number.format(map.roundsSurvived)} observed rounds

    +
    +

    + {map.name} +

    + {map.game === 't5' ? 'Black Ops I' : 'Black Ops II'} · {number.format( + map.gamesPlayed + )} + {map.gamesPlayed === 1 ? 'game' : 'games'} +
    +
    + {#each records as record}
    +
    {record.label}
    +
    + {number.format(map[record.key])}Rank #{number.format(map.ranks[record.key])} +
    +
    {/each} +
    +

    + {number.format(map.kills)} total kills · {number.format(map.roundsSurvived)} observed + rounds +

    {/each}
    + diff --git a/src/lib/ui/components/PlayerMatchCards.svelte b/src/lib/ui/components/PlayerMatchCards.svelte index 0922392..2fbdd32 100644 --- a/src/lib/ui/components/PlayerMatchCards.svelte +++ b/src/lib/ui/components/PlayerMatchCards.svelte @@ -7,22 +7,75 @@ let { matches, returnTo }: { matches: PlayerMatch[]; returnTo?: string } = $props(); const number = new Intl.NumberFormat('en-US'); +
    {#each matches as match (match.id)}
    -

    {#if match.status === 'completed'}{match.mapName} · Match #{match.id}{:else}{match.mapName} · Match #{match.id}{/if}

    +

    + {#if match.status === 'completed'}{match.mapName} · Match #{match.id}{:else}{match.mapName} · Match #{match.id}{/if} +

    {serverTitle(match.server)}{match.server_deleted_at ? ' · Removed server' : ''}

    · {playerCompletionLabel(match)}

    -
    Round
    {number.format(match.highestRound)}
    Kills
    {number.format(match.kills)}
    Revives
    {number.format(match.revives)}
    +
    +
    +
    Round
    +
    {number.format(match.highestRound)}
    +
    +
    +
    Kills
    +
    {number.format(match.kills)}
    +
    +
    +
    Revives
    +
    {number.format(match.revives)}
    +
    +
    {/each}
    + diff --git a/src/lib/ui/components/PlayerScoreboard.svelte b/src/lib/ui/components/PlayerScoreboard.svelte index 37c8ee2..faac0be 100644 --- a/src/lib/ui/components/PlayerScoreboard.svelte +++ b/src/lib/ui/components/PlayerScoreboard.svelte @@ -10,12 +10,32 @@ import Badge from './primitives/Badge.svelte'; import EmptyState from './primitives/EmptyState.svelte'; - let { report, mapLabel, current = true, after = null, matchElapsed = null, activity }: { report: GameTelemetry | undefined; mapLabel?: string; current?: boolean; after?: string | null; matchElapsed?: string | null; activity?: string } = $props(); + let { + report, + mapLabel, + current = true, + after = null, + matchElapsed = null, + activity + }: { + report: GameTelemetry | undefined; + mapLabel?: string; + current?: boolean; + after?: string | null; + matchElapsed?: string | null; + activity?: string; + } = $props(); const fresh = $derived(current && freshTelemetry(report, $uiNow, after)); - const timingNow = $derived(report && (report.event === 'game_end' || !fresh) - ? Date.parse(report.receivedAt) : $uiNow); - const displayElapsed = $derived(fresh ? matchElapsed : report?.timing?.server?.gameStartedAt && !report.timing.server.gameEndedAt - ? elapsedSince(report.timing.server.gameStartedAt, timingNow) : null); + const timingNow = $derived( + report && (report.event === 'game_end' || !fresh) ? Date.parse(report.receivedAt) : $uiNow + ); + const displayElapsed = $derived( + fresh + ? matchElapsed + : report?.timing?.server?.gameStartedAt && !report.timing.server.gameEndedAt + ? elapsedSince(report.timing.server.gameStartedAt, timingNow) + : null + ); const columns = [ { key: 'score', label: 'Score' }, { key: 'kills', label: 'Kills' }, @@ -23,19 +43,30 @@ { key: 'downs', label: 'Downs' }, { key: 'revives', label: 'Revives' } ] as const; - type StatKey = typeof columns[number]['key']; + type StatKey = (typeof columns)[number]['key']; let sortBy = $state('score'); let descending = $state(true); let expanded = $state([]); const uid = $props.id(); const number = new Intl.NumberFormat('en-US'); - const players = $derived(report?.event === 'game_start' - ? report.players.map(player => ({ ...player, score: null, kills: null, headshots: null, downs: null, revives: null })) - : report?.players ?? []); - const sorted = $derived([...players].sort((a, b) => { - const difference = (a[sortBy] ?? 0) - (b[sortBy] ?? 0); - return (descending ? -difference : difference) || a.name.localeCompare(b.name); - })); + const players = $derived( + report?.event === 'game_start' + ? report.players.map((player) => ({ + ...player, + score: null, + kills: null, + headshots: null, + downs: null, + revives: null + })) + : (report?.players ?? []) + ); + const sorted = $derived( + [...players].sort((a, b) => { + const difference = (a[sortBy] ?? 0) - (b[sortBy] ?? 0); + return (descending ? -difference : difference) || a.name.localeCompare(b.name); + }) + ); function sort(key: StatKey) { descending = sortBy === key ? !descending : true; @@ -44,21 +75,36 @@ {#if !report} - + {:else}
    - {report.event === 'game_end' ? 'Final scoreboard' : fresh ? 'Live scoreboard' : 'Last reported scoreboard'} + {report.event === 'game_end' + ? 'Final scoreboard' + : fresh + ? 'Live scoreboard' + : 'Last reported scoreboard'}

    {mapLabel ?? mapTitle(report.map)}

    - {#if report.event !== 'game_start'}Round {report.round}{/if} - {#if displayElapsed}Match {displayElapsed}{fresh ? '' : ' at last report'}{/if} + {#if report.event !== 'game_start'}Round {report.round}{/if} + {#if displayElapsed}Match {displayElapsed}{fresh ? '' : ' at last report'}{/if} {#if report.event === 'game_end'}Game ended {:else if report.event === 'game_start'}Game started{/if}
    - {players.length} {players.length === 1 ? 'player' : 'players'}{fresh ? '' : ' in last report'} + {players.length} + {players.length === 1 ? 'player' : 'players'}{fresh ? '' : ' in last report'} {#if !fresh}Historical report{/if}
    {#if !players.length} - + {:else}
    - + {#each columns as column (column.key)} - {/each} @@ -91,73 +161,301 @@ {@const timing = report.timing?.players[player.guid]} {#each columns as column (column.key)} {/each} - + {/each}
    {report.event === 'game_end' ? 'Final' : 'Latest'} player scores for {report.server}{report.event === 'game_end' ? 'Final' : 'Latest'} player scores for {report.server}
    Player - +
    - {player.name} - + {player.name} + - {#if player[column.key] === null}— + {#if player[column.key] === null}— {:else}{number.format(player[column.key]!)}{/if}
    {/if}
    -
    About these stats

    {report.event === 'game_start' ? 'Player stats arrive with the next round report or heartbeat.' : report.event === 'game_end' ? 'Final stats from this game. The scoreboard resets when the next game starts.' : 'Stats update at each round change, heartbeat, and game end.'}

    Times and joining rounds are estimated from player reports. Time on instance carries across matches; ended or stale reports freeze at the last update.

    {#if activity}

    {activity}

    {/if}
    +
    + About these stats +

    + {report.event === 'game_start' + ? 'Player stats arrive with the next round report or heartbeat.' + : report.event === 'game_end' + ? 'Final stats from this game. The scoreboard resets when the next game starts.' + : 'Stats update at each round change, heartbeat, and game end.'} +

    +

    + Times and joining rounds are estimated from player reports. Time on instance carries + across matches; ended or stale reports freeze at the last update. +

    + {#if activity}

    {activity}

    {/if} +
    View lifetime leaderboard →
    {/if} diff --git a/src/lib/ui/components/PluginGithubImport.svelte b/src/lib/ui/components/PluginGithubImport.svelte index b3bb94e..06a7d8e 100644 --- a/src/lib/ui/components/PluginGithubImport.svelte +++ b/src/lib/ui/components/PluginGithubImport.svelte @@ -7,8 +7,22 @@ import Button from './primitives/Button.svelte'; import Badge from './primitives/Badge.svelte'; - let { pluginId, targetPath, disabled = false, busy = $bindable(false), onimport }: { - pluginId?: number; targetPath?: string; disabled?: boolean; busy?: boolean; onimport?: (result: { pluginId: number; betaUrl: string; alreadyImported: boolean }) => void | Promise; + let { + pluginId, + targetPath, + disabled = false, + busy = $bindable(false), + onimport + }: { + pluginId?: number; + targetPath?: string; + disabled?: boolean; + busy?: boolean; + onimport?: (result: { + pluginId: number; + betaUrl: string; + alreadyImported: boolean; + }) => void | Promise; } = $props(); let url = $state(''); let name = $state(''); @@ -18,55 +32,89 @@ let update = $state(null); let checking = $state(false); let error = $state(''); - let success = $state<{ pluginId: number; betaUrl: string; sha256: string; alreadyImported: boolean } | null>(null); + let success = $state<{ + pluginId: number; + betaUrl: string; + sha256: string; + alreadyImported: boolean; + } | null>(null); let active = true; - onDestroy(() => { active = false; }); + onDestroy(() => { + active = false; + }); function showRelease(value: GithubPluginRelease) { release = value; const preferred = update?.source?.asset_name ?? targetPath?.split('/').pop(); - assetId = String(value.assets.find(a => a.name === preferred)?.id ?? (value.assets.length === 1 ? value.assets[0].id : '')); + assetId = String( + value.assets.find((a) => a.name === preferred)?.id ?? + (value.assets.length === 1 ? value.assets[0].id : '') + ); if (!pluginId && !name) name = value.repository.split('/')[1]; url = value.url; - error = ''; success = null; + error = ''; + success = null; } async function checkUpdates() { if (!pluginId || checking) return; - checking = true; error = ''; + checking = true; + error = ''; try { const response = await fetch(`/api/plugins/updates?pluginId=${pluginId}`); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not check for updates.'); update = body.updates[0]; if (!url && update?.source) url = `https://github.com/${update.source.repository}`; - } catch (e) { error = (e as Error).message; } - finally { checking = false; } + } catch (e) { + error = (e as Error).message; + } finally { + checking = false; + } } - onMount(() => { void checkUpdates(); }); + onMount(() => { + void checkUpdates(); + }); async function preview(event: SubmitEvent) { event.preventDefault(); if (busy || disabled) return; - busy = true; error = ''; release = null; success = null; + busy = true; + error = ''; + release = null; + success = null; try { - const response = await fetch(`/api/plugins/github/release?url=${encodeURIComponent(url.trim())}`); + const response = await fetch( + `/api/plugins/github/release?url=${encodeURIComponent(url.trim())}` + ); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load release.'); showRelease(body.release); - } catch (e) { error = (e as Error).message; } - finally { busy = false; } + } catch (e) { + error = (e as Error).message; + } finally { + busy = false; + } } async function importRelease(startBeta: boolean) { if (!release || !assetId || busy || disabled) return; const startPath = window.location.pathname; - busy = true; error = ''; success = null; + busy = true; + error = ''; + success = null; try { const response = await fetch('/api/plugins/github/import', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ url: release.url, assetId: Number(assetId), pluginId, - name: name.trim(), scopeType: game ? 'game' : null, scopeValue: game || null }) + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: release.url, + assetId: Number(assetId), + pluginId, + name: name.trim(), + scopeType: game ? 'game' : null, + scopeValue: game || null + }) }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Import failed.'); @@ -75,29 +123,57 @@ await onimport?.(body); if (startBeta && window.location.pathname === startPath) await goto(body.betaUrl); else if (pluginId) await checkUpdates(); - } catch (e) { error = (e as Error).message; } - finally { busy = false; } + } catch (e) { + error = (e as Error).message; + } finally { + busy = false; + } }
    -

    {pluginId ? 'GitHub releases' : 'Import from GitHub'}

    Choose a release and DLL, then test it with your fleet.

    - {#if pluginId}{/if} +
    +

    {pluginId ? 'GitHub releases' : 'Import from GitHub'}

    +

    Choose a release and DLL, then test it with your fleet.

    +
    + {#if pluginId}{/if}
    {#if checking}

    Checking GitHub…

    {/if} {#if update?.source} -

    Following {update.source.repository} · {update.source.asset_name}

    +

    + Following {update.source.repository} + · {update.source.asset_name} +

    {/if} {#if update?.status === 'available' && update.release}
    New release: {update.release.tag} - +
    {:else if update?.status === 'imported' && update.release} -
    {update.release.tag} is already imported and available for testing. -
    +
    + {update.release.tag} is already imported and available for testing. + +
    {:else if update?.status === 'current'}

    The latest stable release is promoted.

    {:else if update?.status === 'ahead'} @@ -108,44 +184,105 @@

    Import a release below to track updates for this plugin.

    {/if}
    -
{/if} + {#if loaded}
+ Showing {entries.length} of {total} matching events{#if cursor !== null}{/if} +
{/if}
diff --git a/src/lib/ui/components/ProvisioningProgress.svelte b/src/lib/ui/components/ProvisioningProgress.svelte index f86a37b..64e625c 100644 --- a/src/lib/ui/components/ProvisioningProgress.svelte +++ b/src/lib/ui/components/ProvisioningProgress.svelte @@ -1,86 +1,245 @@
-
Execution

Progress and evidence

+
+
+ Execution +

Progress and evidence

+
+
{#if !progress} -

{detail.deployment.stage}

Detailed checkpoints were not recorded for this operation. Its recorded outcome and original activity remain available below.

+
+

{detail.deployment.stage}

+

+ Detailed checkpoints were not recorded for this operation. Its recorded outcome and + original activity remain available below. +

+
{/if} {#if groups.length}
    {#each groups as group (group.key)} - {@const defaultOpen = detail.deployment.state !== 'done' || group.steps.some(step => step.state === 'failed' || step.checks.some(check => check.result !== 'pass'))} -
  1. + {@const defaultOpen = + detail.deployment.state !== 'done' || + group.steps.some( + (step) => + step.state === 'failed' || + step.checks.some((check) => check.result !== 'pass') + )} +
  2. { event.preventDefault(); expanded[group.key] = !(expanded[group.key] ?? defaultOpen); }} - onkeydown={event => { + onclick={(event) => { + event.preventDefault(); + expanded[group.key] = !(expanded[group.key] ?? defaultOpen); + }} + onkeydown={(event) => { if (event.key !== 'Enter' && event.key !== ' ') return; event.preventDefault(); - if (!event.repeat) expanded[group.key] = !(expanded[group.key] ?? defaultOpen); + if (!event.repeat) + expanded[group.key] = !(expanded[group.key] ?? defaultOpen); }} > - + {group.label} - {progressStateLabels[group.state]} + {progressStateLabels[group.state]}
      {#each group.steps as step (step.key)} -
    1. - +
    2. +
      {step.label} - {progressStateLabels[step.state]}{#if step.at}{' · '}{/if} + {progressStateLabels[ + step.state + ]}{#if step.at}{' · '}{/if} {#each step.checks as check (check.key)}
      - {#if step.checks.length > 1 || check.key === 'plutonium-release'}

      {check.label}{' · '}{evidenceLabels[check.result]}

      {/if} + {#if step.checks.length > 1 || check.key === 'plutonium-release'}

      + {check.label}{' · '}{evidenceLabels[ + check.result + ]} +

      {/if}
      - {#if check.expected !== null}
      Expected
      {provisioningCheckValue(check.key, check.expected)}
      {/if} -
      Observed
      {provisioningCheckValue(check.key, check.actual)}
      + {#if check.expected !== null}
      +
      Expected
      +
      + {provisioningCheckValue( + check.key, + check.expected + )} +
      +
      {/if} +
      +
      Observed
      +
      + {provisioningCheckValue( + check.key, + check.actual + )} +
      +
      {#if check.message}

      {check.message}

      {/if}
      - {check.source ? sources[check.source] : 'Source not recorded'} - {#if check.checkedAt !== step.at}Checked {/if} - {#if check.receivedAt}Evidence received {/if} + {check.source + ? sources[check.source] + : 'Source not recorded'} + {#if check.checkedAt !== step.at}Checked {/if} + {#if check.receivedAt}Evidence received {/if}
      {/each} - {#if step.key === 'verify-map' && detail.deployment.expected_map}

      Engine-map verification does not independently verify a variant or start location.

      {/if} + {#if step.key === 'verify-map' && detail.deployment.expected_map}

      + Engine-map verification does not independently + verify a variant or start location. +

      {/if} {#if step.command} -
      -

      {step.command.state === 'attempted' ? 'Command submission attempted' : step.command.state === 'accepted' ? 'Remote request accepted' : step.command.state === 'rejected' ? 'Remote request rejected' : 'Remote response uncertain'}

      +
      +

      + {step.command.state === 'attempted' + ? 'Command submission attempted' + : step.command.state === 'accepted' + ? 'Remote request accepted' + : step.command.state === 'rejected' + ? 'Remote request rejected' + : 'Remote response uncertain'} +

      {step.command.message}

      - +
      {/if} {#if step.agentOperation}
      -
      Agent state
      {step.agentOperation.state}
      -
      Command queued
      -
      Latest agent report
      +
      +
      Agent state
      +
      {step.agentOperation.state}
      +
      +
      +
      Command queued
      +
      + +
      +
      +
      +
      Latest agent report
      +
      + +
      +
      - {#if step.agentOperation.error}

      {step.agentOperation.error}

      {/if} - View host operation history + {#if step.agentOperation.error}

      + {step.agentOperation.error} +

      {/if} + View host operation history
      {/if} {#if step.checkpoints.length}
        - {#each step.checkpoints as checkpoint (checkpoint.key)} -
      • {checkpoint.label}
        • - {/each} -
      {/if} + {#each step.checkpoints as checkpoint (checkpoint.key)} +
    3. + {checkpoint.label} +
      4. + {/each} + {/if} {/each} @@ -93,50 +252,262 @@
diff --git a/src/lib/ui/components/RecordSource.svelte b/src/lib/ui/components/RecordSource.svelte index 7107ba9..db4d3ca 100644 --- a/src/lib/ui/components/RecordSource.svelte +++ b/src/lib/ui/components/RecordSource.svelte @@ -2,10 +2,42 @@ import type { RecordEvidence } from '$lib/leaderboard.js'; import { timeFormat } from '$ui/stores/time-zone.js'; import { contextHref } from '$lib/navigation-context.js'; - let { evidence, returnTo, metric = 'Record' }: { evidence: RecordEvidence; returnTo?: string; metric?: string; disclosure?: boolean } = $props(); + let { + evidence, + returnTo, + metric = 'Record' + }: { + evidence: RecordEvidence; + returnTo?: string; + metric?: string; + disclosure?: boolean; + } = $props(); -Match #{evidence.matchId} · {evidence.mapName} + +Match #{evidence.matchId} · {evidence.mapName} + diff --git a/src/lib/ui/components/RelativeTime.svelte b/src/lib/ui/components/RelativeTime.svelte index 2e38b97..b17a1d2 100644 --- a/src/lib/ui/components/RelativeTime.svelte +++ b/src/lib/ui/components/RelativeTime.svelte @@ -4,9 +4,15 @@ import { timeFormat } from '$ui/stores/time-zone.js'; import { uiNow } from '$ui/stores/servers.js'; - let { value, missing = '—', class: className }: { value: Timestamp; missing?: string; class?: string } = $props(); + let { + value, + missing = '—', + class: className + }: { value: Timestamp; missing?: string; class?: string } = $props(); const iso = $derived(timestampISO(value)); const now = $derived(browser ? $uiNow : Date.now()); -{#if iso}{:else}{missing}{/if} +{#if iso}{:else}{missing}{/if} diff --git a/src/lib/ui/components/RetentionTools.svelte b/src/lib/ui/components/RetentionTools.svelte index 98a56e8..200c2d5 100644 --- a/src/lib/ui/components/RetentionTools.svelte +++ b/src/lib/ui/components/RetentionTools.svelte @@ -5,63 +5,206 @@ let busy = $state(false); let error = $state(''); let notice = $state(''); - let preview = $state<{ policies: { category: string; label: string; eligible: number; payloadBytes: number; records: { id: string }[] }[]; - controls: { category: string; id: string; protected: number; resolvedAt: string | null }[] } | null>(null); + let preview = $state<{ + policies: { + category: string; + label: string; + eligible: number; + payloadBytes: number; + records: { id: string }[]; + }[]; + controls: { category: string; id: string; protected: number; resolvedAt: string | null }[]; + } | null>(null); let category = $state('campaign'); let recordId = $state(''); - const categories = ['campaign','provisioning','operation','observation','maintenance','recovery','match','beta','file-content','beta-file-content','version','audit','game-file','upload']; + const categories = [ + 'campaign', + 'provisioning', + 'operation', + 'observation', + 'maintenance', + 'recovery', + 'match', + 'beta', + 'file-content', + 'beta-file-content', + 'version', + 'audit', + 'game-file', + 'upload' + ]; async function refresh() { - busy = true; error = ''; - try { const response = await fetch('/api/settings/retention/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(settings) }); if (!response.ok) throw new Error('Could not load cleanup preview.'); preview = await response.json(); } - catch (cause) { error = (cause as Error).message; } finally { busy = false; } - } - async function control(action: { protected?: boolean; resolve?: boolean }, selectedCategory = category, id = recordId) { - busy = true; error = ''; notice = ''; + busy = true; + error = ''; try { - const response = await fetch('/api/settings/retention',{ method: 'POST',headers: { 'Content-Type': 'application/json' },body: JSON.stringify({ category: selectedCategory,id,...action }) }); - const result = await response.json(); if (!response.ok) throw new Error(result.error); - notice = action.resolve ? 'Failure marked resolved. Retention still checks all recovery dependencies.' : action.protected ? 'Record protected from cleanup.' : 'Record protection removed.'; + const response = await fetch('/api/settings/retention/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(settings) + }); + if (!response.ok) throw new Error('Could not load cleanup preview.'); + preview = await response.json(); + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } + } + async function control( + action: { protected?: boolean; resolve?: boolean }, + selectedCategory = category, + id = recordId + ) { + busy = true; + error = ''; + notice = ''; + try { + const response = await fetch('/api/settings/retention', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ category: selectedCategory, id, ...action }) + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error); + notice = action.resolve + ? 'Failure marked resolved. Retention still checks all recovery dependencies.' + : action.protected + ? 'Record protected from cleanup.' + : 'Record protection removed.'; await refresh(); - } catch (cause) { error = (cause as Error).message; } finally { busy = false; } + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } } +
Cleanup preview and protected records -

Preview uses the values currently entered above, without saving them. Background cleanup runs in batches. Known payload sizes are estimates; database file size may not shrink immediately. Expired data cannot be restored by increasing retention.

- +

+ Preview uses the values currently entered above, without saving them. Background cleanup + runs in batches. Known payload sizes are estimates; database file size may not shrink + immediately. Expired data cannot be restored by increasing retention. +

+ {#if error}{/if} {#if notice}

{notice}

{/if} {#if preview} -
- {#each preview.policies as policy}{/each} -
CategoryEligible recordsKnown payload size
{policy.label}{policy.eligible}{policy.payloadBytes ? `${(policy.payloadBytes / 1024 / 1024).toFixed(2)} MiB` : '—'}
- {#each preview.policies.filter(policy => policy.records.length) as policy} -
{policy.label}: eligible IDs (up to 20) - {#each policy.records as record}
{record.id}
{/each} +
+ + + {#each preview.policies as policy}{/each} + +
CategoryEligible recordsKnown payload size
{policy.label}{policy.eligible}{policy.payloadBytes + ? `${(policy.payloadBytes / 1024 / 1024).toFixed(2)} MiB` + : '—'}
+
+ {#each preview.policies.filter((policy) => policy.records.length) as policy} +
+ {policy.label}: eligible IDs (up to 20) + {#each policy.records as record}
+ {record.id} +
{/each}
{/each}

Protected records and resolved failures

- {#each preview.controls as record}
{record.category} · {record.id}{record.resolvedAt ? ' · Resolved' : ''} - {#if record.protected}{/if}
{:else}

No explicit record protections or resolutions.

{/each} + {#each preview.controls as record}
+ {record.category} · {record.id}{record.resolvedAt + ? ' · Resolved' + : ''} + {#if record.protected}{/if} +
{:else}

No explicit record protections or resolutions.

{/each} {/if}

Protect a record or resolve a failure

-

Use the ID from its detail URL or the preview. Resolving a failed campaign stops automatic retry requests for that campaign; unfinished recovery still prevents cleanup.

+

+ Use the ID from its detail URL or the preview. Resolving a failed campaign stops automatic + retry requests for that campaign; unfinished recovery still prevents cleanup. +

- + - - {#if ['campaign','provisioning','operation'].includes(category)}{/if} + + {#if ['campaign', 'provisioning', 'operation'].includes(category)}{/if}
+ diff --git a/src/lib/ui/components/RoundTelemetry.svelte b/src/lib/ui/components/RoundTelemetry.svelte index b95d642..33d34c0 100644 --- a/src/lib/ui/components/RoundTelemetry.svelte +++ b/src/lib/ui/components/RoundTelemetry.svelte @@ -4,21 +4,42 @@ import { freshStatusTimestamp } from '$lib/server-freshness.js'; import { elapsedSince } from '$ui/duration.js'; import type { GameTelemetry } from '$lib/telemetry.js'; - let { name, since = null, report: observation }: { name: string; since?: string | null; report?: GameTelemetry } = $props(); + let { + name, + since = null, + report: observation + }: { name: string; since?: string | null; report?: GameTelemetry } = $props(); let report = $derived(observation ?? $telemetryByServer.get(name)); - let fresh = $derived(report && freshStatusTimestamp(report.receivedAt, $uiNow) - && (!since || Date.parse(report.receivedAt) >= Date.parse(since))); + let fresh = $derived( + report && + freshStatusTimestamp(report.receivedAt, $uiNow) && + (!since || Date.parse(report.receivedAt) >= Date.parse(since)) + ); {#if report && (report.event === 'round_change' || report.event === 'heartbeat') && fresh}
Round {report.round} - Last reported {elapsedSince(report.receivedAt, $uiNow)} ago + Last reported {elapsedSince(report.receivedAt, $uiNow)} ago
{/if} diff --git a/src/lib/ui/components/RuntimeVersion.svelte b/src/lib/ui/components/RuntimeVersion.svelte index b459e4f..7c047e9 100644 --- a/src/lib/ui/components/RuntimeVersion.svelte +++ b/src/lib/ui/components/RuntimeVersion.svelte @@ -1,18 +1,46 @@ - + Plutonium {revision ? `r${revision}` : 'unverified'}{outdated ? ' · Update available' : ''} diff --git a/src/lib/ui/components/ScriptUploadDrawer.svelte b/src/lib/ui/components/ScriptUploadDrawer.svelte index 372f594..b3298d0 100644 --- a/src/lib/ui/components/ScriptUploadDrawer.svelte +++ b/src/lib/ui/components/ScriptUploadDrawer.svelte @@ -4,12 +4,29 @@ import { plural } from '$ui/plural.js'; import { gameTitle } from '$ui/server-labels.js'; import Button from './primitives/Button.svelte'; - import { betaSelectionHref, mapLabel, scopeLabel, type BetaSelection, type LibraryScript } from '$lib/script-library.js'; - let { scripts, onuploaded }: { scripts: LibraryScript[]; onuploaded: () => Promise } = $props(); - interface Duplicate { scriptId: number; versionId: number; version: number; } + import { + betaSelectionHref, + mapLabel, + scopeLabel, + type BetaSelection, + type LibraryScript + } from '$lib/script-library.js'; + let { scripts, onuploaded }: { scripts: LibraryScript[]; onuploaded: () => Promise } = + $props(); + interface Duplicate { + scriptId: number; + versionId: number; + version: number; + } interface Upload { - key: number; file: File; target: number | 'new' | ''; name: string; - duplicates: Duplicate[]; inspecting: boolean; inspected: boolean; error: string; + key: number; + file: File; + target: number | 'new' | ''; + name: string; + duplicates: Duplicate[]; + inspecting: boolean; + inspected: boolean; + error: string; result: (BetaSelection & { version: number; reused: boolean }) | null; } let dialog: HTMLDialogElement; @@ -20,153 +37,315 @@ let scopeType = $state<'none' | 'game' | 'map'>('none'); let scopeGame = $state(''); let scopeMap = $state(''); - let scopeOptions = $state<{ games: string[]; mapsByGame: Record }>({ games: [], mapsByGame: {} }); + let scopeOptions = $state<{ games: string[]; mapsByGame: Record }>({ + games: [], + mapsByGame: {} + }); let servers = $state>([]); let scopeLoaded = $state(false); let message = $state(''); let preferredTarget = $state(null); let nextKey = 0; let activeFile = $state(''); - const checkedCount = $derived(queue.filter(q => q.inspected).length); - const pending = $derived(queue.filter(q => !q.result)); - const inspecting = $derived(queue.some(q => q.inspecting)); - const hasNew = $derived(pending.some(q => q.target === 'new')); - const scopeValue = $derived(scopeType === 'game' ? scopeGame : scopeType === 'map' ? scopeMap : ''); - const matches = $derived(servers.filter(s => scopeType === 'game' ? s.game === scopeValue : s.map === scopeValue)); - const ready = $derived(pending.length > 0 && !busy && !inspecting && pending.every(q => q.inspected && !validation(q)) - && (!hasNew || scopeType === 'none' || scopeLoaded && !!scopeValue)); - const results = $derived(queue.flatMap(q => q.result ? [q.result] : [])); - const testable = $derived(results.filter(r => { - const s = scripts.find(s => s.id === r.scriptId); - return s && s.runningBetaId === null && s.current_version_id !== r.versionId; - })); + const checkedCount = $derived(queue.filter((q) => q.inspected).length); + const pending = $derived(queue.filter((q) => !q.result)); + const inspecting = $derived(queue.some((q) => q.inspecting)); + const hasNew = $derived(pending.some((q) => q.target === 'new')); + const scopeValue = $derived( + scopeType === 'game' ? scopeGame : scopeType === 'map' ? scopeMap : '' + ); + const matches = $derived( + servers.filter((s) => (scopeType === 'game' ? s.game === scopeValue : s.map === scopeValue)) + ); + const ready = $derived( + pending.length > 0 && + !busy && + !inspecting && + pending.every((q) => q.inspected && !validation(q)) && + (!hasNew || scopeType === 'none' || (scopeLoaded && !!scopeValue)) + ); + const results = $derived(queue.flatMap((q) => (q.result ? [q.result] : []))); + const testable = $derived( + results.filter((r) => { + const s = scripts.find((s) => s.id === r.scriptId); + return s && s.runningBetaId === null && s.current_version_id !== r.versionId; + }) + ); export function open(scriptId?: number) { // Reopening preserves unfinished work; a targeted upload starts a fresh review // only when there is no pending file to lose. - if (!pending.length) { queue = []; notes = ''; message = ''; preferredTarget = scriptId ?? null; } + if (!pending.length) { + queue = []; + notes = ''; + message = ''; + preferredTarget = scriptId ?? null; + } dialog.showModal(); if (!scopeLoaded) void loadScope(); } async function loadScope() { try { - const [scopeRes, serversRes] = await Promise.all([fetch('/api/scope-options'), fetch('/api/servers')]); - if (!scopeRes.ok || !serversRes.ok) throw new Error('Scope preview could not be loaded. Reopen the drawer to retry.'); - scopeOptions = await scopeRes.json(); servers = (await serversRes.json()).servers; + const [scopeRes, serversRes] = await Promise.all([ + fetch('/api/scope-options'), + fetch('/api/servers') + ]); + if (!scopeRes.ok || !serversRes.ok) + throw new Error('Scope preview could not be loaded. Reopen the drawer to retry.'); + scopeOptions = await scopeRes.json(); + servers = (await serversRes.json()).servers; scopeLoaded = true; - } catch (e) { message = (e as Error).message; } + } catch (e) { + message = (e as Error).message; + } } function matchingScripts(file: File) { - return scripts.filter(s => s.target_rel_path.replaceAll('\\', '/').split('/').pop()?.toLowerCase() === file.name.toLowerCase()); + return scripts.filter( + (s) => + s.target_rel_path.replaceAll('\\', '/').split('/').pop()?.toLowerCase() === + file.name.toLowerCase() + ); } export async function addFiles(files: File[]) { if (busy || inspecting || !files.length) return; if (!dialog.open) open(); - const items = files.map(file => { + const items = files.map((file) => { const matches = matchingScripts(file); - const item: Upload = { key: nextKey++, file, target: preferredTarget ?? (matches.length === 1 ? matches[0].id : matches.length > 1 ? '' : 'new'), - name: file.name.replace(/\.[^.]+$/, ''), duplicates: [], inspecting: true, inspected: false, error: '', result: null }; + const item: Upload = { + key: nextKey++, + file, + target: + preferredTarget ?? + (matches.length === 1 ? matches[0].id : matches.length > 1 ? '' : 'new'), + name: file.name.replace(/\.[^.]+$/, ''), + duplicates: [], + inspecting: true, + inspected: false, + error: '', + result: null + }; return item; }); queue = [...queue, ...items]; - for (const item of items) await inspect(queue.find(q => q.key === item.key)!); + for (const item of items) await inspect(queue.find((q) => q.key === item.key)!); preferredTarget = null; } async function inspect(item: Upload) { activeFile = item.file.name; - item.inspecting = true; item.error = ''; + item.inspecting = true; + item.error = ''; try { - const form = new FormData(); form.append('content', item.file); + const form = new FormData(); + form.append('content', item.file); const res = await fetch('/api/scripts/inspect', { method: 'POST', body: form }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not inspect this file.'); - item.duplicates = body.duplicates; item.inspected = true; - } catch (e) { item.error = (e as Error).message; } - finally { item.inspecting = false; activeFile = ''; } + item.duplicates = body.duplicates; + item.inspected = true; + } catch (e) { + item.error = (e as Error).message; + } finally { + item.inspecting = false; + activeFile = ''; + } + } + function duplicate(item: Upload) { + return item.duplicates.find((d) => d.scriptId === item.target); } - function duplicate(item: Upload) { return item.duplicates.find(d => d.scriptId === item.target); } function validation(item: Upload): string { - if (item.target === '') return 'Several scripts use this filename. Choose which script to update.'; + if (item.target === '') + return 'Several scripts use this filename. Choose which script to update.'; if (item.target === 'new') { if (!item.name.trim()) return 'Give this script a name.'; - if (scripts.some(s => s.name === item.name.trim())) return 'This name already exists. Choose a new name or update that script.'; - if (queue.some(q => q.key !== item.key && !q.result && q.target === 'new' && q.name.trim() === item.name.trim())) return 'Use a distinct name for each new script.'; + if (scripts.some((s) => s.name === item.name.trim())) + return 'This name already exists. Choose a new name or update that script.'; + if ( + queue.some( + (q) => + q.key !== item.key && + !q.result && + q.target === 'new' && + q.name.trim() === item.name.trim() + ) + ) + return 'Use a distinct name for each new script.'; } else { - const script = scripts.find(s => s.id === item.target); + const script = scripts.find((s) => s.id === item.target); if (!script) return 'This script is no longer available.'; - if (script.runningBetaId !== null) return 'Finish the active beta before uploading another version.'; - if (queue.some(q => q.key !== item.key && !q.result && q.target === item.target)) return 'Choose one file per existing script in this batch.'; + if (script.runningBetaId !== null) + return 'Finish the active beta before uploading another version.'; + if (queue.some((q) => q.key !== item.key && !q.result && q.target === item.target)) + return 'Choose one file per existing script in this batch.'; } return ''; } async function upload(prepareBeta: boolean) { if (!ready) return; - busy = true; message = ''; + busy = true; + message = ''; try { - for (const item of queue.filter(q => !q.result)) { + for (const item of queue.filter((q) => !q.result)) { activeFile = item.file.name; item.error = ''; try { const form = new FormData(); - form.append('content', item.file); form.append('notes', notes.trim()); + form.append('content', item.file); + form.append('notes', notes.trim()); const creating = item.target === 'new'; if (creating) { - form.append('name', item.name.trim()); form.append('scopeType', scopeType); + form.append('name', item.name.trim()); + form.append('scopeType', scopeType); if (scopeValue) form.append('scopeValue', scopeValue); } else form.append('reuseExisting', 'true'); - const res = await fetch(creating ? '/api/scripts' : `/api/scripts/${item.target}/versions`, { method: 'POST', body: form }); + const res = await fetch( + creating ? '/api/scripts' : `/api/scripts/${item.target}/versions`, + { method: 'POST', body: form } + ); const body = await res.json(); - if (!res.ok || !body.ok) throw new Error(body.error ?? 'Upload failed. Try again.'); - item.result = { scriptId: creating ? body.script.id : item.target as number, + if (!res.ok || !body.ok) + throw new Error(body.error ?? 'Upload failed. Try again.'); + item.result = { + scriptId: creating ? body.script.id : (item.target as number), versionId: creating ? body.versionId : body.version.id, - version: creating ? 1 : body.version.version, reused: !!body.reused }; - } catch (e) { item.error = (e as Error).message; } + version: creating ? 1 : body.version.version, + reused: !!body.reused + }; + } catch (e) { + item.error = (e as Error).message; + } } await onuploaded(); - if (queue.some(q => !q.result)) message = 'Some files need attention. Successful uploads are saved; retry only the remaining files.'; + if (queue.some((q) => !q.result)) + message = + 'Some files need attention. Successful uploads are saved; retry only the remaining files.'; else { message = 'Your files are ready. Choose servers and a test plan in beta setup.'; // Read from refreshed props after Svelte applies the parent update. await tick(); - if (testable.length !== results.length) message = `${results.length - testable.length} version(s) are already promoted or unavailable for a new beta. Review the saved files before continuing.`; - if (prepareBeta && testable.length === results.length && testable.length) { dialog.close(); await goto(betaSelectionHref(testable)); } + if (testable.length !== results.length) + message = `${results.length - testable.length} version(s) are already promoted or unavailable for a new beta. Review the saved files before continuing.`; + if (prepareBeta && testable.length === results.length && testable.length) { + dialog.close(); + await goto(betaSelectionHref(testable)); + } } - } catch (e) { message = (e as Error).message; } - finally { busy = false; activeFile = ''; } + } catch (e) { + message = (e as Error).message; + } finally { + busy = false; + activeFile = ''; + } + } + function close(event?: Event) { + if (busy || inspecting) { + event?.preventDefault(); + return; + } + dialog.close(); } - function close(event?: Event) { if (busy || inspecting) { event?.preventDefault(); return; } dialog.close(); }
-
Prepare your next release

Upload scripts

-

Drop files anywhere or choose them below. Review each destination before uploading.

+
+
+ Prepare your next release +

Upload scripts

+
+ +
+

+ Drop files anywhere or choose them below. Review each destination before uploading. +

- {preferredTarget ? `New version for ${scripts.find(s => s.id === preferredTarget)?.name ?? 'script'}` : 'One file or a whole batch'} + {preferredTarget + ? `New version for ${scripts.find((s) => s.id === preferredTarget)?.name ?? 'script'}` + : 'One file or a whole batch'} Existing filenames are recognized automatically. - - { void addFiles([...event.currentTarget.files ?? []]); event.currentTarget.value = ''; }} /> + + { + void addFiles([...(event.currentTarget.files ?? [])]); + event.currentTarget.value = ''; + }} + />
{#each queue as item (item.key)}
-
{item.file.name}{(item.file.size / 1024).toFixed(1)} KB - +
+ {item.file.name}{(item.file.size / 1024).toFixed(1)} KB +
{#if item.result} -

{item.result.reused ? 'Already uploaded' : 'Uploaded'} as v{item.result.version} · View script →

+

+ {item.result.reused ? 'Already uploaded' : 'Uploaded'} as v{item.result + .version} · + View script → +

{:else} - - {#if item.target === 'new'} - {:else if item.target !== ''}{@const s = scripts.find(s => s.id === item.target)}

Keeps scope: {s ? scopeLabel(s) : '—'}. Target: {s?.target_rel_path}

{/if} + + {#if item.target === 'new'} + {:else if item.target !== ''}{@const s = scripts.find( + (s) => s.id === item.target + )} +

+ Keeps scope: {s ? scopeLabel(s) : '—'}. Target: {s?.target_rel_path} +

{/if} {#if item.inspecting}

Checking file contents…

- {:else if duplicate(item)}

Already uploaded as v{duplicate(item)!.version}. We’ll reuse that version.

- {:else if item.target !== 'new' && item.target !== ''}

New version for {scripts.find(s => s.id === item.target)?.name}

{/if} + {:else if duplicate(item)}

+ Already uploaded as v{duplicate(item)!.version}. We’ll reuse that + version. +

+ {:else if item.target !== 'new' && item.target !== ''}

+ New version for {scripts.find((s) => s.id === item.target)?.name} +

{/if} {#if validation(item)}

{validation(item)}

{/if} - {#if item.error}{#if !item.inspected}{/if}{/if} + {#if item.error} + {#if !item.inspected}{/if}{/if} {/if}
{/each} @@ -174,46 +353,277 @@ {#if pending.length}

Shared upload details

- + {#if hasNew} - + {#if scopeType !== 'none'} - - {#if scopeType === 'map'}{/if} - {#if scopeValue}

{matches.length} matching servers: {matches.map(s => s.name).join(', ') || 'None'}

{/if} - {#if scopeType === 'map' && scopeGame === 't6'}

Map scope includes every server using this engine map, including its Survival variants.

{/if} - {:else}

Unscoped scripts deploy only to explicitly selected beta targets.

{/if} + + {#if scopeType === 'map'}{/if} + {#if scopeValue}

+ {matches.length} matching servers: {matches + .map((s) => s.name) + .join(', ') || 'None'} +

{/if} + {#if scopeType === 'map' && scopeGame === 't6'}

+ Map scope includes every server using this engine map, including its + Survival variants. +

{/if} + {:else}

+ Unscoped scripts deploy only to explicitly selected beta targets. +

{/if}

Existing scripts keep their current scope.

{/if}
{/if} {#if message}

{message}

{/if}
- {#if inspecting || busy}

{inspecting ? `Checked ${checkedCount} of ${queue.length} ${plural(queue.length, 'file')}` : `Uploaded ${results.length} of ${queue.length} ${plural(queue.length, 'file')}`}{activeFile ? ` · ${activeFile}` : ''}

{:else if queue.length}{@const failed = queue.filter(q => q.error).length}

{results.length} saved · {pending.length} remaining{failed ? ` · ${failed} ${plural(failed, 'needs', 'need')} attention` : ''}

{/if} - {#if pending.length} - {:else if testable.length}{/if} + {#if inspecting || busy}

+ {inspecting + ? `Checked ${checkedCount} of ${queue.length} ${plural(queue.length, 'file')}` + : `Uploaded ${results.length} of ${queue.length} ${plural(queue.length, 'file')}`}{activeFile + ? ` · ${activeFile}` + : ''} +

{:else if queue.length}{@const failed = queue.filter((q) => q.error).length} +

+ {results.length} saved · {pending.length} remaining{failed + ? ` · ${failed} ${plural(failed, 'needs', 'need')} attention` + : ''} +

{/if} + {#if pending.length} + {:else if testable.length}{/if}
diff --git a/src/lib/ui/components/ServerCard.svelte b/src/lib/ui/components/ServerCard.svelte index 637a626..3aad004 100644 --- a/src/lib/ui/components/ServerCard.svelte +++ b/src/lib/ui/components/ServerCard.svelte @@ -11,7 +11,10 @@ import Badge from './primitives/Badge.svelte'; import StatusBadge from './primitives/StatusBadge.svelte'; - interface Held { target: string; betaId: number; } + interface Held { + target: string; + betaId: number; + } interface Props { telemetry?: import('$lib/telemetry.js').GameTelemetry; name: string; @@ -38,137 +41,441 @@ longestMatch?: boolean; } let { - telemetry, name, players = null, map = null, roster = null, health = 'offline', - lastSeenAt = null, lastError = null, held = [], restarting = false, - inactiveSince = null, gameStartedAt = null, gameEndedAt = null, compactEmpty = true, - mapLabelOverride, round, capacity = null, pinned = false, onPin, longestMatch = false + telemetry, + name, + players = null, + map = null, + roster = null, + health = 'offline', + lastSeenAt = null, + lastError = null, + held = [], + restarting = false, + inactiveSince = null, + gameStartedAt = null, + gameEndedAt = null, + compactEmpty = true, + mapLabelOverride, + round, + capacity = null, + pinned = false, + onPin, + longestMatch = false }: Props = $props(); const id = $props.id(); const number = new Intl.NumberFormat('en-US'); const identity = $derived(serverIdentity(name)); const title = $derived(mapLabelOverride ?? identity.map); - const game = $derived(identity.game === 'Black Ops I' ? 'BO1' : identity.game === 'Black Ops II' ? 'BO2' : identity.game); + const game = $derived( + identity.game === 'Black Ops I' + ? 'BO1' + : identity.game === 'Black Ops II' + ? 'BO2' + : identity.game + ); const compact = $derived(compactEmpty && health === 'online' && players === 0 && !restarting); const state = $derived(restarting ? 'restarting' : compact ? 'ready' : health); const error = $derived(decodeRconError(lastError)); const mapLabel = $derived(mapTitle(map)); const showMap = $derived(mapLabel !== '—' && mapLabel !== title); - const topRoster = $derived((roster ?? []).slice().sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, 4)); + const topRoster = $derived( + (roster ?? []) + .slice() + .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + .slice(0, 4) + ); const report = $derived.by(() => { const streamed = $telemetryByServer.get(name); - return streamed && (!telemetry || Date.parse(streamed.receivedAt) > Date.parse(telemetry.receivedAt)) ? streamed : telemetry; + return streamed && + (!telemetry || Date.parse(streamed.receivedAt) > Date.parse(telemetry.receivedAt)) + ? streamed + : telemetry; }); - const reportedRound = $derived(report && (report.event === 'heartbeat' || report.event === 'round_change') - && freshStatusTimestamp(report.receivedAt, $uiNow) - && (!gameStartedAt || Date.parse(report.receivedAt) >= Date.parse(gameStartedAt)) - ? report.round : null); - const currentRound = $derived(!gameEndedAt ? (round !== undefined ? round : reportedRound) : null); - const gameElapsed = $derived(gameStartedAt && !gameEndedAt ? elapsedSince(gameStartedAt, $uiNow) : null); + const reportedRound = $derived( + report && + (report.event === 'heartbeat' || report.event === 'round_change') && + freshStatusTimestamp(report.receivedAt, $uiNow) && + (!gameStartedAt || Date.parse(report.receivedAt) >= Date.parse(gameStartedAt)) + ? report.round + : null + ); + const currentRound = $derived( + !gameEndedAt ? (round !== undefined ? round : reportedRound) : null + ); + const gameElapsed = $derived( + gameStartedAt && !gameEndedAt ? elapsedSince(gameStartedAt, $uiNow) : null + ); const emptyElapsed = $derived(elapsedSince(inactiveSince, $uiNow)); // These lobbies have four slots when the live status omits the limit. - const displayCapacity = $derived(capacity !== null && Number.isSafeInteger(capacity) && capacity > 0 ? capacity : 4); - const cardStyle = $derived(`view-transition-name: server-${name.replace(/[^a-z0-9_-]/gi, '-')};`); + const displayCapacity = $derived( + capacity !== null && Number.isSafeInteger(capacity) && capacity > 0 ? capacity : 4 + ); + const cardStyle = $derived( + `view-transition-name: server-${name.replace(/[^a-z0-9_-]/gi, '-')};` + ); - +
- +

{title}

- {#if identity.lobby !== null}Lobby {identity.lobby}{/if} + {#if identity.lobby !== null}Lobby {identity.lobby}{/if} {#if game}{game}{/if} - {#if compact}{emptyElapsed ? `Empty for ${emptyElapsed}` : 'Empty lobby'}{/if} + {#if compact}{emptyElapsed ? `Empty for ${emptyElapsed}` : 'Empty lobby'}{/if}
- {#if onPin}{/if} + {#if onPin}{/if}
{#if !compact} - {#if longestMatch && health === 'online'}
Longest match
{/if} - {#if showMap}

{health === 'online' ? 'Current map' : 'Last map'}: {mapLabel}

{/if} + {#if longestMatch && health === 'online'}
+ Longest match +
{/if} + {#if showMap}

+ {health === 'online' ? 'Current map' : 'Last map'}: {mapLabel} +

{/if} {#if health === 'online'}
-
Players
{players ?? '—'}/{displayCapacity}
-
Round
{currentRound ?? '—'}
-
Match time
{gameElapsed ?? '—'}
+
+
Players
+
+ {players ?? '—'}/{displayCapacity} +
+
+
+
Round
+
{currentRound ?? '—'}
+
+
+
Match time
+
{gameElapsed ?? '—'}
+
- {#if gameEndedAt && players !== 0}

Match ended · waiting for the next match

{/if} + {#if gameEndedAt && players !== 0}

+ Match ended · waiting for the next match +

{/if} {:else if health === 'stale'} -

Current player count unknown. Last reported: {players ?? 'unknown'}{players !== null ? ` ${players === 1 ? 'player' : 'players'}` : ''}{lastSeenAt ? ` · ${elapsedSince(lastSeenAt, $uiNow)} ago` : ''}.

+

+ Current player count unknown. Last reported: {players ?? + 'unknown'}{players !== null + ? ` ${players === 1 ? 'player' : 'players'}` + : ''}{lastSeenAt ? ` · ${elapsedSince(lastSeenAt, $uiNow)} ago` : ''}. +

{/if} {#if health === 'online' || health === 'stale'} {#if topRoster.length}
- - - {#each topRoster as player (player.guid ?? player.num)} - - - - {/each} +
PlayerScorePing (ms)
{player.name || '—'}{player.score === null ? '—' : number.format(player.score)}{player.ping === 999 ? 'Connecting' : player.ping === null ? '—' : number.format(player.ping)}
+ + {#each topRoster as player (player.guid ?? player.num)} + + + + {/each}
PlayerScorePing (ms)
{player.name || '—'}{player.score === null + ? '—' + : number.format(player.score)}{player.ping === 999 + ? 'Connecting' + : player.ping === null + ? '—' + : number.format(player.ping)}
- {:else if players !== 0}

{health === 'stale' ? 'Waiting for a fresh report' : 'Player roster unavailable'}

{/if} + {:else if players !== 0}

+ {health === 'stale' + ? 'Waiting for a fresh report' + : 'Player roster unavailable'} +

{/if} {/if} {#if health === 'offline'} -

{error?.detail || 'Server status is unavailable. Open diagnostics for details.'}

- {#if lastSeenAt}

Last report {elapsedSince(lastSeenAt, $uiNow)} ago

{/if} +

+ {error?.detail || 'Server status is unavailable. Open diagnostics for details.'} +

+ {#if lastSeenAt}

+ Last report {elapsedSince(lastSeenAt, $uiNow)} ago +

{/if} {/if} {/if} - {#if held.length > 0}
{#each held as h (h.target)}β {h.target}{/each}
{/if} + {#if held.length > 0}
+ {#each held as h (h.target)}β {h.target}{/each} +
{/if} diff --git a/src/lib/ui/components/ServerIdentity.svelte b/src/lib/ui/components/ServerIdentity.svelte index 5764c83..4d834a7 100644 --- a/src/lib/ui/components/ServerIdentity.svelte +++ b/src/lib/ui/components/ServerIdentity.svelte @@ -1,12 +1,40 @@ -{mapLabel ?? identity.map}{#if identity.lobby !== null}Lobby {identity.lobby}{/if}{#if showGame && identity.game}{identity.game}{/if} + +{mapLabel ?? identity.map}{#if identity.lobby !== null}Lobby {identity.lobby}{/if}{#if showGame && identity.game}{identity.game}{/if} + diff --git a/src/lib/ui/components/SettingsEditor.svelte b/src/lib/ui/components/SettingsEditor.svelte index eba3387..9571ca8 100644 --- a/src/lib/ui/components/SettingsEditor.svelte +++ b/src/lib/ui/components/SettingsEditor.svelte @@ -5,8 +5,16 @@ import Button from './primitives/Button.svelte'; import RetentionTools from './RetentionTools.svelte'; import { - defaultFleetSettings, fleetSettingLabels, settingsGroups, settingHelp, settingMinimum, - settingZeroMode, settingUnit, settingDisplayValue, validateFleetSettings, type FleetSettings + defaultFleetSettings, + fleetSettingLabels, + settingsGroups, + settingHelp, + settingMinimum, + settingZeroMode, + settingUnit, + settingDisplayValue, + validateFleetSettings, + type FleetSettings } from '$lib/fleet-settings.js'; type Key = keyof FleetSettings; @@ -16,7 +24,9 @@ const keys = Object.keys(defaultFleetSettings) as Key[]; let baseline = $state({ ...untrack(() => initial) }); let draft = $state({ ...untrack(() => initial) }); - let remembered = Object.fromEntries(keys.filter(key => baseline[key] > 0).map(key => [key, baseline[key]])) as Partial; + let remembered = Object.fromEntries( + keys.filter((key) => baseline[key] > 0).map((key) => [key, baseline[key]]) + ) as Partial; let query = $state(''); let selectedGroup = $state(settingsGroups[0].id); let reviewInvalid = $state(false); @@ -24,21 +34,34 @@ let error = $state(''); let notice = $state(''); let allowNavigation = false; - const changed = $derived(keys.filter(key => draft[key] !== baseline[key])); - const invalid = $derived(keys.filter(key => !!validationMessage(key, draft[key]))); + const changed = $derived(keys.filter((key) => draft[key] !== baseline[key])); + const invalid = $derived(keys.filter((key) => !!validationMessage(key, draft[key]))); const visibleGroups = $derived.by(() => { const terms = query.trim().toLocaleLowerCase().split(/\s+/).filter(Boolean); - return settingsGroups.map(group => ({ ...group, keys: group.keys.filter(key => { - if (reviewInvalid && invalid.length) return invalid.includes(key); - if (!terms.length) return group.id === selectedGroup; - const text = `${group.title} ${group.description} ${fleetSettingLabels[key]} ${settingHelp[key] ?? ''} ${settingZeroMode(key) ?? ''} ${key}`.toLocaleLowerCase(); - return terms.every(term => text.includes(term)); - }) })).filter(group => group.keys.length); + return settingsGroups + .map((group) => ({ + ...group, + keys: group.keys.filter((key) => { + if (reviewInvalid && invalid.length) return invalid.includes(key); + if (!terms.length) return group.id === selectedGroup; + const text = + `${group.title} ${group.description} ${fleetSettingLabels[key]} ${settingHelp[key] ?? ''} ${settingZeroMode(key) ?? ''} ${key}`.toLocaleLowerCase(); + return terms.every((term) => text.includes(term)); + }) + })) + .filter((group) => group.keys.length); }); - const shownCount = $derived(visibleGroups.reduce((count, group) => count + group.keys.length, 0)); + const shownCount = $derived( + visibleGroups.reduce((count, group) => count + group.keys.length, 0) + ); function validationMessage(key: Key, value: number | undefined): string { - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < settingMinimum(key)) return 'Enter a valid whole number.'; + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < settingMinimum(key) + ) + return 'Enter a valid whole number.'; if (key.endsWith('Days') && value > 36500) return 'Enter 36,500 days or fewer.'; return ''; } @@ -46,63 +69,132 @@ if (saving) return; const current = draft[key]; if (enabled) { - if (current !== undefined && Number.isSafeInteger(current) && current > 0) remembered[key] = current; + if (current !== undefined && Number.isSafeInteger(current) && current > 0) + remembered[key] = current; draft[key] = 0; } else { - draft[key] = remembered[key] ?? (defaultFleetSettings[key] > 0 ? defaultFleetSettings[key] : 1); + draft[key] = + remembered[key] ?? (defaultFleetSettings[key] > 0 ? defaultFleetSettings[key] : 1); } - error = ''; notice = ''; + error = ''; + notice = ''; + } + function selectGroup(groupId: string) { + selectedGroup = groupId; + query = ''; + reviewInvalid = false; } - function selectGroup(groupId: string) { selectedGroup = groupId; query = ''; reviewInvalid = false; } function discard() { if (saving) return; draft = { ...baseline }; - remembered = Object.fromEntries(keys.filter(key => baseline[key] > 0).map(key => [key, baseline[key]])) as Partial; - error = ''; notice = ''; reviewInvalid = false; + remembered = Object.fromEntries( + keys.filter((key) => baseline[key] > 0).map((key) => [key, baseline[key]]) + ) as Partial; + error = ''; + notice = ''; + reviewInvalid = false; } async function save() { if (saving || !changed.length || invalid.length) return; - saving = true; error = ''; notice = ''; + saving = true; + error = ''; + notice = ''; try { const values = validateFleetSettings(draft); - const response = await fetch('/api/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(values) }); + const response = await fetch('/api/settings', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(values) + }); const result = await response.json(); if (!response.ok) throw new Error(result.error ?? 'Could not save fleet settings.'); baseline = validateFleetSettings(result); draft = { ...baseline }; for (const key of keys) if (baseline[key] > 0) remembered[key] = baseline[key]; notice = 'Settings saved. Changes apply to the next queue or cleanup check.'; - } catch (cause) { error = cause instanceof Error ? cause.message : 'Could not save fleet settings.'; } - finally { saving = false; } + } catch (cause) { + error = cause instanceof Error ? cause.message : 'Could not save fleet settings.'; + } finally { + saving = false; + } } beforeNavigate(({ cancel, to, from, type }) => { - if (!changed.length || allowNavigation || type === 'leave' || !to || to.url.pathname === from?.url.pathname) return; + if ( + !changed.length || + allowNavigation || + type === 'leave' || + !to || + to.url.pathname === from?.url.pathname + ) + return; cancel(); - void confirmAction({ title: 'Leave Settings?', message: 'Your unsaved settings will be discarded.', confirmLabel: 'Leave settings', danger: true }).then(async accepted => { + void confirmAction({ + title: 'Leave Settings?', + message: 'Your unsaved settings will be discarded.', + confirmLabel: 'Leave settings', + danger: true + }).then(async (accepted) => { if (!accepted) return; allowNavigation = true; - try { await goto(to.url); } finally { allowNavigation = false; } + try { + await goto(to.url); + } finally { + allowNavigation = false; + } }); }); - function beforeUnload(event: BeforeUnloadEvent) { if (changed.length) { event.preventDefault(); event.returnValue = ''; } } + function beforeUnload(event: BeforeUnloadEvent) { + if (changed.length) { + event.preventDefault(); + event.returnValue = ''; + } + }
@@ -119,19 +184,56 @@ {#snippet campaignTable(rows: Deployment[], showProgress = false)}
- - - + + + {#each rows as campaign (campaign.id)} {@const outcome = campaignOutcome(campaign)} - + - - + + {/each} @@ -140,41 +242,207 @@ {/snippet} diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 49e3d73..b84fbaf 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -21,23 +21,39 @@ import { serverIdentity } from '$ui/server-labels.js'; interface FileRow { - id: number; server: string; verb: string; rel_path: string; - sha256_before: string | null; sha256_after: string | null; - size_before: number | null; size_after: number | null; + id: number; + server: string; + verb: string; + rel_path: string; + sha256_before: string | null; + sha256_after: string | null; + size_before: number | null; + size_after: number | null; apply_state: 'pending' | 'applying' | 'applied' | 'cancelled'; snapshot_pending: number; } interface ServerRow { - deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number; - restart_state: string; players: number | null; - checked_at: string | null; restarted_at: string | null; provisioning_id: string | null; - attempts: number; error: string | null; + deployment_id: string; + server: string; + server_deleted_at?: string | null; + files_changed: number; + restart_state: string; + players: number | null; + checked_at: string | null; + restarted_at: string | null; + provisioning_id: string | null; + attempts: number; + error: string | null; } interface Detail { fileContentsExpired?: boolean; deployment: { - id: string; action: string; started_at: string; - finished_at: string | null; triggered_by: string | null; beta_id?: number | null; + id: string; + action: string; + started_at: string; + finished_at: string | null; + triggered_by: string | null; + beta_id?: number | null; rolled_back_by: string | null; }; servers: ServerRow[]; @@ -57,35 +73,67 @@ let diffError = $state(''); let diffLoading = $state(false); let diffRequest = 0; - let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null); + let openFile = $state<{ + id: number; + before: string | null; + after: string | null; + verb: string; + relPath: string; + } | null>(null); let retryFile = $state(null); let filter = $state<'all' | CampaignServerGroup>('all'); let expandedServers = $state([]); let copyStatus = $state(''); - const serverRows = $derived((detail?.servers ?? []).map(server => ({ - server, identity: serverIdentity(server.server), - status: campaignServerStatus(server, byServer.get(server.server)) - }))); + const serverRows = $derived( + (detail?.servers ?? []).map((server) => ({ + server, + identity: serverIdentity(server.server), + status: campaignServerStatus(server, byServer.get(server.server)) + })) + ); const counts = $derived.by(() => { - const result: Record = { attention: 0, processing: 0, queued: 0, complete: 0, inactive: 0 }; + const result: Record = { + attention: 0, + processing: 0, + queued: 0, + complete: 0, + inactive: 0 + }; for (const row of serverRows) result[row.status.group]++; return result; }); const filters: { value: 'all' | CampaignServerGroup; label: string }[] = [ - { value: 'all', label: 'All' }, { value: 'attention', label: 'Needs attention' }, - { value: 'processing', label: 'Processing' }, { value: 'queued', label: 'Queued' }, - { value: 'complete', label: 'Complete' }, { value: 'inactive', label: 'Inactive' } + { value: 'all', label: 'All' }, + { value: 'attention', label: 'Needs attention' }, + { value: 'processing', label: 'Processing' }, + { value: 'queued', label: 'Queued' }, + { value: 'complete', label: 'Complete' }, + { value: 'inactive', label: 'Inactive' } ]; - const visibleServers = $derived(serverRows.filter(row => filter === 'all' || row.status.group === filter).sort((a, b) => { - const rank: Record = { attention: 0, processing: 1, queued: 2, complete: 3, inactive: 4 }; - return rank[a.status.group] - rank[b.status.group]; - })); - const uniqueFileCount = $derived(new Set(detail?.files.map(file => file.rel_path)).size); + const visibleServers = $derived( + serverRows + .filter((row) => filter === 'all' || row.status.group === filter) + .sort((a, b) => { + const rank: Record = { + attention: 0, + processing: 1, + queued: 2, + complete: 3, + inactive: 4 + }; + return rank[a.status.group] - rank[b.status.group]; + }) + ); + const uniqueFileCount = $derived(new Set(detail?.files.map((file) => file.rel_path)).size); async function copyId() { if (!detail) return; - try { await navigator.clipboard.writeText(detail.deployment.id); copyStatus = 'Copied'; } - catch { copyStatus = 'Could not copy. Select the ID to copy it manually.'; } + try { + await navigator.clipboard.writeText(detail.deployment.id); + copyStatus = 'Copied'; + } catch { + copyStatus = 'Could not copy. Select the ID to copy it manually.'; + } } async function loadDetail(): Promise { @@ -93,59 +141,112 @@ const current = ++request; try { const res = await fetch(`/api/campaigns/${encodeURIComponent(id ?? '')}`); - if (!res.ok) throw new Error(res.status === 404 ? 'Campaign not found.' : 'Could not refresh campaign.'); - const body = await res.json() as Detail; + if (!res.ok) + throw new Error( + res.status === 404 ? 'Campaign not found.' : 'Could not refresh campaign.' + ); + const body = (await res.json()) as Detail; if (current !== request || id !== $page.params.id) return; - detail = body; loadError = ''; + detail = body; + loadError = ''; } catch (err) { if (current === request && id === $page.params.id) loadError = (err as Error).message; } } - $effect(() => { $page.params.id; detail = null; openFile = null; diffOpen = false; loadError = ''; rollbackError = ''; filter = 'all'; expandedServers = []; copyStatus = ''; void loadDetail(); }); + $effect(() => { + void $page.params.id; + detail = null; + openFile = null; + diffOpen = false; + loadError = ''; + rollbackError = ''; + filter = 'all'; + expandedServers = []; + copyStatus = ''; + void loadDetail(); + }); onMount(() => pollVisible(loadDetail, 5000, false)); async function resume() { - resuming = true; resumeError = ''; + resuming = true; + resumeError = ''; try { - const response = await fetch(`/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, { method: 'POST' }); - if (!response.ok) throw new Error((await response.json()).error ?? 'Could not resume campaign.'); + const response = await fetch( + `/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, + { method: 'POST' } + ); + if (!response.ok) + throw new Error((await response.json()).error ?? 'Could not resume campaign.'); await loadDetail(); - } catch (error) { resumeError = (error as Error).message; } - finally { resuming = false; } + } catch (error) { + resumeError = (error as Error).message; + } finally { + resuming = false; + } } async function rollback() { if (!detail || rollingBack) return; const id = detail.deployment.id; - const servers = detail.servers.filter(server => !server.server_deleted_at).length; - if (!await confirmAction({ title: 'Roll back promotion?', message: `Restore the previous files on ${servers} server${servers === 1 ? '' : 's'}. Changes will wait for safe activation.`, confirmLabel: 'Roll back', danger: true })) return; + const servers = detail.servers.filter((server) => !server.server_deleted_at).length; + if ( + !(await confirmAction({ + title: 'Roll back promotion?', + message: `Restore the previous files on ${servers} server${servers === 1 ? '' : 's'}. Changes will wait for safe activation.`, + confirmLabel: 'Roll back', + danger: true + })) + ) + return; if ($page.params.id !== id) return; - rollingBack = true; rollbackError = ''; + rollingBack = true; + rollbackError = ''; try { - const response = await fetch(`/api/campaigns/${encodeURIComponent(id)}/rollback`, { method: 'POST' }); + const response = await fetch(`/api/campaigns/${encodeURIComponent(id)}/rollback`, { + method: 'POST' + }); const body = await response.json(); - if (!response.ok || !body.ok) throw new Error(body.error ?? 'Could not roll back promotion.'); + if (!response.ok || !body.ok) + throw new Error(body.error ?? 'Could not roll back promotion.'); if ($page.params.id !== id) return; - await goto(contextHref(`/campaigns/${encodeURIComponent(body.result.deploymentId)}`, $page.url)); + await goto( + contextHref(`/campaigns/${encodeURIComponent(body.result.deploymentId)}`, $page.url) + ); } catch (error) { if ($page.params.id === id) rollbackError = (error as Error).message; - } finally { rollingBack = false; } + } finally { + rollingBack = false; + } } async function loadDiff(f: FileRow) { retryFile = f; const id = $page.params.id; const current = ++diffRequest; - diffOpen = true; diffLoading = true; diffError = ''; openFile = null; + diffOpen = true; + diffLoading = true; + diffError = ''; + openFile = null; try { const response = await fetch('/api/campaigns/' + id + '/file/' + f.id); - if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : response.status === 409 ? 'The rollback snapshot is not available yet. Refresh after activation to compare files.' : 'Could not load this file comparison. Close and select it again to retry.'); + if (!response.ok) + throw new Error( + response.status === 410 + ? 'File contents expired under the retention policy.' + : response.status === 409 + ? 'The rollback snapshot is not available yet. Refresh after activation to compare files.' + : 'Could not load this file comparison. Close and select it again to retry.' + ); const body = await response.json(); - if (id === $page.params.id && current === diffRequest) openFile = {id:f.id, ...body}; - } catch (error) { if (id === $page.params.id && current === diffRequest) diffError = (error as Error).message; } - finally { if (current === diffRequest) diffLoading = false; } + if (id === $page.params.id && current === diffRequest) openFile = { id: f.id, ...body }; + } catch (error) { + if (id === $page.params.id && current === diffRequest) + diffError = (error as Error).message; + } finally { + if (current === diffRequest) diffLoading = false; + } } let byServer = $derived.by(() => { @@ -158,69 +259,131 @@ return m; }); function verbVariant(v: string): 'online' | 'info' | 'danger' | 'warning' | 'neutral' { - return v === 'added' ? 'online' : - v === 'updated' ? 'info' : - v === 'removed' ? 'danger' : - v === 'held' ? 'warning' : - 'neutral'; + return v === 'added' + ? 'online' + : v === 'updated' + ? 'info' + : v === 'removed' + ? 'danger' + : v === 'held' + ? 'warning' + : 'neutral'; } -{detail ? `${detail.deployment.action} campaign · ${detail.deployment.id}` : 'Campaign'} | Electric Cherry -{#if loadError}{/if} +{detail ? `${detail.deployment.action} campaign · ${detail.deployment.id}` : 'Campaign'} | Electric + Cherry +{#if loadError}{/if} {#if detail} {@const action = detail.deployment.action}
{#snippet crumbs()} - + {/snippet} {#snippet actions()} {#if detail?.deployment.rolled_back_by} - - {:else if detail && action === 'promote' && !detail.fileContentsExpired && detail.servers.some(server => !server.server_deleted_at)} - + + {:else if detail && action === 'promote' && !detail.fileContentsExpired && detail.servers.some((server) => !server.server_deleted_at)} + {/if} {/snippet} {#if rollbackError}{/if}
Started - {#if detail.deployment.beta_id}Source: Beta #{detail.deployment.beta_id}{/if} - {#if detail.watchdogIncidentId}Source: Maintenance incident{/if} + {#if detail.deployment.beta_id}Source: Beta #{detail.deployment.beta_id}{/if} + {#if detail.watchdogIncidentId}Source: Maintenance incident{/if}
Campaign details
-
{detail.deployment.id}{copyStatus}
- {#if detail.deployment.finished_at}

Finished

{/if} - {#if detail.deployment.triggered_by && detail.deployment.triggered_by !== 'ui'}

Triggered by {detail.deployment.triggered_by}

{/if} -

{detail.watchdogIncidentId ? 'Maintenance rebuilds instance files with the latest promoted setup after checking that the server is safe to update.' : 'Changes wait for an empty server and restart capacity. Rollback snapshots are captured before applying files, then servers restart and verify health. Tested servers retain their pre-beta snapshots.'}

+
+ {detail.deployment.id}{copyStatus} +
+ {#if detail.deployment.finished_at}

+ Finished +

{/if} + {#if detail.deployment.triggered_by && detail.deployment.triggered_by !== 'ui'}

+ Triggered by {detail.deployment.triggered_by} +

{/if} +

+ {detail.watchdogIncidentId + ? 'Maintenance rebuilds instance files with the latest promoted setup after checking that the server is safe to update.' + : 'Changes wait for an empty server and restart capacity. Rollback snapshots are captured before applying files, then servers restart and verify health. Tested servers retain their pre-beta snapshots.'} +

- {#if detail.fileContentsExpired}

File contents have expired. File comparisons and rollback are unavailable.

{/if} + {#if detail.fileContentsExpired}

+ File contents have expired. File comparisons and rollback are unavailable. +

{/if}

{counts.complete} of {detail.servers.length} servers complete

- {loadError ? 'Updates interrupted' : 'Updates every 5s'} + {loadError ? 'Updates interrupted' : 'Updates every 5s'}
- +
{#if counts.processing}{counts.processing} processing{/if} {#if counts.queued}{counts.queued} queued{/if} - {#if counts.attention}{counts.attention} {plural(counts.attention, 'needs', 'need')} attention{/if} + {#if counts.attention}{counts.attention} + {plural(counts.attention, 'needs', 'need')} attention{/if} {#if counts.inactive}{counts.inactive} inactive{/if} {#if detail.servers.length === 0}No servers recorded - {:else if counts.complete === detail.servers.length}All servers activated{/if} + {:else if counts.complete === detail.servers.length}All servers activated{/if}
- {#if detail.servers.some(server => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))} - + {#if detail.servers.some((server) => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))} + {/if}
@@ -230,53 +393,144 @@

Servers

{#each filters as option} - {@const count = option.value === 'all' ? detail.servers.length : counts[option.value]} + {@const count = + option.value === 'all' ? detail.servers.length : counts[option.value]} {#if option.value !== 'inactive' || count || filter === option.value} - + {/if} {/each}
{showProgress ? 'Active campaigns' : 'Campaign history'}
CampaignServersStatusStarted
{showProgress ? 'Active campaigns' : 'Campaign history'}
CampaignServersStatusStarted
{campaignTitle(campaign)}{campaignTitle(campaign)} {campaign.servers.length}
{outcome.label} - {#if showProgress}{/if} -
+ {outcome.label} + {#if showProgress}{/if} +
- + {#each visibleServers as { server: s, identity, status } (s.server)} {@const expanded = expandedServers.includes(s.server)} - - + + - + {:else} - + {/each}
ServerLobbyStatusActions
ServerLobbyStatusActions
- {#if s.server_deleted_at}{identity.map} - {:else}{identity.map}{/if} + {#if s.server_deleted_at}{identity.map} + {:else}{identity.map}{/if} {#if identity.game}{identity.game}{/if} {#if s.server_deleted_at}Deleted{/if} {identity.lobby ?? '—'}{status.label}
- {#if s.restart_state === 'failed' && !s.server_deleted_at}Diagnose{/if} - -
{status.label}
+ {#if s.restart_state === 'failed' && !s.server_deleted_at}Diagnose{/if} + +
{filter === 'all' ? 'No servers recorded.' : 'No servers in this status.'}
{filter === 'all' + ? 'No servers recorded.' + : 'No servers in this status.'}
@@ -284,23 +538,59 @@
- Files {uniqueFileCount} file{uniqueFileCount === 1 ? '' : 's'} across {byServer.size} server{byServer.size === 1 ? '' : 's'} + Files {uniqueFileCount} file{uniqueFileCount === 1 ? '' : 's'} across {byServer.size} server{byServer.size === + 1 + ? '' + : 's'} {#if detail.files.length === 0} - + {:else} {#each [...byServer.entries()] as [srv, files] (srv)}
-

server.server === srv)?.server_deleted_at} />

+

+ server.server === srv) + ?.server_deleted_at} + /> +

{#each files as f (f.id)} - @@ -313,9 +603,19 @@ - {#if diffError} + {#if diffError} + {:else if diffLoading} - {:else if openFile}{/if} + {:else if openFile}{/if}
{:else if !loadError} @@ -323,71 +623,332 @@ {/if} diff --git a/src/routes/capacity/+page.svelte b/src/routes/capacity/+page.svelte index 6850121..9db849f 100644 --- a/src/routes/capacity/+page.svelte +++ b/src/routes/capacity/+page.svelte @@ -3,7 +3,12 @@ import { onMount, untrack } from 'svelte'; import { beforeNavigate, afterNavigate, goto } from '$app/navigation'; import { page } from '$app/state'; - import type { CapacityData, CapacityGame, CapacityQuery, CapacityRange } from '$lib/capacity.js'; + import type { + CapacityData, + CapacityGame, + CapacityQuery, + CapacityRange + } from '$lib/capacity.js'; import { pollVisible } from '$ui/poll.js'; import { timeFormat } from '$ui/stores/time-zone.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; @@ -29,14 +34,37 @@ const report = $derived(snapshot ?? data); const summary = $derived(report.current.summary); const canRebalance = $derived(['admin', 'operator'].includes(page.data?.user?.role ?? '')); - const scopeName = $derived(report.query.hostId ? report.current.hostOptions.find(host => host.id === report.query.hostId)?.name ?? 'Selected host' : 'All hosts'); - const hostOptions = $derived([{ value: '', label: 'All hosts' }, ...report.current.hostOptions.map(host => ({ value: host.id, label: host.name })), - ...(report.query.hostId && !report.current.hostOptions.some(host => host.id === report.query.hostId) ? [{ value: report.query.hostId, label: 'Unavailable host' }] : [])]); - const gameOptions = [{ value: 'all', label: 'All games' }, { value: 't5', label: 'Black Ops I' }, { value: 't6', label: 'Black Ops II' }]; - const ranges: { value: CapacityRange; label: string }[] = [{ value: '1h', label: 'Last hour' }, { value: '24h', label: '24 hours' }, { value: '7d', label: '7 days' }]; + const scopeName = $derived( + report.query.hostId + ? (report.current.hostOptions.find((host) => host.id === report.query.hostId)?.name ?? + 'Selected host') + : 'All hosts' + ); + const hostOptions = $derived([ + { value: '', label: 'All hosts' }, + ...report.current.hostOptions.map((host) => ({ value: host.id, label: host.name })), + ...(report.query.hostId && + !report.current.hostOptions.some((host) => host.id === report.query.hostId) + ? [{ value: report.query.hostId, label: 'Unavailable host' }] + : []) + ]); + const gameOptions = [ + { value: 'all', label: 'All games' }, + { value: 't5', label: 'Black Ops I' }, + { value: 't6', label: 'Black Ops II' } + ]; + const ranges: { value: CapacityRange; label: string }[] = [ + { value: '1h', label: 'Last hour' }, + { value: '24h', label: '24 hours' }, + { value: '7d', label: '7 days' } + ]; const numbers = new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }); - function count(value: number | null) { return value === null ? '—' : numbers.format(value); } - function memory(value: number | null) { return value === null ? '—' : numbers.format(value / 1024 ** 3); } + function count(value: number | null) { + return value === null ? '—' : numbers.format(value); + } + function memory(value: number | null) { + return value === null ? '—' : numbers.format(value / 1024 ** 3); + } function queryParams(query: CapacityQuery) { const params = new URLSearchParams({ range: query.range, game: query.game }); if (query.hostId) params.set('host', query.hostId); @@ -52,7 +80,12 @@ } return `/capacity?${params}`; } - function cancel() { sequence++; request?.abort(); request = null; refreshing = false; } + function cancel() { + sequence++; + request?.abort(); + request = null; + refreshing = false; + } async function refresh(signal?: AbortSignal) { if (navigating || refreshing) return; const current = ++sequence; @@ -60,97 +93,422 @@ refreshing = true; try { const response = await fetch(`/api/capacity?${queryParams(data.query)}`, { - signal: AbortSignal.any([request.signal, AbortSignal.timeout(25_000), ...(signal ? [signal] : [])]) + signal: AbortSignal.any([ + request.signal, + AbortSignal.timeout(25_000), + ...(signal ? [signal] : []) + ]) }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not refresh capacity.'); - if (current === sequence) { snapshot = body; refreshError = ''; checkedAt = Date.now(); } + if (current === sequence) { + snapshot = body; + refreshError = ''; + checkedAt = Date.now(); + } } catch (cause) { - if (current === sequence && !signal?.aborted && !request?.signal.aborted) refreshError = (cause as Error).message; + if (current === sequence && !signal?.aborted && !request?.signal.aborted) + refreshError = (cause as Error).message; return false; - } finally { if (current === sequence) { refreshing = false; request = null; } } + } finally { + if (current === sequence) { + refreshing = false; + request = null; + } + } } async function changeScope(change: Partial) { navigationError = ''; - try { await goto(scopeUrl(change), { keepFocus: true, noScroll: true }); } - catch (cause) { navigating = false; navigationError = (cause as Error).message || 'Could not load that capacity view.'; } + try { + await goto(scopeUrl(change), { keepFocus: true, noScroll: true }); + } catch (cause) { + navigating = false; + navigationError = (cause as Error).message || 'Could not load that capacity view.'; + } } - beforeNavigate(() => { cancel(); navigating = true; }); - afterNavigate(() => { navigating = false; }); - $effect(() => { void data; untrack(() => { cancel(); snapshot = null; refreshError = ''; checkedAt = Date.now(); }); }); - $effect(() => { if (page.url.searchParams.get('rebalance') === '1') rebalanceOpen = true; }); - onMount(() => { const stop = pollVisible(refresh, 30_000, false); return () => { stop(); cancel(); }; }); + beforeNavigate(() => { + cancel(); + navigating = true; + }); + afterNavigate(() => { + navigating = false; + }); + $effect(() => { + void data; + untrack(() => { + cancel(); + snapshot = null; + refreshError = ''; + checkedAt = Date.now(); + }); + }); + $effect(() => { + if (page.url.searchParams.get('rebalance') === '1') rebalanceOpen = true; + }); + onMount(() => { + const stop = pollVisible(refresh, 30_000, false); + return () => { + stop(); + cancel(); + }; + }); -Capacity · Electric Cherry +Capacity · Electric Cherry
- {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet} {#if page.url.searchParams.has('returnTo')}{/if} - {#if refreshError || navigationError}{/if} - {#if rebalanceOpen}{#key `${page.url.searchParams.get('rebalanceHost') ?? ''}:${page.url.searchParams.get('rebalanceRun') ?? ''}`}{/key}{/if} + {#if refreshError || navigationError}{/if} + {#if rebalanceOpen}{#key `${page.url.searchParams.get('rebalanceHost') ?? ''}:${page.url.searchParams.get('rebalanceRun') ?? ''}`}{/key}{/if}
-
void changeScope({ game: (event.currentTarget as HTMLSelectElement).value as CapacityGame })} />
-
{#if navigating}Updating capacity…{:else if !refreshError}{/if}
+
+ + void changeScope({ + game: (event.currentTarget as HTMLSelectElement).value as CapacityGame + })} + /> +
+
+ {#if navigating}Updating capacity…{:else if !refreshError}{/if} +
-
-

{#if summary.estimatedServers === null}Capacity estimate unavailable{:else}Estimated room for {count(summary.estimatedServers)} more {plural(summary.estimatedServers, 'server')}{/if}

-

{scopeName} · {summary.eligibleHosts} eligible {summary.eligibleHosts === 1 ? 'host' : 'hosts'}{report.query.game === 'all' ? '' : ` · ${report.query.game === 't5' ? 'Black Ops I' : 'Black Ops II'}`}

-
- {#if summary.queued !== null && summary.queued > 0}{count(summary.queued)} {plural(summary.queued, 'server')} queued → - {:else}{summary.queued === null ? 'Queue unavailable' : 'No queued servers'}{/if}Fleet-wide -
+
+
+

+ {#if summary.estimatedServers === null}Capacity estimate unavailable{:else}Estimated + room for {count(summary.estimatedServers)} more {plural( + summary.estimatedServers, + 'server' + )}{/if} +

+

+ {scopeName} · {summary.eligibleHosts} eligible {summary.eligibleHosts === 1 + ? 'host' + : 'hosts'}{report.query.game === 'all' + ? '' + : ` · ${report.query.game === 't5' ? 'Black Ops I' : 'Black Ops II'}`} +

+
+
+ {#if summary.queued !== null && summary.queued > 0}{count(summary.queued)} + {plural(summary.queued, 'server')} queued → + {:else}{summary.queued === null ? 'Queue unavailable' : 'No queued servers'}{/if}Fleet-wide +
+
-
CPU available
{count(summary.cpuCoresAvailable)} cores
-
Memory available
{memory(summary.memoryBytesAvailable)} GiB
-
Free ports
{count(summary.portsAvailable)}
-
Concurrent starts · Fleet-wide
{summary.startupSlots === null ? 'Unavailable' : `${count(summary.startupSlots)}${summary.startupLimit === null ? '' : ` of ${summary.startupLimit}`} available`}
+
+
CPU available
+
{count(summary.cpuCoresAvailable)} cores
+
+
+
Memory available
+
{memory(summary.memoryBytesAvailable)} GiB
+
+
+
Free ports
+
{count(summary.portsAvailable)}
+
+
+
Concurrent starts · Fleet-wide
+
+ {summary.startupSlots === null + ? 'Unavailable' + : `${count(summary.startupSlots)}${summary.startupLimit === null ? '' : ` of ${summary.startupLimit}`} available`} +
+
- {#if report.current.warnings.length}{/if} + {#if report.current.warnings.length}{/if}
-

Host capacity

{summary.eligibleHosts} / {summary.totalHosts} eligible
- {#if !report.current.hosts.length}

{report.query.hostId ? 'This host is unavailable' : 'No hosts yet'}

{report.query.hostId ? 'Choose another host to view its capacity.' : 'Connect a host to start seeing capacity and history.'}

- {:else}{/if} +
+

Host capacity

+ {summary.eligibleHosts} / {summary.totalHosts} eligible +
+ {#if !report.current.hosts.length}
+

{report.query.hostId ? 'This host is unavailable' : 'No hosts yet'}

+

+ {report.query.hostId + ? 'Choose another host to view its capacity.' + : 'Connect a host to start seeing capacity and history.'} +

+ +
+ {:else}{/if}
-
Capacity details -

Additional servers are estimated from each eligible host’s available CPU, memory, and unreserved ports, using its learned running-server allowance. The smallest resource allowance sets that host’s limit. With All games selected, hosts ready for either Black Ops I or Black Ops II are counted once.

-

CPU and memory availability account for the operating-system reserve and pending starts. Free ports are counted on eligible hosts.

-

This is a planning estimate. Queued launches do not reserve capacity, and starting servers may require a larger allowance. Placement also checks current host readiness, game files, runtime, and concurrent start limits.

-

{summary.activeStartups === null ? 'The number of starting servers is unavailable.' : `${count(summary.activeStartups)} ${plural(summary.activeStartups, 'server')} starting across the fleet.`} Servers wait for capacity and a concurrent start slot automatically.

-

{report.current.updatedAt ? `Report updated ${$timeFormat.dateTime(report.current.updatedAt)}.` : 'Waiting for host reports.'} Refreshes every 30 seconds while visible.

+
+ Capacity details +

+ Additional servers are estimated from each eligible host’s available CPU, memory, and + unreserved ports, using its learned running-server allowance. The smallest resource + allowance sets that host’s limit. With All games selected, hosts ready for either Black + Ops I or Black Ops II are counted once. +

+

+ CPU and memory availability account for the operating-system reserve and pending starts. + Free ports are counted on eligible hosts. +

+

+ This is a planning estimate. Queued launches do not reserve capacity, and starting + servers may require a larger allowance. Placement also checks current host readiness, + game files, runtime, and concurrent start limits. +

+

+ {summary.activeStartups === null + ? 'The number of starting servers is unavailable.' + : `${count(summary.activeStartups)} ${plural(summary.activeStartups, 'server')} starting across the fleet.`} + Servers wait for capacity and a concurrent start slot automatically. +

+

+ {report.current.updatedAt + ? `Report updated ${$timeFormat.dateTime(report.current.updatedAt)}.` + : 'Waiting for host reports.'} Refreshes every 30 seconds while visible. +

- {#snippet controls()}{/snippet} + {#snippet controls()}{/snippet}
diff --git a/src/routes/collections/+page.svelte b/src/routes/collections/+page.svelte index 4076a8f..113babf 100644 --- a/src/routes/collections/+page.svelte +++ b/src/routes/collections/+page.svelte @@ -31,39 +31,61 @@ let tab = $state('collections'); let limit = $state(25); let launchLimit = $state(10); - const canOperate = $derived($page.data.user?.role === 'admin' || $page.data.user?.role === 'operator'); + const canOperate = $derived( + $page.data.user?.role === 'admin' || $page.data.user?.role === 'operator' + ); const inProgress = (launch: CollectionSummary) => launch.active > 0 || launch.cleanupActive > 0; const activeLaunches = $derived(launches.filter(inProgress)); - const launchHistory = $derived(launches.filter(launch => !inProgress(launch))); + const launchHistory = $derived(launches.filter((launch) => !inProgress(launch))); const visibleLaunches = $derived(tab === 'active' ? activeLaunches : launchHistory); const tabs = $derived([ { id: 'collections', label: 'Your collections' }, - { id: 'active', label: 'Active launches', badge: launchesLoading ? undefined : activeLaunches.length }, + { + id: 'active', + label: 'Active launches', + badge: launchesLoading ? undefined : activeLaunches.length + }, { id: 'history', label: 'Launch history' } ]); function launchLabel(collection: CollectionSummary) { - if (collection.cancelled_at) return inProgress(collection) ? 'Cancellation requested' : collection.failed || collection.cleanupFailed ? 'Cancelled · needs attention' : 'Cancelled'; + if (collection.cancelled_at) + return inProgress(collection) + ? 'Cancellation requested' + : collection.failed || collection.cleanupFailed + ? 'Cancelled · needs attention' + : 'Cancelled'; if (collection.cleanupActive) return 'Cleaning up'; - if (collection.active) return collection.provisioning + collection.verifying ? collection.mode === 'exact' ? 'Applying collection' : 'Launching' : 'Queued'; + if (collection.active) + return collection.provisioning + collection.verifying + ? collection.mode === 'exact' + ? 'Applying collection' + : 'Launching' + : 'Queued'; if (collection.failed || collection.cleanupFailed) return 'Needs attention'; return collection.cancelled ? 'Finished with cancellations' : 'Completed'; } function launchVariant(collection: CollectionSummary) { - if (collection.cleanupFailed || (!inProgress(collection) && collection.failed)) return 'danger'; + if (collection.cleanupFailed || (!inProgress(collection) && collection.failed)) + return 'danger'; if (inProgress(collection)) return 'info'; return collection.cancelled_at || collection.cancelled ? 'neutral' : 'online'; } $effect(() => { const params = $page.url.searchParams; - tab = ['active', 'history'].includes(params.get('view') ?? '') ? params.get('view')! : 'collections'; + tab = ['active', 'history'].includes(params.get('view') ?? '') + ? params.get('view')! + : 'collections'; limit = Math.max(25, Number(params.get('limit')) || 25); launchLimit = Math.max(10, Number(params.get('launches')) || 10); }); function saveContext() { const url = new URL($page.url); - if (limit > 25) url.searchParams.set('limit', String(limit)); else url.searchParams.delete('limit'); - if (tab !== 'collections') url.searchParams.set('view', tab); else url.searchParams.delete('view'); - if (launchLimit > 10) url.searchParams.set('launches', String(launchLimit)); else url.searchParams.delete('launches'); + if (limit > 25) url.searchParams.set('limit', String(limit)); + else url.searchParams.delete('limit'); + if (tab !== 'collections') url.searchParams.set('view', tab); + else url.searchParams.delete('view'); + if (launchLimit > 10) url.searchParams.set('launches', String(launchLimit)); + else url.searchParams.delete('launches'); void goto(url, { replaceState: true, noScroll: true, keepFocus: true }); } onMount(() => { @@ -71,8 +93,13 @@ return pollVisible(loadLaunches, 5000); }); function blueprintSaved(blueprint: BlueprintSummary, launch: boolean) { - blueprints = [...blueprints.filter(item => item.id !== blueprint.id), blueprint].sort((a, b) => a.name.localeCompare(b.name)); - if (launch) { blueprintId = blueprint.id; collectionOpen = true; } + blueprints = [...blueprints.filter((item) => item.id !== blueprint.id), blueprint].sort( + (a, b) => a.name.localeCompare(b.name) + ); + if (launch) { + blueprintId = blueprint.id; + collectionOpen = true; + } } async function loadBlueprints() { try { @@ -80,8 +107,11 @@ if (!res.ok) throw new Error('Could not load your collections.'); blueprints = (await res.json()).blueprints; blueprintError = ''; - } catch (err) { blueprintError = (err as Error).message; } - finally { blueprintsLoading = false; } + } catch (err) { + blueprintError = (err as Error).message; + } finally { + blueprintsLoading = false; + } } async function loadLaunches() { if (launchesRefreshing) return; @@ -91,104 +121,447 @@ if (!res.ok) throw new Error('Could not load collection launches.'); launches = (await res.json()).collections; launchError = ''; - } catch (err) { launchError = (err as Error).message; } - finally { launchesLoading = false; launchesRefreshing = false; } + } catch (err) { + launchError = (err as Error).message; + } finally { + launchesLoading = false; + launchesRefreshing = false; + } } async function removeBlueprint(id: string) { - if (!await confirmAction({title:'Remove collection?', message:'Existing servers and launch history will be kept.', confirmLabel:'Remove collection', danger:true})) return; - removing = id; blueprintError = ''; + if ( + !(await confirmAction({ + title: 'Remove collection?', + message: 'Existing servers and launch history will be kept.', + confirmLabel: 'Remove collection', + danger: true + })) + ) + return; + removing = id; + blueprintError = ''; try { - const res = await fetch('/api/blueprints/' + encodeURIComponent(id), { method: 'DELETE' }); + const res = await fetch('/api/blueprints/' + encodeURIComponent(id), { + method: 'DELETE' + }); if (!res.ok) throw new Error('Could not remove collection.'); - blueprints = blueprints.filter(b => b.id !== id); - } catch (err) { blueprintError = (err as Error).message; } - finally { removing = ''; } + blueprints = blueprints.filter((b) => b.id !== id); + } catch (err) { + blueprintError = (err as Error).message; + } finally { + removing = ''; + } } + Collections | Electric Cherry
- - {#snippet actions()}{#if canOperate && (blueprintsLoading || blueprints.length || blueprintError)}{/if}{/snippet} + + {#snippet actions()}{#if canOperate && (blueprintsLoading || blueprints.length || blueprintError)}{/if}{/snippet} - {#if canOperate} - {/if} - { launchLimit = 10; saveContext(); }} ariaLabel="Collection views" idPrefix="collection-tabs" panelId="collection-panel" /> -
- {#if tab === 'collections'} -
-

Your collections {blueprints.length}

- {#if blueprintError}{/if} - {#if blueprintsLoading}

Loading collections…

- {:else if !blueprints.length && !blueprintError} - {#snippet actions()}{#if canOperate}{/if}{/snippet} - + {#if canOperate} + {/if} + { + launchLimit = 10; + saveContext(); + }} + ariaLabel="Collection views" + idPrefix="collection-tabs" + panelId="collection-panel" + /> +
+ {#if tab === 'collections'} +
+

Your collections {blueprints.length}

+ {#if blueprintError}{/if} + {#if blueprintsLoading}

Loading collections…

+ {:else if !blueprints.length && !blueprintError} + {#snippet actions()}{#if canOperate}{/if}{/snippet} + + {:else} +
+ {#each blueprints.slice(0, limit) as blueprint (blueprint.id)} +
+
+ Server collection + {blueprint.totalServers} target server{blueprint.totalServers === + 1 + ? '' + : 's'} +
+

{blueprint.name}

+

+ {blueprint.description || + 'Fill missing lobbies or match your fleet to this lineup exactly.'} +

+
    + {#each blueprint.entries as entry}
  • + {entry.count}× + {entry.mapName} + {entry.game.toUpperCase()} +
    • {/each} +
+ {#if canOperate}
+ + + +
{/if} +
+ {/each} +
+ {/if} +
+ saveContext()} + label="collections" + /> {:else} -
- {#each blueprints.slice(0,limit) as blueprint (blueprint.id)} -
-
- Server collection - {blueprint.totalServers} target server{blueprint.totalServers === 1 ? '' : 's'} -
-

{blueprint.name}

-

{blueprint.description || 'Fill missing lobbies or match your fleet to this lineup exactly.'}

-
    {#each blueprint.entries as entry}
  • {entry.count}× {entry.mapName} {entry.game.toUpperCase()}
    • {/each}
- {#if canOperate}
- - - -
{/if} -
- {/each} -
+
+

+ {tab === 'active' ? 'Collection launches' : 'Launch history'} + {visibleLaunches.length} +

+ {#if launchError}{/if} + {#if launchesLoading}

Loading collection launches…

+ {:else if visibleLaunches.length} +
+ {#each visibleLaunches.slice(0, launchLimit) as collection (collection.id)} + +
+ {collection.name}{launchLabel(collection)} +
+
+ {collection.done} / {collection.total} + {collection.mode === 'exact' ? 'completed' : 'ready'}{#if collection.provisioning + collection.verifying}{collection.provisioning + collection.verifying} + {collection.mode === 'exact' + ? 'in progress' + : 'launching'}{/if}{#if collection.queued}{collection.queued} queued{/if}{#if collection.failed}{collection.failed} + {plural(collection.failed, 'needs', 'need')} attention{/if}{#if collection.cancelled}{collection.cancelled} cancelled{/if}{#if collection.cleanupActive}{collection.cleanupActive} cleaning up{/if}{#if collection.cleanupFailed}{collection.cleanupFailed} + {plural(collection.cleanupFailed, 'cleanup')} + {plural(collection.cleanupFailed, 'needs', 'need')} attention{/if} +
+ +
+ Queued {inProgress(collection) + ? 'Follow progress' + : 'View results'} → +
+ + {/each} +
+ {:else if !launchError} + + {#snippet actions()}{/snippet} + + {/if} + saveContext()} + step={10} + label="launches" + /> +
{/if} -
- saveContext()} label="collections" /> - {:else} -
-

{tab === 'active' ? 'Collection launches' : 'Launch history'} {visibleLaunches.length}

- {#if launchError}{/if} - {#if launchesLoading}

Loading collection launches…

- {:else if visibleLaunches.length} -
{#each visibleLaunches.slice(0,launchLimit) as collection (collection.id)} - -
{collection.name}{launchLabel(collection)}
-
{collection.done} / {collection.total} {collection.mode === 'exact' ? 'completed' : 'ready'}{#if collection.provisioning + collection.verifying}{collection.provisioning + collection.verifying} {collection.mode === 'exact' ? 'in progress' : 'launching'}{/if}{#if collection.queued}{collection.queued} queued{/if}{#if collection.failed}{collection.failed} {plural(collection.failed, 'needs', 'need')} attention{/if}{#if collection.cancelled}{collection.cancelled} cancelled{/if}{#if collection.cleanupActive}{collection.cleanupActive} cleaning up{/if}{#if collection.cleanupFailed}{collection.cleanupFailed} {plural(collection.cleanupFailed, 'cleanup')} {plural(collection.cleanupFailed, 'needs', 'need')} attention{/if}
- -
Queued {inProgress(collection) ? 'Follow progress' : 'View results'} →
- - {/each}
- {:else if !launchError} - - {#snippet actions()}{/snippet} - - {/if} - saveContext()} step={10} label="launches" /> -
- {/if}
+ diff --git a/src/routes/collections/launches/[id]/+page.svelte b/src/routes/collections/launches/[id]/+page.svelte index 3019619..a8ed504 100644 --- a/src/routes/collections/launches/[id]/+page.svelte +++ b/src/routes/collections/launches/[id]/+page.svelte @@ -21,27 +21,64 @@ let cancellationAccepted = $state(false); let requestNumber = 0; let collection = $derived(detail?.collection); - const launching = (server: { state: string; server_deleted_at?: string | null }) => !server.server_deleted_at && ['running', 'awaiting-health'].includes(server.state); - const needsAttention = (server: { state: string }) => ['failed', 'interrupted', 'warning'].includes(server.state); - const cleanupNeedsAttention = (cancellation: NonNullable) => needsAttention(cancellation) || cancellation.state === 'cancelled'; - const canOperate = $derived($page.data.user?.role === 'admin' || $page.data.user?.role === 'operator'); + const launching = (server: { state: string; server_deleted_at?: string | null }) => + !server.server_deleted_at && ['running', 'awaiting-health'].includes(server.state); + const needsAttention = (server: { state: string }) => + ['failed', 'interrupted', 'warning'].includes(server.state); + const cleanupNeedsAttention = (cancellation: NonNullable) => + needsAttention(cancellation) || cancellation.state === 'cancelled'; + const canOperate = $derived( + $page.data.user?.role === 'admin' || $page.data.user?.role === 'operator' + ); const cancellationRequested = $derived(!!collection?.cancelled_at || cancellationAccepted); - const inProgress = $derived(!!collection && (collection.active > 0 || collection.cleanupActive > 0)); + const inProgress = $derived( + !!collection && (collection.active > 0 || collection.cleanupActive > 0) + ); const exact = $derived(collection?.mode === 'exact'); const currentServers = $derived(detail?.servers.filter(launching) ?? []); const heading = $derived.by(() => { if (!collection) return 'Collection launch'; - if (cancellationRequested) return inProgress ? 'Cancellation requested' : collection.failed || collection.cleanupFailed ? 'Cancellation needs attention' : 'Collection launch cancelled'; + if (cancellationRequested) + return inProgress + ? 'Cancellation requested' + : collection.failed || collection.cleanupFailed + ? 'Cancellation needs attention' + : 'Collection launch cancelled'; if (collection.cleanupActive) return 'Cleaning up cancelled launches'; - if (collection.active) return exact ? 'Applying your collection' : currentServers.length ? 'Bringing your lineup online' : 'Your lineup is queued'; - if (collection.failed || collection.cleanupFailed) return exact ? 'Collection finished with issues' : 'Launch complete with issues'; + if (collection.active) + return exact + ? 'Applying your collection' + : currentServers.length + ? 'Bringing your lineup online' + : 'Your lineup is queued'; + if (collection.failed || collection.cleanupFailed) + return exact ? 'Collection finished with issues' : 'Launch complete with issues'; if (collection.cancelled) return 'Collection finished with cancellations'; return exact ? 'Collection applied' : 'Your lineup is ready'; }); - const otherLaunches = $derived(detail?.queue.active.filter(active => !detail?.servers.some(server => server.id === active.id)) ?? []); - const firstQueued = $derived(detail?.servers.find(server => !server.server_deleted_at && server.state === 'queued')); - const labels: Record = { queued: 'Queued', running: 'Launching', 'awaiting-health': 'Checking readiness', done: 'Ready', warning: 'Needs review', failed: 'Failed', interrupted: 'Interrupted', waiting: 'Waiting', cancelled: 'Cancelled' }; - const waitingForPort = (server: CollectionServer) => server.state === 'queued' && (server.stage.startsWith('Waiting for removal') || server.stage.startsWith('Waiting for port')); + const otherLaunches = $derived( + detail?.queue.active.filter( + (active) => !detail?.servers.some((server) => server.id === active.id) + ) ?? [] + ); + const firstQueued = $derived( + detail?.servers.find((server) => !server.server_deleted_at && server.state === 'queued') + ); + const labels: Record = { + queued: 'Queued', + running: 'Launching', + 'awaiting-health': 'Checking readiness', + done: 'Ready', + warning: 'Needs review', + failed: 'Failed', + interrupted: 'Interrupted', + waiting: 'Waiting', + cancelled: 'Cancelled' + }; + const waitingForPort = (server: CollectionServer) => + server.state === 'queued' && + (server.stage.startsWith('Waiting for removal') || + server.stage.startsWith('Waiting for port')); function stateLabel(server: CollectionServer) { if (waitingForPort(server)) return 'Waiting for removal'; if (server.action === 'destroy' && server.state === 'done') return 'Removed'; @@ -52,7 +89,8 @@ function queueHint(server: CollectionServer) { const reason = server.stage || 'Waiting for a startup slot and available host capacity.'; return server.queuePosition && server.queuePosition > 1 - ? `${reason} ${server.queuePosition - 1} queued server${server.queuePosition === 2 ? '' : 's'} ahead.` : reason; + ? `${reason} ${server.queuePosition - 1} queued server${server.queuePosition === 2 ? '' : 's'} ahead.` + : reason; } async function load() { @@ -62,15 +100,22 @@ const res = await fetch(`/api/deployment-collections/${encodeURIComponent(id)}`); if (!res.ok) throw new Error('Could not refresh this collection.'); const body = await res.json(); - if (current === requestNumber && id === $page.params.id) { detail = body; error = ''; } - } catch (err) { if (current === requestNumber) error = (err as Error).message; } + if (current === requestNumber && id === $page.params.id) { + detail = body; + error = ''; + } + } catch (err) { + if (current === requestNumber) error = (err as Error).message; + } } async function cancelLaunch() { - if (!collection || !detail?.canCancel || cancellationRequested || cancelling || !canOperate) return; + if (!collection || !detail?.canCancel || cancellationRequested || cancelling || !canOperate) + return; const id = collection.id; const accepted = await confirmAction({ title: 'Cancel collection launch?', - message: 'Stops unfinished server creations, including eligible failed or interrupted launches, and deletes their containers and instance files. This can disconnect active players, even when player status is unknown. Shared game files are kept. Keys and ports are released only after removal is confirmed.\n\nCompleted launches and retained servers are kept. Pending removals are cancelled where safe; removals that have already started or are shared with another collection may still finish.', + message: + 'Stops unfinished server creations, including eligible failed or interrupted launches, and deletes their containers and instance files. This can disconnect active players, even when player status is unknown. Shared game files are kept. Keys and ports are released only after removal is confirmed.\n\nCompleted launches and retained servers are kept. Pending removals are cancelled where safe; removals that have already started or are shared with another collection may still finish.', confirmLabel: 'Cancel launch and clean up', danger: true }); @@ -78,7 +123,10 @@ cancelling = true; cancelError = ''; try { - const res = await fetch(`/api/deployment-collections/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); + const res = await fetch( + `/api/deployment-collections/${encodeURIComponent(id)}/cancel`, + { method: 'POST' } + ); const body = await res.json(); if (!res.ok) throw new Error(body.error || 'Could not cancel this collection launch.'); if (id !== $page.params.id) return; @@ -89,48 +137,139 @@ cancelError = (err as Error).message; await load(); } - } finally { if (id === $page.params.id) cancelling = false; } + } finally { + if (id === $page.params.id) cancelling = false; + } } - $effect(() => { $page.params.id; detail = null; cancelError = ''; cancelling = false; cancellationAccepted = false; void load(); }); - onMount(() => { const stop = pollVisible(load, 5000, false); return () => { stop(); requestNumber++; }; }); + $effect(() => { + void $page.params.id; + detail = null; + cancelError = ''; + cancelling = false; + cancellationAccepted = false; + void load(); + }); + onMount(() => { + const stop = pollVisible(load, 5000, false); + return () => { + stop(); + requestNumber++; + }; + }); -{collection?.name ?? 'Collection launch'} | Electric Cherry +{collection?.name ?? 'Collection launch'} | Electric Cherry
- {#snippet crumbs()}{/snippet} + {#snippet crumbs()}{/snippet} {#if error}{/if} {#if collection && detail}
-
Collection progress

{heading}

- {collection.done + collection.failed + collection.cancelled} / {collection.total} finished +
+ Collection progress +

{heading}

+
+ {collection.done + collection.failed + collection.cancelled} / {collection.total} + finished
-
- - - +
+ + +
-
0}> -
{exact ? 'Completed' : 'Ready'}
{collection.done}
-
{exact ? 'In progress' : 'Launching'}
{collection.provisioning + collection.verifying}
-
Queued
{collection.queued}
-
0}>
Need attention
{collection.failed}
- {#if cancellationRequested || collection.cancelled > 0}
Cancelled
{collection.cancelled}
{/if} +
0} + > +
+
{exact ? 'Completed' : 'Ready'}
+
{collection.done}
+
+
+
{exact ? 'In progress' : 'Launching'}
+
{collection.provisioning + collection.verifying}
+
+
+
Queued
+
{collection.queued}
+
+
0}> +
Need attention
+
{collection.failed}
+
+ {#if cancellationRequested || collection.cancelled > 0}
+
Cancelled
+
{collection.cancelled}
+
{/if}
-

{collection.done} completed, {collection.provisioning + collection.verifying} in progress, {collection.queued} queued, {collection.failed} {plural(collection.failed, 'needs', 'need')} attention, {collection.cancelled} cancelled, {collection.cleanupActive} cleaning up, {collection.cleanupFailed} {plural(collection.cleanupFailed, 'cleanup')} {plural(collection.cleanupFailed, 'needs', 'need')} attention.

+

+ {collection.done} completed, {collection.provisioning + collection.verifying} in progress, + {collection.queued} queued, {collection.failed} + {plural(collection.failed, 'needs', 'need')} attention, {collection.cancelled} cancelled, + {collection.cleanupActive} cleaning up, {collection.cleanupFailed} + {plural(collection.cleanupFailed, 'cleanup')} + {plural(collection.cleanupFailed, 'needs', 'need')} attention. +

{#if cancellationRequested} -

Completed launches and retained servers are kept. Removals that already started or are shared with another collection may still finish. Follow each cancelled server's cleanup below until removal is confirmed.

+

+ Completed launches and retained servers are kept. Removals that already started + or are shared with another collection may still finish. Follow each cancelled + server's cleanup below until removal is confirmed. +

{/if} {#if collection.cleanupActive || collection.cleanupFailed}
- {#if collection.cleanupActive}

{collection.cleanupActive} server cleanup{collection.cleanupActive === 1 ? '' : 's'} in progress. Keys and ports remain reserved until removal is confirmed.

{/if} - {#if collection.cleanupFailed}

{collection.cleanupFailed} cleanup{collection.cleanupFailed === 1 ? ' needs' : 's need'} attention. Open the cleanup operation below to review and recover.

{/if} + {#if collection.cleanupActive}

+ {collection.cleanupActive} server cleanup{collection.cleanupActive === 1 + ? '' + : 's'} in progress. Keys and ports remain reserved until removal is confirmed. +

{/if} + {#if collection.cleanupFailed}

+ {collection.cleanupFailed} cleanup{collection.cleanupFailed === 1 + ? ' needs' + : 's need'} attention. Open the cleanup operation below to review and + recover. +

{/if}
{/if} {#if canOperate && detail.canCancel && !cancellationRequested} -

Stops unfinished launches and removes their containers and instance files. Completed launches and retained servers are kept.

+
+ +

+ Stops unfinished launches and removes their containers and instance files. + Completed launches and retained servers are kept. +

+
{/if} {#if cancelError}{/if}
@@ -139,49 +278,170 @@
In progress {#each currentServers as server (server.id)} - + -
{server.action === 'destroy' ? 'Removing: ' : ''}

{server.state === 'awaiting-health' ? 'Checking the container, connection, and expected map.' : server.stage}

- Follow progress +
+ {server.action === 'destroy' + ? 'Removing: ' + : ''} +

+ {server.state === 'awaiting-health' + ? 'Checking the container, connection, and expected map.' + : server.stage} +

+
+ Follow progress {/each}
{:else if firstQueued} {/if} {#if exact && detail.retainedServers.length} -
{detail.retainedServers.length} retained server{detail.retainedServers.length === 1 ? '' : 's'} (unchanged) -
    {#each detail.retainedServers as name}
  • {name}
    • {/each}
+
+ {detail.retainedServers.length} retained server{detail.retainedServers + .length === 1 + ? '' + : 's'} (unchanged) +
    + {#each detail.retainedServers as name}
  • + {name} +
    • {/each} +
{/if}
-

{exact ? 'Fleet changes' : 'Launch order'}

{collection.total} operation{collection.total === 1 ? '' : 's'}
+
+

{exact ? 'Fleet changes' : 'Launch order'}

+ {collection.total} operation{collection.total === 1 ? '' : 's'} +
    {#each detail.servers as server, index (server.id)} -
  1. - - +
  2. + +
    -
    {stateLabel(server)}
    -

    {server.error ?? (server.state === 'queued' ? queueHint(server) : server.state === 'done' ? server.action === 'destroy' ? server.stage : 'Startup checks passed.' : server.stage)}

    -
    {server.action === 'destroy' ? 'Destroy server' : server.port === null ? 'Create server · Port not assigned yet' : `Create server · Port ${server.port}`}{needsAttention(server) ? 'Review & recover' : 'View operation'}
    +
    + {stateLabel(server)} +
    +

    + {server.error ?? + (server.state === 'queued' + ? queueHint(server) + : server.state === 'done' + ? server.action === 'destroy' + ? server.stage + : 'Startup checks passed.' + : server.stage)} +

    +
    + {server.action === 'destroy' + ? 'Destroy server' + : server.port === null + ? 'Create server · Port not assigned yet' + : `Create server · Port ${server.port}`}{needsAttention(server) + ? 'Review & recover' + : 'View operation'} + +
    {#if server.cancellation} - - {server.cancellation.state === 'done' ? 'Cleanup completed' : cleanupNeedsAttention(server.cancellation) ? 'Cleanup needs attention' : 'Cleanup in progress'} - {server.cancellation.error ?? server.cancellation.stage} - {cleanupNeedsAttention(server.cancellation) ? 'Review cleanup' : 'View cleanup progress'} + + {server.cancellation.state === 'done' + ? 'Cleanup completed' + : cleanupNeedsAttention(server.cancellation) + ? 'Cleanup needs attention' + : 'Cleanup in progress'} + {server.cancellation.error ?? server.cancellation.stage} + {cleanupNeedsAttention(server.cancellation) + ? 'Review cleanup' + : 'View cleanup progress'} + {/if}
    3. @@ -192,17 +452,406 @@
diff --git a/src/routes/deployments/[...path]/+page.server.ts b/src/routes/deployments/[...path]/+page.server.ts index d1599fe..e26da6c 100644 --- a/src/routes/deployments/[...path]/+page.server.ts +++ b/src/routes/deployments/[...path]/+page.server.ts @@ -3,7 +3,10 @@ import { redirect } from '@sveltejs/kit'; /** Preserve bookmarked operation links and list filters. */ export function load({ url }: { url: URL }) { if (url.pathname.startsWith('/deployments/collections/')) { - redirect(308, url.pathname.replace('/deployments/collections/', '/collections/launches/') + url.search); + redirect( + 308, + url.pathname.replace('/deployments/collections/', '/collections/launches/') + url.search + ); } redirect(308, url.pathname.replace(/^\/deployments(?=\/|$)/, '/provisioning') + url.search); } diff --git a/src/routes/game-files/+page.svelte b/src/routes/game-files/+page.svelte index 8ba38a5..f03dfa3 100644 --- a/src/routes/game-files/+page.svelte +++ b/src/routes/game-files/+page.svelte @@ -15,150 +15,332 @@ import Badge from '$ui/components/primitives/Badge.svelte'; import GameFileHosts from '$ui/components/GameFileHosts.svelte'; - type OverviewLibrary = Omit & { versions: Omit[] }; - interface UploadFile { path: string; size: number; lastModified: number; receivedBytes: number; complete: boolean } - interface UploadStatus { status: 'pending' | 'verifying' | 'published' | 'failed'; chunkSize: number; files: UploadFile[]; error?: string | null; version: { id: string; state: string } } + type OverviewLibrary = Omit & { + versions: Omit[]; + }; + interface UploadFile { + path: string; + size: number; + lastModified: number; + receivedBytes: number; + complete: boolean; + } + interface UploadStatus { + status: 'pending' | 'verifying' | 'published' | 'failed'; + chunkSize: number; + files: UploadFile[]; + error?: string | null; + version: { id: string; state: string }; + } type Phase = 'idle' | 'preparing' | 'uploading' | 'paused' | 'verifying' | 'published'; - let libraries = $state([]); let hosts = $state([]); - let loaded = $state(false); let error = $state(''); let notice = $state(''); let busy = $state(false); - let showUpload = $state(false); let game = $state<'t5' | 't6'>('t5'); + let libraries = $state([]); + let hosts = $state([]); + let loaded = $state(false); + let error = $state(''); + let notice = $state(''); + let busy = $state(false); + let showUpload = $state(false); + let game = $state<'t5' | 't6'>('t5'); let targets = $state>({}); let folder = $state.raw(null); let uploadStatus = $state.raw(null); - let uploadId = $state(''); let uploadBusy = $state(false); let uploadError = $state(''); - let phase = $state('idle'); let currentPath = $state(''); let pauseRequested = $state(false); - let acceptedBytes = $state(0); let sendingBytes = $state(0); let completedFiles = $state(0); + let uploadId = $state(''); + let uploadBusy = $state(false); + let uploadError = $state(''); + let phase = $state('idle'); + let currentPath = $state(''); + let pauseRequested = $state(false); + let acceptedBytes = $state(0); + let sendingBytes = $state(0); + let completedFiles = $state(0); let folderInput = $state(); - const activeRequests = new Map(); let disposed = false; let failedTransfer = false; - const updateSending = () => { sendingBytes = [...activeRequests.values()].reduce((a, b) => a + b, 0); }; - const abortUploads = () => { for (const request of activeRequests.keys()) request.abort(); }; + const activeRequests = new Map(); + let disposed = false; + let failedTransfer = false; + const updateSending = () => { + sendingBytes = [...activeRequests.values()].reduce((a, b) => a + b, 0); + }; + const abortUploads = () => { + for (const request of activeRequests.keys()) request.abort(); + }; const canManage = $derived($page.data.user?.role === 'admin'); - const onlineHosts = $derived(hosts.filter(host => host.connected && host.credentialActive && host.mode !== 'disabled')); - const totalBytes = $derived(folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0); + const onlineHosts = $derived( + hosts.filter((host) => host.connected && host.credentialActive && host.mode !== 'disabled') + ); + const totalBytes = $derived( + folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0 + ); const totalFiles = $derived(folder?.files.length ?? uploadStatus?.files.length ?? 0); const progressBytes = $derived(Math.min(totalBytes, acceptedBytes + sendingBytes)); const transferring = $derived(uploadBusy && (phase === 'preparing' || phase === 'uploading')); - const hostName = (id: string | null) => id ? hosts.find(host => host.id === id)?.name ?? id : 'Browser upload'; - const gameName = (game: 't5' | 't6') => game === 't5' ? 'Black Ops I' : 'Black Ops II'; + const hostName = (id: string | null) => + id ? (hosts.find((host) => host.id === id)?.name ?? id) : 'Browser upload'; + const gameName = (game: 't5' | 't6') => (game === 't5' ? 'Black Ops I' : 'Black Ops II'); async function responseBody(response: Response) { let body; - try { body = await response.json(); } catch { throw new Error('The controller returned an unexpected response. Sign in again if your session expired, then resume the upload.'); } - if (!response.ok) throw new Error(body.error ?? 'The controller could not complete this request.'); + try { + body = await response.json(); + } catch { + throw new Error( + 'The controller returned an unexpected response. Sign in again if your session expired, then resume the upload.' + ); + } + if (!response.ok) + throw new Error(body.error ?? 'The controller could not complete this request.'); return body; } async function load() { try { const body = await responseBody(await fetch('/api/game-files')); - libraries = body.libraries; hosts = body.hosts; loaded = true; - } catch (cause) { error = (cause as Error).message; } + libraries = body.libraries; + hosts = body.hosts; + loaded = true; + } catch (cause) { + error = (cause as Error).message; + } } async function download(versionId: string) { - busy = true; error = ''; notice = ''; + busy = true; + error = ''; + notice = ''; try { - await responseBody(await fetch(`/api/game-files/${versionId}/download`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ hostId: targets[versionId] }) })); - notice = 'Game files queued for download. Progress will update below.'; await load(); - } catch (cause) { error = (cause as Error).message; } finally { busy = false; } + await responseBody( + await fetch(`/api/game-files/${versionId}/download`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ hostId: targets[versionId] }) + }) + ); + notice = 'Game files queued for download. Progress will update below.'; + await load(); + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } } function resetUpload() { - folder = null; uploadStatus = null; uploadId = ''; phase = 'idle'; uploadError = ''; currentPath = ''; - acceptedBytes = 0; sendingBytes = 0; completedFiles = 0; pauseRequested = false; + folder = null; + uploadStatus = null; + uploadId = ''; + phase = 'idle'; + uploadError = ''; + currentPath = ''; + acceptedBytes = 0; + sendingBytes = 0; + completedFiles = 0; + pauseRequested = false; if (folderInput) folderInput.value = ''; } async function newUpload(library: OverviewLibrary) { - resetUpload(); showUpload = true; game = library.game; - await tick(); document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' }); + resetUpload(); + showUpload = true; + game = library.game; + await tick(); + document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' }); } function chooseFolder(event: Event) { - uploadError = ''; folder = null; + uploadError = ''; + folder = null; const input = event.currentTarget as HTMLInputElement; if (!input.files?.length) return; try { const selected = selectGameFolder(Array.from(input.files)); if (uploadStatus) checkSelection(selected, uploadStatus); folder = selected; - } catch (cause) { uploadError = (cause as Error).message; input.value = ''; } + } catch (cause) { + uploadError = (cause as Error).message; + input.value = ''; + } } function checkSelection(selected: SelectedGameFolder, status: UploadStatus) { - const originals = new Map(status.files.map(file => [file.path, file])); - if (selected.files.length !== originals.size || selected.files.some(({ path, file }) => { - const original = originals.get(path); return !original || original.size !== file.size || original.lastModified !== file.lastModified; - })) throw new Error('This folder does not match the original upload. Select the same unchanged folder, including its original file names, sizes, and modification dates. Start a new upload for changed game files.'); + const originals = new Map(status.files.map((file) => [file.path, file])); + if ( + selected.files.length !== originals.size || + selected.files.some(({ path, file }) => { + const original = originals.get(path); + return ( + !original || + original.size !== file.size || + original.lastModified !== file.lastModified + ); + }) + ) + throw new Error( + 'This folder does not match the original upload. Select the same unchanged folder, including its original file names, sizes, and modification dates. Start a new upload for changed game files.' + ); } function applyStatus(status: UploadStatus) { uploadStatus = status; acceptedBytes = status.files.reduce((total, file) => total + file.receivedBytes, 0); - completedFiles = status.files.filter(file => file.complete).length; sendingBytes = 0; + completedFiles = status.files.filter((file) => file.complete).length; + sendingBytes = 0; } async function getStatus(): Promise { - const status = await responseBody(await fetch(`/api/game-files/${uploadId}/upload`)) as UploadStatus; - applyStatus(status); return status; + const status = (await responseBody( + await fetch(`/api/game-files/${uploadId}/upload`) + )) as UploadStatus; + applyStatus(status); + return status; } async function followVerification(initial: UploadStatus) { - let status = initial; phase = 'verifying'; currentPath = ''; + let status = initial; + phase = 'verifying'; + currentPath = ''; while (!disposed && status.status === 'verifying') { - await new Promise(resolve => setTimeout(resolve, 1500)); + await new Promise((resolve) => setTimeout(resolve, 1500)); if (disposed) return; status = await getStatus(); } if (disposed) return; - if (status.status !== 'published') throw new Error(status.error || 'Verification did not finish. Retry this upload to continue.'); - phase = 'published'; folder = null; notice = `${gameTitle(game, true)} game files are ready. Active hosts will download and verify the latest files automatically.`; + if (status.status !== 'published') + throw new Error( + status.error || 'Verification did not finish. Retry this upload to continue.' + ); + phase = 'published'; + folder = null; + notice = `${gameTitle(game, true)} game files are ready. Active hosts will download and verify the latest files automatically.`; if (folderInput) folderInput.value = ''; await load(); } - async function resumeUpload(library: OverviewLibrary, item: OverviewLibrary['versions'][number]) { - resetUpload(); showUpload = true; uploadId = item.id; game = library.game; + async function resumeUpload( + library: OverviewLibrary, + item: OverviewLibrary['versions'][number] + ) { + resetUpload(); + showUpload = true; + uploadId = item.id; + game = library.game; uploadBusy = true; try { const status = await getStatus(); - if (status.status === 'verifying' || status.status === 'published') await followVerification(status); - else { phase = 'paused'; uploadError = status.error ?? ''; } - } catch (cause) { uploadError = (cause as Error).message; phase = 'paused'; } - finally { uploadBusy = false; } - await tick(); document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' }); + if (status.status === 'verifying' || status.status === 'published') + await followVerification(status); + else { + phase = 'paused'; + uploadError = status.error ?? ''; + } + } catch (cause) { + uploadError = (cause as Error).message; + phase = 'paused'; + } finally { + uploadBusy = false; + } + await tick(); + document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' }); } - function pauseUpload() { pauseRequested = true; abortUploads(); } - async function sendChunk(path: string, file: File, offset: number, length: number): Promise { + function pauseUpload() { + pauseRequested = true; + abortUploads(); + } + async function sendChunk( + path: string, + file: File, + offset: number, + length: number + ): Promise { const chunk = file.slice(offset, offset + length); const hash = await crypto.subtle.digest('SHA-256', await chunk.arrayBuffer()); if (pauseRequested || disposed || failedTransfer) throw new Error('Upload paused.'); - const sha256 = [...new Uint8Array(hash)].map(value => value.toString(16).padStart(2, '0')).join(''); + const sha256 = [...new Uint8Array(hash)] + .map((value) => value.toString(16).padStart(2, '0')) + .join(''); return new Promise((resolve, reject) => { - const request = new XMLHttpRequest(); activeRequests.set(request, 0); - request.open('PUT', `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}`); + const request = new XMLHttpRequest(); + activeRequests.set(request, 0); + request.open( + 'PUT', + `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}` + ); request.setRequestHeader('Content-Type', 'application/octet-stream'); - request.setRequestHeader('X-Upload-Offset', String(offset)); request.setRequestHeader('X-Upload-Length', String(length)); - request.setRequestHeader('X-Content-SHA256', sha256); request.timeout = 300_000; - request.upload.onprogress = event => { activeRequests.set(request, Math.min(length, event.loaded)); updateSending(); }; + request.setRequestHeader('X-Upload-Offset', String(offset)); + request.setRequestHeader('X-Upload-Length', String(length)); + request.setRequestHeader('X-Content-SHA256', sha256); + request.timeout = 300_000; + request.upload.onprogress = (event) => { + activeRequests.set(request, Math.min(length, event.loaded)); + updateSending(); + }; request.onload = () => { - activeRequests.delete(request); updateSending(); + activeRequests.delete(request); + updateSending(); try { const body = JSON.parse(request.responseText); - if (request.status < 200 || request.status >= 300) throw new Error(body.error ?? `Upload failed (${request.status}).`); - if (body.receivedBytes !== offset + length) throw new Error('The upload position changed. Resume to check the saved progress.'); + if (request.status < 200 || request.status >= 300) + throw new Error(body.error ?? `Upload failed (${request.status}).`); + if (body.receivedBytes !== offset + length) + throw new Error( + 'The upload position changed. Resume to check the saved progress.' + ); resolve(body.receivedBytes); - } catch (cause) { reject(cause); } + } catch (cause) { + reject(cause); + } + }; + request.onerror = () => { + activeRequests.delete(request); + updateSending(); + reject( + new Error( + 'The connection was interrupted. Resume to continue from the last saved chunk.' + ) + ); + }; + request.ontimeout = () => { + activeRequests.delete(request); + updateSending(); + reject( + new Error('This chunk timed out. Resume to check saved progress and retry.') + ); + }; + request.onabort = () => { + activeRequests.delete(request); + updateSending(); + reject(new Error('Upload paused.')); }; - request.onerror = () => { activeRequests.delete(request); updateSending(); reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); }; - request.ontimeout = () => { activeRequests.delete(request); updateSending(); reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); }; - request.onabort = () => { activeRequests.delete(request); updateSending(); reject(new Error('Upload paused.')); }; request.send(chunk); }); } async function startUpload(event: SubmitEvent) { - event.preventDefault(); if (uploadBusy || !folder) return; - if (!globalThis.crypto?.subtle) { uploadError = 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; return; } - uploadBusy = true; pauseRequested = false; failedTransfer = false; uploadError = ''; notice = ''; phase = 'preparing'; + event.preventDefault(); + if (uploadBusy || !folder) return; + if (!globalThis.crypto?.subtle) { + uploadError = + 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; + return; + } + uploadBusy = true; + pauseRequested = false; + failedTransfer = false; + uploadError = ''; + notice = ''; + phase = 'preparing'; try { if (!uploadId) { - const result = await responseBody(await fetch('/api/game-files', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, files: folder.files.map(({ path, file }) => ({ path, size: file.size, lastModified: file.lastModified })) }) })); + const result = await responseBody( + await fetch('/api/game-files', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + game, + files: folder.files.map(({ path, file }) => ({ + path, + size: file.size, + lastModified: file.lastModified + })) + }) + }) + ); uploadId = result.version.id; } const status = await getStatus(); - if (status.status === 'verifying' || status.status === 'published') { await followVerification(status); return; } + if (status.status === 'verifying' || status.status === 'published') { + await followVerification(status); + return; + } checkSelection(folder, status); - const localFiles = new Map(folder.files.map(item => [item.path, item.file])); + const localFiles = new Map(folder.files.map((item) => [item.path, item.file])); const chunkSize = Math.min(8_388_608, Math.max(1, status.chunkSize)); phase = 'uploading'; let nextFile = 0; @@ -174,30 +356,71 @@ do { if (pauseRequested || disposed || failure) return; const length = Math.min(chunkSize, remote.size - remote.receivedBytes); - remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length); + remote.receivedBytes = await sendChunk( + remote.path, + file, + remote.receivedBytes, + length + ); acceptedBytes += length; - if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; } + if (remote.receivedBytes === remote.size) { + remote.complete = true; + completedFiles++; + } } while (!remote.complete); - } catch (cause) { failure ??= cause; failedTransfer = true; abortUploads(); } + } catch (cause) { + failure ??= cause; + failedTransfer = true; + abortUploads(); + } } }; await Promise.all(Array.from({ length: Math.min(3, status.files.length) }, transfer)); if (failure) throw failure; - if (pauseRequested || disposed) { phase = 'paused'; return; } - phase = 'verifying'; currentPath = ''; - const verification = await responseBody(await fetch(`/api/game-files/${uploadId}/upload/complete`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })) as UploadStatus; - applyStatus(verification); await followVerification(verification); + if (pauseRequested || disposed) { + phase = 'paused'; + return; + } + phase = 'verifying'; + currentPath = ''; + const verification = (await responseBody( + await fetch(`/api/game-files/${uploadId}/upload/complete`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + }) + )) as UploadStatus; + applyStatus(verification); + await followVerification(verification); } catch (cause) { - if (!disposed) { uploadError = pauseRequested ? '' : (cause as Error).message; phase = 'paused'; } - } finally { uploadBusy = false; sendingBytes = 0; if (!disposed) void load(); } + if (!disposed) { + uploadError = pauseRequested ? '' : (cause as Error).message; + phase = 'paused'; + } + } finally { + uploadBusy = false; + sendingBytes = 0; + if (!disposed) void load(); + } } - beforeNavigate(navigation => { + beforeNavigate((navigation) => { if (!transferring) return; - if (navigation.type === 'leave' || !window.confirm('This upload is still transferring files. Leave this page? You can resume later by selecting the original unchanged folder.')) navigation.cancel(); + if ( + navigation.type === 'leave' || + !window.confirm( + 'This upload is still transferring files. Leave this page? You can resume later by selecting the original unchanged folder.' + ) + ) + navigation.cancel(); }); onMount(() => { const stop = pollVisible(load, 5000); - return () => { disposed = true; pauseRequested = true; abortUploads(); stop(); }; + return () => { + disposed = true; + pauseRequested = true; + abortUploads(); + stop(); + }; }); @@ -207,48 +430,166 @@ {#if $page.url.searchParams.has('returnTo')}{/if} {#if error}{/if} {#if notice}

{notice}

{/if} - {#if !canManage}

Administrators upload game files. Active hosts prepare verified copies automatically.

{/if} + {#if !canManage}

+ Administrators upload game files. Active hosts prepare verified copies automatically. +

{/if} {#if canManage && showUpload}
-

{phase === 'published' ? `${gameTitle(game, true)} files uploaded` : uploadId ? `Continue ${gameTitle(game, true)} upload` : `Upload ${gameTitle(game, true)} game folder`}

-

Select your complete {gameName(game)} game folder, including its installed DLC. Once verified, this upload becomes the latest files and hosts download it automatically. Running servers keep using their existing files.

+
+

+ {phase === 'published' + ? `${gameTitle(game, true)} files uploaded` + : uploadId + ? `Continue ${gameTitle(game, true)} upload` + : `Upload ${gameTitle(game, true)} game folder`} +

+ +
+

+ Select your complete {gameName(game)} game folder, including its installed DLC. Once verified, + this upload becomes the latest files and hosts download it automatically. Running servers + keep using their existing files. +

{#if uploadError}{/if}
{#if phase !== 'verifying' && phase !== 'published'} - -

{uploadId ? 'To resume, file names, sizes, and modification dates must match the original upload.' : 'Choose the game directory itself. Its top-level folder name is removed once; all files and subfolders inside it keep their paths.'}

+ +

+ {uploadId + ? 'To resume, file names, sizes, and modification dates must match the original upload.' + : 'Choose the game directory itself. Its top-level folder name is removed once; all files and subfolders inside it keep their paths.'} +

{/if} - {#if folder}
{folder.name}{folder.files.length.toLocaleString()} files · {formatGameBytes(folder.size)}
Preview folder paths
    {#each folder.files.slice(0, 8) as file}
  • {file.path}
    • {/each}
{#if folder.files.length > 8}

And {(folder.files.length - 8).toLocaleString()} more files.

{/if}
{/if} + {#if folder}
+ {folder.name}{folder.files.length.toLocaleString()} files · {formatGameBytes( + folder.size + )} +
+ Preview folder paths +
    + {#each folder.files.slice(0, 8) as file}
  • {file.path}
    • {/each} +
+ {#if folder.files.length > 8}

+ And {(folder.files.length - 8).toLocaleString()} more files. +

{/if} +
+
{/if} {#if phase !== 'idle'}
-
{phase === 'preparing' ? 'Preparing upload…' : phase === 'uploading' ? 'Uploading game files' : phase === 'verifying' ? 'Verifying files…' : phase === 'published' ? 'Ready · host downloads are automatic' : 'Upload paused'}{completedFiles.toLocaleString()} / {totalFiles.toLocaleString()} files
- -

{formatGameBytes(progressBytes)} / {formatGameBytes(totalBytes)} saved or transferring

- {#if currentPath}

{currentPath}

{/if} - {#if phase === 'verifying'}

The controller checks every file before making this upload available to hosts. Verification continues if you leave this page.

{:else if phase === 'paused'}

Saved chunks are kept on the controller. Resume this upload to check the saved progress and retry.

{/if} -
{/if} +
+ {phase === 'preparing' + ? 'Preparing upload…' + : phase === 'uploading' + ? 'Uploading game files' + : phase === 'verifying' + ? 'Verifying files…' + : phase === 'published' + ? 'Ready · host downloads are automatic' + : 'Upload paused'}{completedFiles.toLocaleString()} / {totalFiles.toLocaleString()} files +
+ +

+ {formatGameBytes(progressBytes)} / {formatGameBytes(totalBytes)} saved or + transferring +

+ {#if currentPath}

{currentPath}

{/if} + {#if phase === 'verifying'}

+ The controller checks every file before making this upload available + to hosts. Verification continues if you leave this page. +

{:else if phase === 'paused'}

+ Saved chunks are kept on the controller. Resume this upload to check + the saved progress and retry. +

{/if} +
{/if}
- {#if transferring} - {:else if phase !== 'verifying' && phase !== 'published'}{/if} - {#if phase === 'idle' || phase === 'uploading'}Keep this page open while files transfer. Interrupted uploads can be resumed below.{/if} + {#if transferring} + {:else if phase !== 'verifying' && phase !== 'published'}{/if} + {#if phase === 'idle' || phase === 'uploading'}Keep this page open while files transfer. Interrupted uploads can be + resumed below.{/if}
{/if} {#if !loaded}

Loading game files…

{/if} {#each libraries as library (library.game)} - {@const latest = library.versions.find(item => item.id === library.latestVersionId)} - {@const unfinished = library.versions.filter(item => item.state !== 'ready')} - {@const history = library.versions.filter(item => item.state === 'ready' && item.id !== library.latestVersionId)} + {@const latest = library.versions.find((item) => item.id === library.latestVersionId)} + {@const unfinished = library.versions.filter((item) => item.state !== 'ready')} + {@const history = library.versions.filter( + (item) => item.state === 'ready' && item.id !== library.latestVersionId + )}
-

{gameName(library.game)}

{#if latest}

{formatGameBytes(latest.sizeBytes)} · {latest.fileCount.toLocaleString()} files

{/if}
- {#if canManage}{/if} +
+

{gameName(library.game)}

+ {#if latest}

+ {formatGameBytes(latest.sizeBytes)} · {latest.fileCount.toLocaleString()} + files +

{/if} +
+ {#if canManage}{/if}
{#if latest} -
Latest verified upload cache.status === 'failed') ? 'danger' : latest.caches.length && latest.caches.every(cache => cache.status === 'ready') ? 'online' : 'neutral'}>{latest.caches.length ? `${latest.caches.filter(cache => cache.status === 'ready').length} / ${latest.caches.length} host copies ready` : 'No hosts'}
+
+ Latest verified upload cache.status === 'failed') + ? 'danger' + : latest.caches.length && + latest.caches.every((cache) => cache.status === 'ready') + ? 'online' + : 'neutral'} + >{latest.caches.length + ? `${latest.caches.filter((cache) => cache.status === 'ready').length} / ${latest.caches.length} host copies ready` + : 'No hosts'} +
{:else} -

Upload the complete {gameName(library.game)} game folder once. Hosts will download and verify it automatically as part of setup.

+

+ Upload the complete {gameName(library.game)} game folder once. Hosts will download + and verify it automatically as part of setup. +

{/if} {#if unfinished.length}
@@ -256,64 +597,287 @@ {#each unfinished as item (item.id)} {@const browserUpload = !item.sourceHostId}
-

Started · {formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files

{item.state === 'failed' ? 'Failed' : browserUpload ? item.id === uploadId && phase === 'verifying' ? 'Verifying' : 'Uploading' : 'Importing'}
+
+

+ Started · {formatGameBytes( + item.sizeBytes + )} · {item.fileCount.toLocaleString()} files +

+ {item.state === 'failed' + ? 'Failed' + : browserUpload + ? item.id === uploadId && phase === 'verifying' + ? 'Verifying' + : 'Uploading' + : 'Importing'} +
{#if item.error}

{item.error}

{/if} {#if browserUpload} -

Resume with the original unchanged folder. The latest files stay available while this upload completes.

- {#if canManage}
{/if} - {:else if item.state === 'importing'}

The existing host import continues independently. It is reconciled when the host reconnects.

{/if} +

+ Resume with the original unchanged folder. The latest files stay + available while this upload completes. +

+ {#if canManage}
+ +
{/if} + {:else if item.state === 'importing'}

+ The existing host import continues independently. It is + reconciled when the host reconnects. +

{/if}
{/each}
{/if} - {#if latest || history.length}
Details
- {#if latest}

Uploaded

{@render uploadDetails(latest)}{@render downloadControls(latest)}{/if} - {#if history.length} -
- Previous uploads ({history.length}) -

Old host copies are removed once no instances use them. Controller files are removed after all host copies are gone; upload history is kept.

- {#each history as item (item.id)} -
-

Uploaded

- {#if item.filesRemoved}

Files removed from controller · upload history retained

{:else if item.filesRemoving}

Removing unused controller files · upload history retained

{/if} -

{formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files

- {@render uploadDetails(item)} -
Host copies{@render downloadControls(item)}
-
- {/each} -
- {/if} -
{/if} + {#if latest || history.length}
+ Details +
+ {#if latest}

+ Uploaded +

+ {@render uploadDetails(latest)}{@render downloadControls(latest)}{/if} + {#if history.length} +
+ Previous uploads ({history.length}) +

+ Old host copies are removed once no instances use them. + Controller files are removed after all host copies are gone; + upload history is kept. +

+ {#each history as item (item.id)} +
+

+ Uploaded +

+ {#if item.filesRemoved}

+ Files removed from controller · upload history + retained +

{:else if item.filesRemoving}

+ Removing unused controller files · upload history + retained +

{/if} +

+ {formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} + files +

+ {@render uploadDetails(item)} +
+ Host copies{@render downloadControls(item)} +
+
+ {/each} +
+ {/if} +
+
{/if}
{/each}
{#snippet uploadDetails(item: OverviewLibrary['versions'][number])} -
Upload details
Upload ID
{item.id}
Manifest SHA256
{item.manifestSha256 ?? 'Available after verification'}
Uploaded from
{!item.sourceHostId ? 'Game folder in browser' : `${hostName(item.sourceHostId)} / ${item.sourceVolume}`}
View complete file manifest
+
+ Upload details +
+
Upload ID
+
{item.id}
+
Manifest SHA256
+
{item.manifestSha256 ?? 'Available after verification'}
+
Uploaded from
+
+ {!item.sourceHostId + ? 'Game folder in browser' + : `${hostName(item.sourceHostId)} / ${item.sourceVolume}`} +
+
+ View complete file manifest +
{/snippet} {#snippet downloadControls(item: OverviewLibrary['versions'][number])} - {#if canManage && !item.filesRemoved && !item.filesRemoving}
Advanced: download or verify these files

Retry a download or prepare a previous upload on a host. The latest files are downloaded automatically.

{/if} + {#if canManage && !item.filesRemoved && !item.filesRemoving}
+ Advanced: download or verify these files +

+ Retry a download or prepare a previous upload on a host. The latest files are + downloaded automatically. +

+
+ +
+
{/if} {/snippet} diff --git a/src/routes/hosts/+page.server.ts b/src/routes/hosts/+page.server.ts index 8813891..c1d9754 100644 --- a/src/routes/hosts/+page.server.ts +++ b/src/routes/hosts/+page.server.ts @@ -1,3 +1,5 @@ import { hostOverview } from '$lib/server/hosts/overview.js'; -export function load() { return { hosts: hostOverview(), serverNow: Date.now() }; } +export function load() { + return { hosts: hostOverview(), serverNow: Date.now() }; +} diff --git a/src/routes/hosts/+page.svelte b/src/routes/hosts/+page.svelte index 31dc1ff..0875364 100644 --- a/src/routes/hosts/+page.svelte +++ b/src/routes/hosts/+page.svelte @@ -13,64 +13,188 @@ let order = $state(untrack(() => stableHostOrder([], data.hosts, data.serverNow))); let clockOffset = untrack(() => data.serverNow - Date.now()); let now = $state(untrack(() => data.serverNow)); - let error = $state(''); let addOpen = $state(false); - let search = $state(''); let status = $state(''); + let error = $state(''); + let addOpen = $state(false); + let search = $state(''); + let status = $state(''); let loadSequence = 0; const canManage = $derived($page.data.user?.role === 'admin'); - $effect(() => { if (canManage && $page.url.searchParams.get('action') === 'add') addOpen = true; }); - const attention = $derived(hosts.filter(host => ['attention', 'offline'].includes(hostListStatus(host, now).key)).length); - const ready = $derived(hosts.filter(host => hostListStatus(host, now).key === 'ready').length); + $effect(() => { + if (canManage && $page.url.searchParams.get('action') === 'add') addOpen = true; + }); + const attention = $derived( + hosts.filter((host) => ['attention', 'offline'].includes(hostListStatus(host, now).key)) + .length + ); + const ready = $derived( + hosts.filter((host) => hostListStatus(host, now).key === 'ready').length + ); const filtersVisible = $derived(hosts.length > 5 || !!search || !!status); const visible = $derived.by(() => { - const byId = new Map(hosts.map(host => [host.id, host])); - return order.flatMap(id => { + const byId = new Map(hosts.map((host) => [host.id, host])); + return order.flatMap((id) => { const host = byId.get(id); - if (!host || !`${host.name} ${host.hostname}`.toLowerCase().includes(search.trim().toLowerCase())) return []; + if ( + !host || + !`${host.name} ${host.hostname}`.toLowerCase().includes(search.trim().toLowerCase()) + ) + return []; const state = hostListStatus(host, now).key; - if (status && (status === 'attention' ? !['attention', 'offline'].includes(state) : state !== status)) return []; + if ( + status && + (status === 'attention' + ? !['attention', 'offline'].includes(state) + : state !== status) + ) + return []; return [host]; }); }); async function load(signal?: AbortSignal) { const sequence = ++loadSequence; try { - const response = await fetch('/api/hosts', { signal: signal ?? AbortSignal.timeout(15000) }); - const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); - if (sequence === loadSequence) { hosts = body.hosts; clockOffset = body.serverNow - Date.now(); now = body.serverNow; order = stableHostOrder(order, hosts, now); error = ''; } - } catch (cause) { if (!signal?.aborted && sequence === loadSequence) error = (cause as Error).message; return false; } + const response = await fetch('/api/hosts', { + signal: signal ?? AbortSignal.timeout(15000) + }); + const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); + if (sequence === loadSequence) { + hosts = body.hosts; + clockOffset = body.serverNow - Date.now(); + now = body.serverNow; + order = stableHostOrder(order, hosts, now); + error = ''; + } + } catch (cause) { + if (!signal?.aborted && sequence === loadSequence) error = (cause as Error).message; + return false; + } } onMount(() => { const stop = pollVisible(load, 30000, false); - const timer = setInterval(() => now = Date.now() + clockOffset, 15000); - return () => { stop(); clearInterval(timer); }; + const timer = setInterval(() => (now = Date.now() + clockOffset), 15000); + return () => { + stop(); + clearInterval(timer); + }; }); Hosts · Electric Cherry
- {#snippet actions()}{#if canManage}{/if}{/snippet} + {#snippet actions()}{#if canManage}{/if}{/snippet} - {#if error}{/if} + {#if error}{/if} {#if hosts.length} -
{countLabel(hosts.length, 'host')} · {ready === hosts.length ? 'All ready' : `${ready} ready`}{#if attention}{/if}
+
+ {countLabel(hosts.length, 'host')} · {ready === hosts.length + ? 'All ready' + : `${ready} ready`}{#if attention}{/if} +
{#if filtersVisible}
- - - {#if search || status}{/if} -
{/if} - {#if visible.length}{:else}

No hosts match these filters.

{/if} - {:else}

No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}

{/if} + + + {#if search || status}{/if} + {/if} + {#if visible.length}{:else}

No hosts match these filters.

{/if} + {:else}

+ No hosts connected yet.{canManage ? ' Add a host to begin.' : ''} +

{/if}
{#if canManage}{/if} diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index 856f2ef..01ade5c 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -12,7 +12,15 @@ import type { Host, CapacityCalibration } from '$lib/hosts.js'; import { hostCalibrationHint } from '$ui/host-overview.js'; import { countLabel } from '$ui/plural.js'; - import { hostAgentLaneLabel, hostAgentLaneState, hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js'; + import { + hostAgentLaneLabel, + hostAgentLaneState, + hostAgentUpdateLabel, + hostAgentUpdateMessage, + hostAgentUpdatesEnabled, + hostDeletionMessage, + hostReadinessLabel + } from '$lib/hosts.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; @@ -25,13 +33,31 @@ let host = $state(null); let instances = $state([]); let calibration = $state(null); - let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state(''); + let loading = $state(true); + let busy = $state(false); + let error = $state(''); + let notice = $state(''); let deleteOpen = $state(false); let drainOpen = $state(false); - let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1); - let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null); - const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'servers', label: 'Servers' }, { id: 'setup', label: 'Setup' }, { id: 'settings', label: 'Settings' }]; - const tab = $derived(tabs.some(tab => tab.id === $page.url.searchParams.get('tab')) ? $page.url.searchParams.get('tab')! : 'overview'); + let name = $state(''); + let portMin = $state(28960); + let portMax = $state(28980); + let bindAddress = $state('0.0.0.0'); + let publicAddress = $state(''); + let minimumFreeGB = $state(1); + let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); + let enrollment = $state<{ command: string; expiresAt: string } | null>(null); + const tabs = [ + { id: 'overview', label: 'Overview' }, + { id: 'servers', label: 'Servers' }, + { id: 'setup', label: 'Setup' }, + { id: 'settings', label: 'Settings' } + ]; + const tab = $derived( + tabs.some((tab) => tab.id === $page.url.searchParams.get('tab')) + ? $page.url.searchParams.get('tab')! + : 'overview' + ); function changeTab(value: string) { const url = new URL($page.url); url.searchParams.set('tab', value); @@ -41,127 +67,702 @@ const endpoint = $derived(`/api/hosts/${$page.params.id}`); const canManage = $derived($page.data.user?.role === 'admin'); const cleaning = $derived(!!host?.decommission); - function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; } + function bytes(value: unknown) { + return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; + } async function load(initialize = false, signal?: AbortSignal) { - const target = endpoint; const sequence = ++requestSequence; + const target = endpoint; + const sequence = ++requestSequence; try { - const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; instances = body.instances ?? []; syncServerTime(body.serverNow); calibration = body.calibration ?? null; error = ''; - if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; } - } catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; } + const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); + const body = await response.json(); + if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; + if (response.status === 404 && host?.decommission) { + await goto('/hosts'); + return; + } + if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); + host = body.host; + instances = body.instances ?? []; + syncServerTime(body.serverNow); + calibration = body.calibration ?? null; + error = ''; + if (initialize && host) { + name = host.name; + portMin = host.portMin; + portMax = host.portMax; + bindAddress = host.bindAddress; + publicAddress = host.publicAddress; + minimumFreeGB = host.minimumFreeBytes / 1073741824; + runtimeImage = body.defaultRuntimeImage ?? runtimeImage; + } + } catch (cause) { + if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; + return false; + } finally { + if (sequence === requestSequence) loading = false; + } } async function action(path: string, method: string, body?: unknown, message?: string) { - busy = true; error = ''; notice = ''; - try { const response = await fetch(endpoint + path, { method, headers: { 'Content-Type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); const result = await response.json(); if (!response.ok) throw new Error(result.error ?? 'Host request failed.'); if (result.enrollment) enrollment = result.enrollment; notice = message ?? ''; await load(); } - catch (cause) { error = (cause as Error).message; } finally { busy = false; } + busy = true; + error = ''; + notice = ''; + try { + const response = await fetch(endpoint + path, { + method, + headers: { 'Content-Type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error ?? 'Host request failed.'); + if (result.enrollment) enrollment = result.enrollment; + notice = message ?? ''; + await load(); + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } + } + async function save(event: SubmitEvent) { + event.preventDefault(); + await action( + '', + 'PATCH', + { + name, + portMin, + portMax, + bindAddress, + publicAddress, + minimumFreeBytes: Math.round(minimumFreeGB * 1073741824) + }, + 'Host settings saved.' + ); } - async function save(event: SubmitEvent) { event.preventDefault(); await action('', 'PATCH', { name, portMin, portMax, bindAddress, publicAddress, minimumFreeBytes: Math.round(minimumFreeGB * 1073741824) }, 'Host settings saved.'); } async function mode(value: 'active' | 'draining') { - await action('', 'PATCH', { mode: value }, value === 'draining' - ? 'Drain started. Empty instances migrate as destination capacity becomes available.' - : 'Automatic setup started. This host will enable after the runtime and both game libraries are verified.'); + await action( + '', + 'PATCH', + { mode: value }, + value === 'draining' + ? 'Drain started. Empty instances migrate as destination capacity becomes available.' + : 'Automatic setup started. This host will enable after the runtime and both game libraries are verified.' + ); if (!error) drainOpen = false; } $effect(() => { void endpoint; return untrack(() => { - host = null; instances = []; calibration = null; loading = true; error = ''; notice = ''; enrollment = null; + host = null; + instances = []; + calibration = null; + loading = true; + error = ''; + notice = ''; + enrollment = null; let initialize = true; - return pollVisible(async signal => { const result = await load(initialize, signal); if (result !== false) initialize = false; return result; }, 15000); + return pollVisible(async (signal) => { + const result = await load(initialize, signal); + if (result !== false) initialize = false; + return result; + }, 15000); }); }); {host?.name ?? 'Host'} · Electric Cherry -
- {#snippet crumbs()}{/snippet} - {#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}
More actions
{/if}{/snippet} - - {#if error}{/if}{#if notice}

{notice}

{/if} +
+ + {#snippet crumbs()}{/snippet} + {#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}
+ More actions +
+ +
+
{/if}{/snippet} + + {#if error}{/if}{#if notice}

+ {notice} +

{/if} {#if loading}

Loading host…

{:else if host} - {#if host.decommission}

Host cleanup

{hostDeletionMessage(host)}

{#if host.decommission.error}

{host.decommission.error}

{/if}

Requested . This page closes only after the cleanup worker confirms removal.

{#if host.decommission.stage === 'cleaned'}

The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.

{/if}
{/if} -

{host.connectionState} · {hostReadinessLabel(host.readiness)}

Last heartbeat:
- {#if host.readinessReasons.length}
    {#each host.readinessReasons as reason}
  • {reason}
    • {/each}
{:else}

Connected and ready. Each deployment also checks its latest game files, runtime, and ports.

{/if} - {#if host.readiness !== 'ready' && tab !== 'setup'}{/if} + {#if host.decommission}
+

Host cleanup

+

{hostDeletionMessage(host)}

+ {#if host.decommission.error}

{host.decommission.error}

{/if} +

+ Requested . This page + closes only after the cleanup worker confirms removal. +

+ {#if host.decommission.stage === 'cleaned'}

+ The agent is expected to go offline during uninstall. If final confirmation + never arrives, inspect the machine; loss of contact alone does not prove + completion. +

{/if} +
{/if} +
+
+

{host.connectionState} · {hostReadinessLabel(host.readiness)}

+ Last heartbeat: +
+ {#if host.readinessReasons.length}
    + {#each host.readinessReasons as reason}
  • {reason}
    • {/each} +
{:else}

+ Connected and ready. Each deployment also checks its latest game files, runtime, + and ports. +

{/if} + {#if host.readiness !== 'ready' && tab !== 'setup'}{/if}
- -
- {#if tab === 'overview'} -

Agent workers

- {#if host.inventory.agentLanes?.length}
{#each host.inventory.agentLanes as lane (lane.name)} -
{hostAgentLaneLabel(lane.name)}
{hostAgentLaneState(lane)}{#if !host.connected} (last reported){/if}
Last successful poll:
- {/each}
{:else}

This agent has not reported worker status yet.

{/if} -
-

Inventory

-
Hostname
{host.hostname || 'Pending'}
Agent
{host.inventory.agentVersion ?? 'Unknown'}
Platform
{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}
CPU
{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)
CPU usage
{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}
Memory available / total
{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}
Game storage available / total
{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}
Docker
{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}
Current operation
{host.inventory.currentOperation ?? 'None reported'}
-
{#if calibration}

Learned server capacity

-

Allowances update from observed container usage. Hosts reserve 5% CPU and 500 MB of memory. Startup and running usage are tracked separately.

-
{#each ['startup', 'running'] as phase}{@const allowance = calibration[phase as keyof CapacityCalibration]} -
{phase === 'startup' ? 'Starting a server' : 'Running a server'}
-
{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({countLabel(allowance.samples, 'sample')})
- {/each}
- {#if host.inventory.instanceResourcesError}

{host.inventory.instanceResourcesError}

{/if} - {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}

{hostCalibrationHint(host.inventory.agentVersion)} Measurements arrive with agent heartbeats and are combined into one peak sample per container per minute. The first five minutes after a container starts count toward startup; later measurements count toward running.

{/if} - {#key host.id}{/key} -
{/if} - {:else if tab === 'servers'} -
-

Running servers · {instances.length}

-

Current instances on this host, including any that are starting or unavailable.

- {#if !host.connected && instances.length}

The host is offline. Instance details reflect the latest reports.

{/if} - {#if instances.length} -
- {#each instances as instance (instance.name)} - - {/each} + +
+ {#if tab === 'overview'} +
+

Agent workers

+ {#if host.inventory.agentLanes?.length}
+ {#each host.inventory.agentLanes as lane (lane.name)} +
{hostAgentLaneLabel(lane.name)}
+
+ {hostAgentLaneState(lane)}{#if !host.connected} + (last reported){/if}
Last successful poll: +
+ {/each} +
{:else}

This agent has not reported worker status yet.

{/if} +
+
+

Inventory

+
+
Hostname
+
{host.hostname || 'Pending'}
+
Agent
+
{host.inventory.agentVersion ?? 'Unknown'}
+
Platform
+
+ {host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? + 'Unknown'} +
+
CPU
+
+ {host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores) +
+
CPU usage
+
+ {host.inventory.cpuUsagePercent === undefined + ? 'Not reported' + : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} + (last reported){/if} +
+
Memory available / total
+
+ {bytes(host.inventory.memoryAvailable)} / {bytes( + host.inventory.memoryTotal + )} +
+
Game storage available / total
+
+ {bytes(host.inventory.storageAvailable)} / {bytes( + host.inventory.storageTotal + )} +
+
Docker
+
+ {host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory + .dockerRunning + ? 'Running' + : 'Unavailable'} +
+
Current operation
+
{host.inventory.currentOperation ?? 'None reported'}
+
+
+ {#if calibration}
+

Learned server capacity

+

+ Allowances update from observed container usage. Hosts reserve 5% CPU + and 500 MB of memory. Startup and running usage are tracked separately. +

+
+ {#each ['startup', 'running'] as phase}{@const allowance = + calibration[phase as keyof CapacityCalibration]} +
+ {phase === 'startup' ? 'Starting a server' : 'Running a server'} +
+
+ {allowance.cpuCores.toFixed(2)} CPU cores · {bytes( + allowance.memoryBytes + )} · {allowance.source === 'host' + ? 'Learned on this host' + : allowance.source === 'fleet' + ? 'Learned from the fleet' + : 'Calibrating; temporary allowance'} ({countLabel( + allowance.samples, + 'sample' + )}) +
+ {/each} +
+ {#if host.inventory.instanceResourcesError}

+ {host.inventory.instanceResourcesError} +

{/if} + {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}

+ {hostCalibrationHint(host.inventory.agentVersion)} Measurements arrive + with agent heartbeats and are combined into one peak sample per container + per minute. The first five minutes after a container starts count toward + startup; later measurements count toward running. +

{/if} + {#key host.id}{/key} +
{/if} + {:else if tab === 'servers'} +
+

Running servers · {instances.length}

+

+ Current instances on this host, including any that are starting or + unavailable. +

+ {#if !host.connected && instances.length}

+ The host is offline. Instance details reflect the latest reports. +

{/if} + {#if instances.length} +
+ {#each instances as instance (instance.name)} + + {/each} +
+ {:else}

No instances are currently deployed on this host.

{/if} +
+ {:else if tab === 'setup'} + {#if !cleaning}
+

Automatic host setup

+ +
{/if} +
+

+ Automatic agent updates · {host.enrolled + ? hostAgentUpdateLabel(host.inventory) + : 'Awaiting installation'} +

+
+
Running version
+
+ {host.inventory.agentVersion ?? + host.inventory.agentUpdate?.currentVersion ?? + 'Not reported'} +
+
Target version
+
{host.inventory.agentUpdate?.targetVersion ?? 'None reported'}
+
Last update check
+
+ +
+
+ {#if !host.connected && host.enrolled}

+ This is the last reported update status. The host is offline. +

{/if} + {#if host.inventory.agentUpdate?.error}

+ {host.inventory.agentUpdate.error} +

{/if} + {#if host.enrolled}

{hostAgentUpdateMessage(host.inventory)}

{:else}

+ The installer enables automatic updates when this host first enrolls. +

{/if} + {#if host.enrolled && !hostAgentUpdatesEnabled(host.inventory)}

+ {canManage + ? 'Generate an installer under Agent access in Settings, run it on this host, and wait for a fresh heartbeat. The installer preserves the existing host identity and game data.' + : 'An administrator can generate an installer under Agent access and run it on this host.'} +

{:else if hostAgentUpdatesEnabled(host.inventory)}

+ The host checks this controller about every five minutes. Updates + replace agent code between operations; running game containers continue. +

{/if} +
+
+

Cached game files and runtimes

+ {#if host.inventory.libraries?.length}
    + {#each host.inventory.libraries as library}
  • + {library.id}: {library.state}{library.progress !== undefined + ? ` (${library.progress}%)` + : ''} +
    • {/each} +
{:else}

No cached game libraries reported yet.

{/if} +

+ Cached images: {host.inventory.runtimeImages?.join(', ') || + 'None reported yet'} +

+ {#if canManage}
+ Advanced: custom runtime or manual retry +

+ The configured default runtime is prepared automatically. Use this + to prepare another image or retry a pull now. +

+
{ + event.preventDefault(); + void action( + '/operations', + 'POST', + { kind: 'runtime.pull', image: runtimeImage }, + 'Runtime download queued.' + ); + }} + > +
+ +
+
+
{/if} +
+
+
- {:else}

No instances are currently deployed on this host.

{/if} -
- {:else if tab === 'setup'} - {#if !cleaning}

Automatic host setup

{/if} -

Automatic agent updates · {host.enrolled ? hostAgentUpdateLabel(host.inventory) : 'Awaiting installation'}

-
Running version
{host.inventory.agentVersion ?? host.inventory.agentUpdate?.currentVersion ?? 'Not reported'}
Target version
{host.inventory.agentUpdate?.targetVersion ?? 'None reported'}
Last update check
- {#if !host.connected && host.enrolled}

This is the last reported update status. The host is offline.

{/if} - {#if host.inventory.agentUpdate?.error}

{host.inventory.agentUpdate.error}

{/if} - {#if host.enrolled}

{hostAgentUpdateMessage(host.inventory)}

{:else}

The installer enables automatic updates when this host first enrolls.

{/if} - {#if host.enrolled && !hostAgentUpdatesEnabled(host.inventory)}

{canManage ? 'Generate an installer under Agent access in Settings, run it on this host, and wait for a fresh heartbeat. The installer preserves the existing host identity and game data.' : 'An administrator can generate an installer under Agent access and run it on this host.'}

{:else if hostAgentUpdatesEnabled(host.inventory)}

The host checks this controller about every five minutes. Updates replace agent code between operations; running game containers continue.

{/if} -
-

Cached game files and runtimes

{#if host.inventory.libraries?.length}
    {#each host.inventory.libraries as library}
  • {library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}
    • {/each}
{:else}

No cached game libraries reported yet.

{/if} -

Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported yet'}

- {#if canManage}
Advanced: custom runtime or manual retry

The configured default runtime is prepared automatically. Use this to prepare another image or retry a pull now.

{ event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}>
{/if} -
-
- {:else if tab === 'settings'} -

Host availability

- {#if canManage && !cleaning}
{/if} -

Draining blocks new deployments and migrates empty instances to other hosts with capacity. The host disables when migration is complete and it is empty. Enabling refreshes the runtime and verifies BO1 and BO2 game files before returning the host to the capacity pool.

-

Host configuration

{#if canManage}
-
-
{:else}
Friendly name
{host.name}
Game ports
{host.portMin}–{host.portMax}
Local bind address
{host.bindAddress}
Public / NAT address
{host.publicAddress || 'Not configured'}
Minimum free storage
{bytes(host.minimumFreeBytes)}

Host configuration is managed by administrators.

{/if}

Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.

Agent access

Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.

{#if canManage}
- {#if enrollment}

Expires {$timeFormat.dateTime(enrollment.expiresAt)}

{enrollment.command}
{/if}{:else}

Administrators manage enrollment and agent credentials.

{/if} -
- {/if} + {:else if tab === 'settings'} +
+

Host availability

+ {#if canManage && !cleaning}
+ +
{/if} +

+ Draining blocks new deployments and migrates empty instances to other hosts + with capacity. The host disables when migration is complete and it is empty. + Enabling refreshes the runtime and verifies BO1 and BO2 game files before + returning the host to the capacity pool. +

+
+
+

Host configuration

+ {#if canManage}
+
+ +
+ +
+ +
+
{:else}
+
Friendly name
+
{host.name}
+
Game ports
+
{host.portMin}–{host.portMax}
+
Local bind address
+
{host.bindAddress}
+
Public / NAT address
+
{host.publicAddress || 'Not configured'}
+
Minimum free storage
+
{bytes(host.minimumFreeBytes)}
+
+

Host configuration is managed by administrators.

{/if} +

+ Public addressing is separate from local port reservations. Configure your + firewall and NAT for the game-port range. +

+
+
+

Agent access

+

+ Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment + links expire after 15 minutes and work once. +

+ {#if canManage}
+ +
+ {#if enrollment}

Expires {$timeFormat.dateTime(enrollment.expiresAt)}

+ 
{enrollment.command}
{/if}{:else}

+ Administrators manage enrollment and agent credentials. +

{/if} +
+ {/if}
{/if}
- { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} /> + { + if (result.deleted) await goto('/hosts'); + else { + enrollment = null; + await load(); + } + }} +/> -

New deployments will stop on this host. Each instance will wait until its lobby is empty and another host has capacity, then be destroyed here and rebuilt there with the same server identity and map. Its address or port may change.

-

If capacity is unavailable, instances stay here until space becomes available. After all migrations finish, this host will be disabled.

+

+ New deployments will stop on this host. Each instance will wait until its lobby is empty and + another host has capacity, then be destroyed here and rebuilt there with the same server + identity and map. Its address or port may change. +

+

+ If capacity is unavailable, instances stay here until space becomes available. After all + migrations finish, this host will be disabled. +

{#if error}{/if} - {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet} diff --git a/src/routes/hosts/operations/+page.server.ts b/src/routes/hosts/operations/+page.server.ts index 130f215..c5ef553 100644 --- a/src/routes/hosts/operations/+page.server.ts +++ b/src/routes/hosts/operations/+page.server.ts @@ -8,15 +8,22 @@ export const load: PageServerLoad = ({ url }) => { const requestedFilter = url.searchParams.get('filter'); const filter = isHostOperationFilter(requestedFilter) ? requestedFilter : 'important'; const scope = hostOperationScope(url.searchParams); - if (url.searchParams.has('operation') && !scope.operation) error(400, 'Invalid host operation.'); + if (url.searchParams.has('operation') && !scope.operation) + error(400, 'Invalid host operation.'); const history = listHostOperationHistory(Number(requestedPage ?? 1), filter, scope); - if ((requestedPage !== null && requestedPage !== String(history.page)) - || (requestedFilter !== null && requestedFilter !== filter)) { + if ( + (requestedPage !== null && requestedPage !== String(history.page)) || + (requestedFilter !== null && requestedFilter !== filter) + ) { const params = new URLSearchParams(url.searchParams); params.delete('page'); if (requestedFilter !== null) params.set('filter', filter); if (history.page !== 1) params.set('page', String(history.page)); redirect(307, `/hosts/operations${params.size ? `?${params}` : ''}`); } - return { history, scope, hostName: scope.host ? getHost(scope.host)?.name ?? scope.host : null }; + return { + history, + scope, + hostName: scope.host ? (getHost(scope.host)?.name ?? scope.host) : null + }; }; diff --git a/src/routes/hosts/operations/+page.svelte b/src/routes/hosts/operations/+page.svelte index 0379413..f22b314 100644 --- a/src/routes/hosts/operations/+page.svelte +++ b/src/routes/hosts/operations/+page.svelte @@ -26,24 +26,51 @@ const changingPage = $derived($navigating?.to?.url.pathname === '/hosts/operations'); const firstOperation = $derived((history.page - 1) * history.pageSize + 1); const numbers = new Intl.NumberFormat('en-US'); - const filters: Array<{ id: HostOperationFilter; label: string; description: string; empty: string }> = [ - { id: 'important', label: 'Actions & errors', description: 'Actions, failures, and checks that need attention.', empty: 'No actions or checks needing attention recorded.' }, - { id: 'routine', label: 'Routine checks', description: 'Monitoring and file reads across all states.', empty: 'No routine checks recorded.' }, - { id: 'all', label: 'All operations', description: 'Every recorded remote operation.', empty: 'No remote operations yet.' } + const filters: Array<{ + id: HostOperationFilter; + label: string; + description: string; + empty: string; + }> = [ + { + id: 'important', + label: 'Actions & errors', + description: 'Actions, failures, and checks that need attention.', + empty: 'No actions or checks needing attention recorded.' + }, + { + id: 'routine', + label: 'Routine checks', + description: 'Monitoring and file reads across all states.', + empty: 'No routine checks recorded.' + }, + { + id: 'all', + label: 'All operations', + description: 'Every recorded remote operation.', + empty: 'No remote operations yet.' + } ]; - const selectedFilter = $derived(filters.find(filter => filter.id === history.filter)!); + const selectedFilter = $derived(filters.find((filter) => filter.id === history.filter)!); const allOperationsHref = $derived.by(() => { const params = new URLSearchParams($page.url.searchParams); - params.delete('operation'); params.delete('page'); + params.delete('operation'); + params.delete('page'); return `/hosts/operations${params.size ? `?${params}` : ''}`; }); function pageHref(pageNumber: number, filter: HostOperationFilter = history.filter) { const params = new URLSearchParams($page.url.searchParams); - params.set('filter', filter); params.delete('page'); + params.set('filter', filter); + params.delete('page'); if (pageNumber !== 1) params.set('page', String(pageNumber)); return `/hosts/operations?${params}`; } - function cancel() { sequence++; request?.abort(); request = null; refreshing = false; } + function cancel() { + sequence++; + request?.abort(); + request = null; + refreshing = false; + } async function refresh(signal?: AbortSignal) { if (refreshing || $navigating) return; @@ -53,74 +80,203 @@ request = controller; refreshing = true; try { - const params = new URLSearchParams({ ...data.scope, page: String(history.page), filter: history.filter }); + const params = new URLSearchParams({ + ...data.scope, + page: String(history.page), + filter: history.filter + }); const response = await fetch(`/api/hosts/operations?${params}`, { - signal: AbortSignal.any([controller.signal, AbortSignal.timeout(15000), ...(signal ? [signal] : [])]) + signal: AbortSignal.any([ + controller.signal, + AbortSignal.timeout(15000), + ...(signal ? [signal] : []) + ]) }); const body = await response.json(); if (!response.ok) throw new Error('Could not refresh operation history.'); - if (current === sequence && data === source && !$navigating) { snapshot = { source, history: body }; refreshError = ''; updatedAt = Date.now(); } - } - catch { - if (current === sequence && data === source && !signal?.aborted && !controller.signal.aborted) { - refreshError = 'Could not refresh operation history. Showing the last successfully loaded page.'; + if (current === sequence && data === source && !$navigating) { + snapshot = { source, history: body }; + refreshError = ''; + updatedAt = Date.now(); + } + } catch { + if ( + current === sequence && + data === source && + !signal?.aborted && + !controller.signal.aborted + ) { + refreshError = + 'Could not refresh operation history. Showing the last successfully loaded page.'; } return false; - } finally { if (current === sequence) { refreshing = false; request = null; } } + } finally { + if (current === sequence) { + refreshing = false; + request = null; + } + } } beforeNavigate(cancel); - $effect(() => { void data; untrack(() => { cancel(); snapshot = null; refreshError = ''; updatedAt = Date.now(); }); }); - onMount(() => { const stop = pollVisible(refresh, 15000, false); return () => { stop(); cancel(); }; }); + $effect(() => { + void data; + untrack(() => { + cancel(); + snapshot = null; + refreshError = ''; + updatedAt = Date.now(); + }); + }); + onMount(() => { + const stop = pollVisible(refresh, 15000, false); + return () => { + stop(); + cancel(); + }; + }); Host operation history · Electric Cherry
- - {#snippet actions()} void refresh()} />{/snippet} + + {#snippet actions()} void refresh()} + />{/snippet} - {#if data.scope.operation || $page.url.searchParams.has('returnTo')}{/if} - {#if data.scope.operation}
Selected host operation{data.scope.operation}
View all operations
{/if} - {#if data.scope.host || data.scope.server}
Showing {data.hostName ? `host: ${data.hostName}` : ''}{data.scope.host && data.scope.server ? ' · ' : ''}{data.scope.server ? `server: ${serverTitle(data.scope.server)}` : ''}Clear scope
{/if} + {#if data.scope.operation || $page.url.searchParams.has('returnTo')}{/if} + {#if data.scope.operation}
+
Selected host operation{data.scope.operation}
+ View all operations +
{/if} + {#if data.scope.host || data.scope.server}
+ Showing {data.hostName ? `host: ${data.hostName}` : ''}{data.scope.host && + data.scope.server + ? ' · ' + : ''}{data.scope.server + ? `server: ${serverTitle(data.scope.server)}` + : ''}Clear scope +
{/if}

{selectedFilter.description}

- {#if !history.operations.length}

{data.scope.operation ? 'This operation is unavailable in the selected history. It may have expired or be excluded by the current filters.' : selectedFilter.empty}

+ {#if !history.operations.length}

+ {data.scope.operation + ? 'This operation is unavailable in the selected history. It may have expired or be excluded by the current filters.' + : selectedFilter.empty} +

{:else}
- - {#each history.operations as operation (operation.id)} - - - - - - - - - - {/each} + + {#each history.operations as operation (operation.id)} + + + + + + + + + + {/each}
OperationHostTargetSourceStateUpdatedResult
{hostOperationTitle(operation.kind)}
Technical details{operation.kind}{operation.id}
{#if operation.hostName !== null}{operation.hostName}{:else}Deleted host{operation.hostId}{/if} - {operation.serverName ? serverTitle(operation.serverName) : operation.instanceId ? 'Server container' : 'Host-level'} - {#if operation.instanceId && !operation.serverName}{operation.instanceId}{/if} - {#if operation.targetPath}{operation.targetPath}{/if} - {operation.source ?? 'Not recorded'}{operation.error ?? '—'}
OperationHostTargetSourceStateUpdatedResult
{hostOperationTitle(operation.kind)} +
+ Technical details{operation.kind}{operation.id} +
{#if operation.hostName !== null}{operation.hostName}{:else}Deleted host{operation.hostId}{/if} + {operation.serverName + ? serverTitle(operation.serverName) + : operation.instanceId + ? 'Server container' + : 'Host-level'} + {#if operation.instanceId && !operation.serverName}{operation.instanceId}{/if} + {#if operation.targetPath}{operation.targetPath}{/if} + {operation.source ?? 'Not recorded'}{operation.error ?? '—'}
{/if} @@ -128,40 +284,183 @@
diff --git a/src/routes/leaderboard/+page.svelte b/src/routes/leaderboard/+page.svelte index 0e94129..eb4d246 100644 --- a/src/routes/leaderboard/+page.svelte +++ b/src/routes/leaderboard/+page.svelte @@ -5,10 +5,24 @@ import { goto } from '$app/navigation'; import { page, navigating } from '$app/stores'; const changingFilters = $derived($navigating?.to?.route.id === '/leaderboard'); - import { leaderboardMetrics, playerStatMetrics, type LeaderboardMetric, type PlayerStatMetric, type LeaderboardEntry, type LeaderboardMap, type LeaderboardResponse } from '$lib/leaderboard.js'; + import { + leaderboardMetrics, + playerStatMetrics, + type LeaderboardMetric, + type PlayerStatMetric, + type LeaderboardEntry, + type LeaderboardMap, + type LeaderboardResponse + } from '$lib/leaderboard.js'; import { createLeaderboardUpdates, leaderboardPageSize } from '$lib/leaderboard-updates.js'; import { playerContextHref } from '$lib/leaderboard-navigation.js'; - import { efficiencyMetrics, isEfficiencyMetric, efficiencyRequirement, efficiencySample, formatEfficiencyValue } from '$lib/leaderboard-efficiency.js'; + import { + efficiencyMetrics, + isEfficiencyMetric, + efficiencyRequirement, + efficiencySample, + formatEfficiencyValue + } from '$lib/leaderboard-efficiency.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; @@ -18,9 +32,17 @@ const number = new Intl.NumberFormat('en-US'); const pageSize = leaderboardPageSize; - const games = [{ key: 't5', label: 'Black Ops I' }, { key: 't6', label: 'Black Ops II' }]; + const games = [ + { key: 't5', label: 'Black Ops I' }, + { key: 't6', label: 'Black Ops II' } + ]; const defaultColumns: PlayerStatMetric[] = ['kills', 'roundsSurvived', 'gamesPlayed']; - const squadSizes = [{ key: '1', label: 'Solo' }, { key: '2', label: 'Duo' }, { key: '3', label: 'Trio' }, { key: '4', label: 'Four players' }]; + const squadSizes = [ + { key: '1', label: 'Solo' }, + { key: '2', label: 'Duo' }, + { key: '3', label: 'Trio' }, + { key: '4', label: 'Four players' } + ]; const descriptions: Record = { highestRound: 'The highest recorded round, including players who joined mid-game.', kills: 'Cumulative kills across recorded games.', @@ -28,22 +50,30 @@ headshots: 'Cumulative headshots across recorded games.', revives: 'Cumulative teammates revived across recorded games.', downs: 'Cumulative downs across recorded games, ranked from most to fewest.', - gamesPlayed: 'Finished games with recorded player stats, excluding players who left in the same round they joined.', + gamesPlayed: + 'Finished games with recorded player stats, excluding players who left in the same round they joined.', gamesCompleted: 'Games where the player appears in the final game-end report.', bestKills: 'The most kills reported for a player in a single game.', - bestScore: 'The highest reported points balance. Spending points reduces this balance; it is not total points earned.' + bestScore: + 'The highest reported points balance. Spending points reduces this balance; it is not total points earned.' }; - const description = (key: LeaderboardMetric) => isEfficiencyMetric(key) - ? efficiencyMetrics.find(item => item.key === key)!.description : descriptions[key]; - const formatValue = (entry: LeaderboardEntry, key: LeaderboardMetric) => isEfficiencyMetric(key) - ? formatEfficiencyValue(entry[key], key) : number.format(entry[key]); + const description = (key: LeaderboardMetric) => + isEfficiencyMetric(key) + ? efficiencyMetrics.find((item) => item.key === key)!.description + : descriptions[key]; + const formatValue = (entry: LeaderboardEntry, key: LeaderboardMetric) => + isEfficiencyMetric(key) + ? formatEfficiencyValue(entry[key], key) + : number.format(entry[key]); let data = $state(untrack(() => initial.leaderboard)); let maps = $state(untrack(() => initial.leaderboard.maps)); let loading = $state(false); let error = $state(''); - let search = $state(untrack(() => ($page.url.searchParams.get('q') ?? '').trim().slice(0, 100))); - let checkedAt = $state(untrack(() => initial.leaderboard ? new Date() : null)); + let search = $state( + untrack(() => ($page.url.searchParams.get('q') ?? '').trim().slice(0, 100)) + ); + let checkedAt = $state(untrack(() => (initial.leaderboard ? new Date() : null))); let pendingUpdate = $state(null); let filtersOpen = $state(false); let showAllStats = $state(false); @@ -54,21 +84,50 @@ let pendingUrl: URL | undefined; let navigationId = 0; - const game = $derived(games.some(item => item.key === $page.url.searchParams.get('game')) ? $page.url.searchParams.get('game')! : ''); + const game = $derived( + games.some((item) => item.key === $page.url.searchParams.get('game')) + ? $page.url.searchParams.get('game')! + : '' + ); const map = $derived($page.url.searchParams.get('map') ?? ''); - const players = $derived(squadSizes.some(size => size.key === $page.url.searchParams.get('players')) ? $page.url.searchParams.get('players')! : ''); - const playersLabel = $derived(squadSizes.find(size => size.key === players)?.label ?? 'All squad sizes'); + const players = $derived( + squadSizes.some((size) => size.key === $page.url.searchParams.get('players')) + ? $page.url.searchParams.get('players')! + : '' + ); + const playersLabel = $derived( + squadSizes.find((size) => size.key === players)?.label ?? 'All squad sizes' + ); const query = $derived(($page.url.searchParams.get('q') ?? '').trim().slice(0, 100)); - const metric = $derived(leaderboardMetrics.find(item => item.key === $page.url.searchParams.get('metric'))?.key ?? 'highestRound'); - const offset = $derived(Math.max(0, Math.min(1_000_000, Math.floor(Number($page.url.searchParams.get('offset')) || 0)))); - const filteredMaps = $derived(maps.filter(item => !game || item.game === game)); - const mapName = $derived(maps.find(item => item.key === map)?.name); - const scope = $derived(mapName ?? (game ? games.find(item => item.key === game)!.label : 'All games & maps')); - const metricLabel = $derived(leaderboardMetrics.find(item => item.key === metric)!.label); - const columns = $derived([metric, ...(showAllStats ? playerStatMetrics.map(item => item.key) : defaultColumns).filter(key => key !== metric)]); + const metric = $derived( + leaderboardMetrics.find((item) => item.key === $page.url.searchParams.get('metric'))?.key ?? + 'highestRound' + ); + const offset = $derived( + Math.max( + 0, + Math.min(1_000_000, Math.floor(Number($page.url.searchParams.get('offset')) || 0)) + ) + ); + const filteredMaps = $derived(maps.filter((item) => !game || item.game === game)); + const mapName = $derived(maps.find((item) => item.key === map)?.name); + const scope = $derived( + mapName ?? (game ? games.find((item) => item.key === game)!.label : 'All games & maps') + ); + const metricLabel = $derived(leaderboardMetrics.find((item) => item.key === metric)!.label); + const columns = $derived([ + metric, + ...(showAllStats ? playerStatMetrics.map((item) => item.key) : defaultColumns).filter( + (key) => key !== metric + ) + ]); const filterCount = $derived([game, map, players, query].filter(Boolean).length); const apiUrl = $derived.by(() => { - const params = new URLSearchParams({ metric, limit: String(pageSize), offset: String(offset) }); + const params = new URLSearchParams({ + metric, + limit: String(pageSize), + offset: String(offset) + }); if (game) params.set('game', game); params.set('map', map); if (players) params.set('players', players); @@ -78,18 +137,30 @@ const updates = untrack(() => createLeaderboardUpdates(apiUrl, initial.leaderboard)); - $effect(() => { search = query; }); + $effect(() => { + search = query; + }); $effect(() => { const report = initial.leaderboard; untrack(() => { - ++requestId; controller?.abort(); data = report; maps = report.maps; - loadedUrl = apiUrl; loading = false; error = ''; checkedAt = new Date(); - pendingUpdate = null; updates.reset(apiUrl, report); + ++requestId; + controller?.abort(); + data = report; + maps = report.maps; + loadedUrl = apiUrl; + loading = false; + error = ''; + checkedAt = new Date(); + pendingUpdate = null; + updates.reset(apiUrl, report); }); }); onMount(() => { - const refreshVisible = () => { if (!document.hidden && !loading && !changingFilters && !pendingUrl) void refresh(apiUrl); }; + const refreshVisible = () => { + if (!document.hidden && !loading && !changingFilters && !pendingUrl) + void refresh(apiUrl); + }; const interval = setInterval(refreshVisible, 15_000); document.addEventListener('visibilitychange', refreshVisible); return () => { @@ -110,7 +181,10 @@ error = ''; const timeout = setTimeout(() => activeController.abort(), 20_000); try { - const response = await fetch(url, { signal: activeController.signal, cache: 'no-store' }); + const response = await fetch(url, { + signal: activeController.signal, + cache: 'no-store' + }); if (!response.ok) throw new Error(`Leaderboard request failed (${response.status}).`); const result: LeaderboardResponse = await response.json(); if (id !== requestId || url !== apiUrl || changingFilters || pendingUrl) return; @@ -124,7 +198,11 @@ checkedAt = new Date(); } catch (cause) { if (id !== requestId) return; - error = activeController.signal.aborted ? 'The leaderboard request timed out.' : cause instanceof Error ? cause.message : 'Unable to load the leaderboard.'; + error = activeController.signal.aborted + ? 'The leaderboard request timed out.' + : cause instanceof Error + ? cause.message + : 'Unable to load the leaderboard.'; } finally { clearTimeout(timeout); if (id === requestId) loading = false; @@ -138,19 +216,25 @@ } function updateFilters(changes: Record) { - ++requestId; controller?.abort(); loading = false; + ++requestId; + controller?.abort(); + loading = false; // Keep rapid changes together while the preceding navigation settles. const url = new URL(pendingUrl ?? $page.url); url.searchParams.delete('offset'); for (const [key, value] of Object.entries(changes)) { - if (value || key === 'map' && value !== null) url.searchParams.set(key, value); + if (value || (key === 'map' && value !== null)) url.searchParams.set(key, value); else url.searchParams.delete(key); } pendingUrl = url; const id = ++navigationId; void goto(`${url.pathname}${url.search}`, { keepFocus: true, noScroll: true }) - .catch(() => { error = 'Unable to update the leaderboard filters. Please try again.'; }) - .finally(() => { if (id === navigationId) pendingUrl = undefined; }); + .catch(() => { + error = 'Unable to update the leaderboard filters. Please try again.'; + }) + .finally(() => { + if (id === navigationId) pendingUrl = undefined; + }); } function clearFilters() { @@ -161,107 +245,264 @@ Leaderboard · Electric Cherry - +
- {#snippet actions()}
{/snippet} + {#snippet actions()}
+ +
{/snippet} {#snippet filterFields(suffix: string)}
-
- + updateFilters({ game: event.currentTarget.value, map: null })} + > + {#each games as item (item.key)}{/each}
-
- updateFilters({ map: event.currentTarget.value })} + > - {#if map && !filteredMaps.some(item => item.key === map)}{/if} - {#each filteredMaps as item (item.key)}{/each} + {#if map && !filteredMaps.some((item) => item.key === map)}{/if} + {#each filteredMaps as item (item.key)}{/each}
-
- Players recorded at the start of each match. Matches with unknown starting counts appear only under All squad sizes. +
+ + Players recorded at the start of each match. Matches with unknown starting + counts appear only under All squad sizes.
-
- updateFilters({ metric: event.currentTarget.value })} + > + {#each leaderboardMetrics as item (item.key)}{/each}
-
{ event.preventDefault(); updateFilters({ q: search.trim() }); if (suffix) filtersOpen = false; }} role="search"> + { + event.preventDefault(); + updateFilters({ q: search.trim() }); + if (suffix) filtersOpen = false; + }} + role="search" + > - Press Enter to search. Player ranks are preserved. + Press Enter to search. Player ranks are preserved.
{/snippet}
{@render filterFields('')}
- + {@render filterFields('-mobile')} - {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet}
-
{scope}{playersLabel}All time
- {#if game || players || query}{/if} +
+ {scope}{playersLabel}All time +
+ {#if game || players || query}{/if}
{#if error} - + {/if} {#if data}
-
Highest round{number.format(data.summary.highestRound)}{game || map ? 'Selection record' : 'Fleet record'}
-
Total kills{number.format(data.summary.kills)}Across all players
-
Total rounds{number.format(data.summary.roundsSurvived)}Player rounds
-
Games tracked{number.format(data.summary.matches)}Completed games
-
Players{number.format(data.summary.players)}Unique players
+
+ Highest round{number.format(data.summary.highestRound)}{game || map ? 'Selection record' : 'Fleet record'} +
+
+ Total kills{number.format(data.summary.kills)}Across all players +
+
+ Total rounds{number.format(data.summary.roundsSurvived)}Player rounds +
+
+ Games tracked{number.format(data.summary.matches)}Completed games +
+
+ Players{number.format(data.summary.players)}Unique players +
-
+
-

{metricLabel}

- {#if isEfficiencyMetric(metric)}

{efficiencyRequirement(metric)} within the selected game and map.

{/if} -

{description(metric)} Equal values share a rank.

+
+

{metricLabel}

+ {#if isEfficiencyMetric(metric)}

+ {efficiencyRequirement(metric)} within the selected game and map. +

{/if} +

+ {description(metric)} Equal values share a rank. +

+
+
+ {#if !error}{/if}
-
{#if !error}{/if}
- {#if pendingUpdate}
{/if} + {#if pendingUpdate}
+ +
{/if} {#if data.entries.length} -
+
- - - - - {#each columns as key (key)} - - {/each} - + + + + + {#each columns as key (key)} + + {/each} + {#each data.entries as entry (entry.guid)} - + {#each columns as key (key)} {/each} @@ -271,129 +512,566 @@
{scope}: players ranked by {metricLabel.toLowerCase()}, highest first
RankPlayer - -
{scope}: players ranked by {metricLabel.toLowerCase()}, highest first
RankPlayer + +
{number.format(entry.rank)}{number.format(entry.rank)} - {entry.name} - + {entry.name} {formatValue(entry, key)} {#if isEfficiencyMetric(key)} - {efficiencySample(entry, key)} · {number.format(entry.gamesPlayed)} games + {efficiencySample(entry, key)} · {number.format( + entry.gamesPlayed + )} games {/if} {#if key === metric && (key === 'highestRound' || key === 'bestKills' || key === 'bestScore')} - + {/if}
{:else if isEfficiencyMetric(metric)} - - {#snippet actions()}{/snippet} + + {#snippet actions()}{/snippet} {:else if query || game || map || players || offset} - - {#snippet actions()}{/snippet} + + {#snippet actions()}{/snippet} {:else} - + {/if}
{:else if loading}
Loading player records…
{/if} - {#if data}
Selection totals
Highest round
{number.format(data.summary.highestRound)}
Total kills
{number.format(data.summary.kills)}
Total rounds
{number.format(data.summary.roundsSurvived)}
Games tracked
{number.format(data.summary.matches)}
Players
{number.format(data.summary.players)}
{/if} + {#if data}
+ Selection totals +
+
+
Highest round
+
{number.format(data.summary.highestRound)}
+
+
+
Total kills
+
{number.format(data.summary.kills)}
+
+
+
Total rounds
+
{number.format(data.summary.roundsSurvived)}
+
+
+
Games tracked
+
{number.format(data.summary.matches)}
+
+
+
Players
+
{number.format(data.summary.players)}
+
+
+
{/if}
How these records work
-

Each GUID has one leaderboard entry across Black Ops I and Black Ops II, so name changes keep a player's records together. The latest reported name across both games appears here and on the player's profile, regardless of filters.

-

Only completed games contribute to records. Games in progress and interrupted games stay out of totals and rankings. Players who left a completed game keep their recorded stats. Game, map, and starting squad-size filters choose which matches contribute; player search preserves ranks. Player totals count each GUID once. The top stats cover all recorded players in the selected game and map; player search and ranking qualification do not change those totals.

-

Highest round includes mid-game joins. Total rounds counts round progression during observed participation, excluding earlier rounds. Joining at round 8 records a highest round of 8; reaching round 9 adds one to total rounds. The fleet total adds all players’ rounds.

-

Kills, headshots, revives, and downs accumulate across recorded games. Highest reported score is the largest observed points balance, not the total points a player has earned. Equal values share a rank.

-

Records cover telemetry saved since tracking began; older unrecorded games cannot be included.{#if data?.summary.firstTrackedAt}{' '}Tracking began for this selection.{/if}{#if data?.summary.updatedAt}{' '}Latest report: .{/if}

+

+ Each GUID has one leaderboard entry across Black Ops I and Black Ops II, so name + changes keep a player's records together. The latest reported name across both games + appears here and on the player's profile, regardless of filters. +

+

+ Only completed games contribute to records. Games in progress and interrupted games + stay out of totals and rankings. Players who left a completed game keep their + recorded stats. Game, map, and starting squad-size filters choose which matches + contribute; player search preserves ranks. Player totals count each GUID once. The + top stats cover all recorded players in the selected game and map; player search and + ranking qualification do not change those totals. +

+

+ Highest round includes mid-game joins. Total rounds counts round progression during + observed participation, excluding earlier rounds. Joining at round 8 records a + highest round of 8; reaching round 9 adds one to total rounds. The fleet total adds + all players’ rounds. +

+

+ Kills, headshots, revives, and downs accumulate across recorded games. Highest + reported score is the largest observed points balance, not the total points a player + has earned. Equal values share a rank. +

+

+ Records cover telemetry saved since tracking began; older unrecorded games cannot be + included.{#if data?.summary.firstTrackedAt}{' '}Tracking began for this selection.{/if}{#if data?.summary.updatedAt}{' '}Latest report: .{/if} +

diff --git a/src/routes/leaderboard/players/[guid]/+page.server.ts b/src/routes/leaderboard/players/[guid]/+page.server.ts index ab6b6b9..b916d1e 100644 --- a/src/routes/leaderboard/players/[guid]/+page.server.ts +++ b/src/routes/leaderboard/players/[guid]/+page.server.ts @@ -9,16 +9,36 @@ export async function load({ params, url }: { params: { guid: string }; url: URL const map = url.searchParams.get('map'); const requestedMetric = url.searchParams.get('metric'); const players = url.searchParams.get('players'); - if ((game !== null && game !== 't5' && game !== 't6') || (map !== null && map.length > 1024) - || (requestedMetric !== null && !isLeaderboardMetric(requestedMetric)) - || (players !== null && !/^[1-4]$/.test(players))) { - error(400, 'Use game t5 or t6, a supported metric, players 1–4, and a map key up to 1024 characters.'); + if ( + (game !== null && game !== 't5' && game !== 't6') || + (map !== null && map.length > 1024) || + (requestedMetric !== null && !isLeaderboardMetric(requestedMetric)) || + (players !== null && !/^[1-4]$/.test(players)) + ) { + error( + 400, + 'Use game t5 or t6, a supported metric, players 1–4, and a map key up to 1024 characters.' + ); } - const metric = requestedMetric !== null && isLeaderboardMetric(requestedMetric) ? requestedMetric : 'highestRound'; - const options = { game: game || undefined, map: map || undefined, - players: players === null ? undefined : Number(players) as LeaderboardPlayerCount, metric }; - const { profile, entry } = await playerReport(params.guid, Number(url.searchParams.get('offset') ?? 0), options); + const metric = + requestedMetric !== null && isLeaderboardMetric(requestedMetric) + ? requestedMetric + : 'highestRound'; + const options = { + game: game || undefined, + map: map || undefined, + players: players === null ? undefined : (Number(players) as LeaderboardPlayerCount), + metric + }; + const { profile, entry } = await playerReport( + params.guid, + Number(url.searchParams.get('offset') ?? 0), + options + ); if (!profile) error(404, 'No recorded stats found for this player'); - return { profile, returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')), - scope: { game: game ?? '', map: map ?? '', players: players ?? '', metric, entry } }; + return { + profile, + returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')), + scope: { game: game ?? '', map: map ?? '', players: players ?? '', metric, entry } + }; } diff --git a/src/routes/leaderboard/players/[guid]/+page.svelte b/src/routes/leaderboard/players/[guid]/+page.svelte index 3ae64f6..efafa37 100644 --- a/src/routes/leaderboard/players/[guid]/+page.svelte +++ b/src/routes/leaderboard/players/[guid]/+page.svelte @@ -5,8 +5,22 @@ import { goto, invalidateAll } from '$app/navigation'; import { page } from '$app/stores'; import { contextHref } from '$lib/navigation-context.js'; - import { leaderboardMetrics, playerStatMetrics, type PlayerProfile, type LeaderboardMetric, type LeaderboardEntry } from '$lib/leaderboard.js'; - import { efficiencyMetrics, efficiencyValue, efficiencyEligible, efficiencyRequirement, efficiencySample, formatEfficiencyValue, isEfficiencyMetric } from '$lib/leaderboard-efficiency.js'; + import { + leaderboardMetrics, + playerStatMetrics, + type PlayerProfile, + type LeaderboardMetric, + type LeaderboardEntry + } from '$lib/leaderboard.js'; + import { + efficiencyMetrics, + efficiencyValue, + efficiencyEligible, + efficiencyRequirement, + efficiencySample, + formatEfficiencyValue, + isEfficiencyMetric + } from '$lib/leaderboard-efficiency.js'; import { profileViewHref } from '$lib/leaderboard-navigation.js'; import { timeFormat } from '$ui/stores/time-zone.js'; import { serverTitle } from '$ui/server-labels.js'; @@ -22,28 +36,65 @@ import PlayerMatchCards from '$ui/components/PlayerMatchCards.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; - let { data }: { data: { profile: PlayerProfile; returnTo: string; - scope: { game: string; map: string; players?: string; metric: LeaderboardMetric; entry: LeaderboardEntry | null } } } = $props(); + let { + data + }: { + data: { + profile: PlayerProfile; + returnTo: string; + scope: { + game: string; + map: string; + players?: string; + metric: LeaderboardMetric; + entry: LeaderboardEntry | null; + }; + }; + } = $props(); const number = new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }); const profile = $derived(data.profile); const stats = $derived(profile.stats); const scope = $derived(data.scope); const returnTo = $derived(`${$page.url.pathname}${$page.url.search}`); - const filteredMaps = $derived(profile.maps.filter(map => !scope.game || map.game === scope.game)); - const selectedMetric = $derived(leaderboardMetrics.find(metric => metric.key === scope.metric)!); - const selectedValue = $derived(scope.entry ? isEfficiencyMetric(scope.metric) ? formatEfficiencyValue(scope.entry[scope.metric], scope.metric) : number.format(scope.entry[scope.metric]) : '—'); - const gameName = (game: string) => game === 't5' ? 'Black Ops I' : game === 't6' ? 'Black Ops II' : 'All games'; - const squadName = (players?: string) => ({ '1': 'Solo', '2': 'Duo', '3': 'Trio', '4': 'Four players' } as Record)[players ?? ''] ?? 'All squad sizes'; - const scopeLabel = $derived(`${gameName(scope.game)} · ${profile.maps.find(map => map.key === scope.map)?.name ?? (scope.map || 'All maps')} · ${squadName(scope.players)}`); + const filteredMaps = $derived( + profile.maps.filter((map) => !scope.game || map.game === scope.game) + ); + const selectedMetric = $derived( + leaderboardMetrics.find((metric) => metric.key === scope.metric)! + ); + const selectedValue = $derived( + scope.entry + ? isEfficiencyMetric(scope.metric) + ? formatEfficiencyValue(scope.entry[scope.metric], scope.metric) + : number.format(scope.entry[scope.metric]) + : '—' + ); + const gameName = (game: string) => + game === 't5' ? 'Black Ops I' : game === 't6' ? 'Black Ops II' : 'All games'; + const squadName = (players?: string) => + (({ '1': 'Solo', '2': 'Duo', '3': 'Trio', '4': 'Four players' }) as Record)[ + players ?? '' + ] ?? 'All squad sizes'; + const scopeLabel = $derived( + `${gameName(scope.game)} · ${profile.maps.find((map) => map.key === scope.map)?.name ?? (scope.map || 'All maps')} · ${squadName(scope.players)}` + ); const scopedLeaderboard = $derived.by(() => { const params = new URLSearchParams({ metric: scope.metric, map: scope.map }); if (scope.game) params.set('game', scope.game); if (scope.players) params.set('players', scope.players); return `/leaderboard?${params}`; }); - const records = [{ key: 'highestRound', label: 'Best round' }, { key: 'bestKills', label: 'Best kills' }, { key: 'bestScore', label: 'Best score' }] as const; - const overviewMetrics = playerStatMetrics.filter(metric => ['kills', 'roundsSurvived', 'revives', 'gamesCompleted'].includes(metric.key)); - const otherMetrics = playerStatMetrics.filter(metric => ['headshots', 'downs', 'bestKills', 'bestScore'].includes(metric.key)); + const records = [ + { key: 'highestRound', label: 'Best round' }, + { key: 'bestKills', label: 'Best kills' }, + { key: 'bestScore', label: 'Best score' } + ] as const; + const overviewMetrics = playerStatMetrics.filter((metric) => + ['kills', 'roundsSurvived', 'revives', 'gamesCompleted'].includes(metric.key) + ); + const otherMetrics = playerStatMetrics.filter((metric) => + ['headshots', 'downs', 'bestKills', 'bestScore'].includes(metric.key) + ); let filterOpen = $state(false); let filtering = $state(false); let refreshError = $state(''); @@ -54,106 +105,544 @@ const url = new URL(profileViewHref(pendingUrl ?? $page.url, changes), $page.url); pendingUrl = url; const id = ++navigationId; - filtering = true; refreshError = ''; - try { await goto(`${url.pathname}${url.search}`, { keepFocus: true, noScroll: true }); updatedAt = Date.now(); } - catch { refreshError = 'Could not update player filters.'; } - finally { if (id === navigationId) { pendingUrl = undefined; filtering = false; } } + filtering = true; + refreshError = ''; + try { + await goto(`${url.pathname}${url.search}`, { keepFocus: true, noScroll: true }); + updatedAt = Date.now(); + } catch { + refreshError = 'Could not update player filters.'; + } finally { + if (id === navigationId) { + pendingUrl = undefined; + filtering = false; + } + } } async function refresh() { if (filtering) return; - try { await invalidateAll(); updatedAt = Date.now(); refreshError = ''; } - catch { refreshError = 'Player updates paused.'; return false; } + try { + await invalidateAll(); + updatedAt = Date.now(); + refreshError = ''; + } catch { + refreshError = 'Player updates paused.'; + return false; + } } - onMount(() => { updatedAt = Date.now(); return pollVisible(refresh, 30_000, false); }); + onMount(() => { + updatedAt = Date.now(); + return pollVisible(refresh, 30_000, false); + }); {profile.name} · Player stats · Electric Cherry
- - {#snippet crumbs()}{/snippet} - {#snippet actions()} void refresh()} />{/snippet} + + {#snippet crumbs()}{/snippet} + {#snippet actions()} void refresh()} + />{/snippet} - +

Player overview

-
{selectedMetric.label}

{scopeLabel}

-
-
Rank in selection{scope.entry ? `#${number.format(scope.entry.rank)}` : '—'}{scope.entry ? `${selectedMetric.label}: ${selectedValue}` : profile.matchesTotal ? 'Not yet qualified for this ranking' : 'No completed matches in this selection'}Selected leaderboard
-
Lifetime best round{number.format(stats.highestRound)}
-
Lifetime games played{number.format(stats.gamesPlayed)}{number.format(profile.maps.length)} {profile.maps.length === 1 ? 'map' : 'maps'}
+
+
+ {selectedMetric.label} +

{scopeLabel}

+
+ +
+
+
+ Rank in selection{scope.entry ? `#${number.format(scope.entry.rank)}` : '—'}{scope.entry + ? `${selectedMetric.label}: ${selectedValue}` + : profile.matchesTotal + ? 'Not yet qualified for this ranking' + : 'No completed matches in this selection'}Selected leaderboard +
+
+ Lifetime best round{number.format(stats.highestRound)} +
+
+ Lifetime games played{number.format(stats.gamesPlayed)}{number.format(profile.maps.length)} + {profile.maps.length === 1 ? 'map' : 'maps'} +
+
+ {#if isEfficiencyMetric(scope.metric)}

+ Ranking requires {efficiencyRequirement(scope.metric).toLowerCase()} in the selection. +

{/if} +
+ {#each overviewMetrics as metric}
+ Lifetime {metric.label.toLowerCase()}{number.format(stats[metric.key])} +
{/each}
- {#if isEfficiencyMetric(scope.metric)}

Ranking requires {efficiencyRequirement(scope.metric).toLowerCase()} in the selection.

{/if} -
{#each overviewMetrics as metric}
Lifetime {metric.label.toLowerCase()}{number.format(stats[metric.key])}
{/each}
-

Per-map records

Lifetime personal bests across all squad sizes, ranked against other players on each map. Tied records share a rank. Each record links to the match that established it.

-
+
+

Per-map records

+

+ Lifetime personal bests across all squad sizes, ranked against other players on each + map. Tied records share a rank. Each record links to the match that established it. +

+
+
+ +
-
- {#each records as record}{/each} - {#each profile.maps as map (`${map.game}:${map.key}`)}{#each records as record}{/each}{/each} +
+
Map{record.label}GamesTotal killsTotal rounds
{map.name}{gameName(map.game)}{number.format(map[record.key])}Rank #{number.format(map.ranks[record.key])}{number.format(map.gamesPlayed)}{number.format(map.kills)}{number.format(map.roundsSurvived)}
+ {#each records as record}{/each} + {#each profile.maps as map (`${map.game}:${map.key}`)}{#each records as record}{/each}{/each}
Map{record.label}GamesTotal killsTotal rounds
{map.name}{gameName(map.game)}{number.format(map[record.key])}Rank #{number.format(map.ranks[record.key])}{number.format(map.gamesPlayed)}{number.format(map.kills)}{number.format(map.roundsSurvived)}
-

Match history

{scopeLabel}. Stats reflect this player's participation.

+
+

Match history

+

{scopeLabel}. Stats reflect this player's participation.

+
{#if profile.matches.length} -
- -
- {#each profile.matches as match (match.id)}{/each}
Match / serverFirst observedRoundKillsRevivesDownsFinal report
{match.mapName} · Match #{match.id}{serverTitle(match.server)}{match.server_deleted_at ? ' · Removed server' : ''}{number.format(match.highestRound)}{number.format(match.kills)}{number.format(match.revives)}{number.format(match.downs)}{playerCompletionLabel(match)}
+
+
- - {:else}{#snippet actions()}{/snippet}{/if} + +
+ + {#each profile.matches as match (match.id)}{/each} +
Match / serverFirst observedRoundKillsRevivesDownsFinal report
{match.mapName} · Match #{match.id}{serverTitle(match.server)}{match.server_deleted_at + ? ' · Removed server' + : ''}{number.format(match.highestRound)}{number.format(match.kills)}{number.format(match.revives)}{number.format(match.downs)}{playerCompletionLabel(match)}
+
+ + {:else}{#snippet actions()}{/snippet}{/if}
-

Lifetime statistics

All games, maps and squad sizes. Only completed matches contribute.

{#each otherMetrics as metric}
{metric.label}
{number.format(stats[metric.key])}
{/each}{#each efficiencyMetrics as metric}
{metric.label}
{formatEfficiencyValue(efficiencyValue(stats, metric.key), metric.key)}
{efficiencySample(stats, metric.key)}{efficiencyEligible(stats, metric.key) ? 'Qualified for lifetime ranking' : `${efficiencyRequirement(metric.key)} to rank`}
{/each}
-
Identity and how these stats work
First seen Last seen

Lifetime stats combine completed games from Black Ops I and Black Ops II. Games in progress and interrupted games do not contribute. Highest round includes mid-game joins; total rounds counts observed progression. Kills, headshots, revives and downs use each match's highest reported counters.

Highest reported score is the largest observed points balance, not total points earned. Games completed counts matches where the player appears in the final report; it does not indicate a win or uninterrupted survival. Records follow the same GUID across name changes. Recorded presence does not establish continuous survival.

+
+
+

Lifetime statistics

+

All games, maps and squad sizes. Only completed matches contribute.

+
+
+ {#each otherMetrics as metric}
+
{metric.label}
+
{number.format(stats[metric.key])}
+
{/each}{#each efficiencyMetrics as metric}
+
{metric.label}
+
{formatEfficiencyValue(efficiencyValue(stats, metric.key), metric.key)}
+ {efficiencySample(stats, metric.key)}{efficiencyEligible(stats, metric.key) + ? 'Qualified for lifetime ranking' + : `${efficiencyRequirement(metric.key)} to rank`} +
{/each} +
+
+
+ Identity and how these stats work +
+ First seen Last seen +
+

+ Lifetime stats combine completed games from Black Ops I and Black Ops II. Games in + progress and interrupted games do not contribute. Highest round includes mid-game joins; + total rounds counts observed progression. Kills, headshots, revives and downs use each + match's highest reported counters. +

+

+ Highest reported score is the largest observed points balance, not total points earned. + Games completed counts matches where the player appears in the final report; it does not + indicate a win or uninterrupted survival. Records follow the same GUID across name + changes. Recorded presence does not establish continuous survival. +

+
- +
- - - - + + + +
- {#snippet actions()}{/snippet} + {#snippet actions()}{/snippet} + diff --git a/src/routes/login/+page.svelte b/src/routes/login/+page.svelte index 4a059cc..f5b9adc 100644 --- a/src/routes/login/+page.svelte +++ b/src/routes/login/+page.svelte @@ -2,45 +2,145 @@ import { page } from '$app/stores'; import Button from '$ui/components/primitives/Button.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; - let username = $state(''); let password = $state(''); let busy = $state(false); let error = $state(''); + let username = $state(''); + let password = $state(''); + let busy = $state(false); + let error = $state(''); function destination(): string { const next = $page.url.searchParams.get('next'); - if (!next?.startsWith('/') || next.startsWith('//') || /[\\\x00-\x1f]/.test(next)) return '/'; + // eslint-disable-next-line no-control-regex -- Redirects must reject control characters. + if (!next?.startsWith('/') || next.startsWith('//') || /[\\\x00-\x1f]/.test(next)) + return '/'; const target = new URL(next, window.location.origin); - return target.origin === window.location.origin && !['/login', '/setup'].includes(target.pathname) ? target.pathname + target.search + target.hash : '/'; + return target.origin === window.location.origin && + !['/login', '/setup'].includes(target.pathname) + ? target.pathname + target.search + target.hash + : '/'; } async function login(event: SubmitEvent) { - event.preventDefault(); if (busy) return; busy = true; error = ''; + event.preventDefault(); + if (busy) return; + busy = true; + error = ''; try { - const response = await fetch('/api/auth/login', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ username, password }) }); + const response = await fetch('/api/auth/login', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username, password }) + }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not sign in. Try again.'); password = ''; window.location.assign(body.user?.mustChangePassword ? '/account' : destination()); - } catch (cause) { error = (cause as Error).message; password = ''; busy = false; } + } catch (cause) { + error = (cause as Error).message; + password = ''; + busy = false; + } } -Sign in · Electric Cherry +Sign in · Electric Cherry
- + {#if error}{/if}
- - + +

Need access or a password reset? Contact an Electric Cherry administrator.

- {#if $page.data.setupRequired}Set up the first administrator →{/if} + {#if $page.data.setupRequired}Set up the first administrator →{/if}
diff --git a/src/routes/maintenance/+page.svelte b/src/routes/maintenance/+page.svelte index 383381e..f5e5387 100644 --- a/src/routes/maintenance/+page.svelte +++ b/src/routes/maintenance/+page.svelte @@ -23,7 +23,10 @@ import Freshness from '$ui/components/Freshness.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; import ListFooter from '$ui/components/primitives/ListFooter.svelte'; - import { groupMaintenanceScanErrors, groupMaintenanceVersions } from '$ui/maintenance-presentation.js'; + import { + groupMaintenanceScanErrors, + groupMaintenanceVersions + } from '$ui/maintenance-presentation.js'; import { countLabel, plural } from '$ui/plural.js'; let data = $state(null); @@ -41,59 +44,169 @@ let scheduleLoaded = false; let displayedZone = $state(''); let request = 0; - const focusedJob = $derived(/^[1-9]\d*$/.test($page.url.searchParams.get('job') ?? '') ? $page.url.searchParams.get('job') : null); + const focusedJob = $derived( + /^[1-9]\d*$/.test($page.url.searchParams.get('job') ?? '') + ? $page.url.searchParams.get('job') + : null + ); let scrolledJob: string | null = null; $effect(() => { - if (!focusedJob) { scrolledJob = null; return; } - if (data?.runtime.jobs.some(job => String(job.id) === focusedJob) && scrolledJob !== focusedJob) { + if (!focusedJob) { + scrolledJob = null; + return; + } + if ( + data?.runtime.jobs.some((job) => String(job.id) === focusedJob) && + scrolledJob !== focusedJob + ) { scrolledJob = focusedJob; - void tick().then(() => document.getElementById(`job-${focusedJob}`)?.scrollIntoView({ block: 'start' })); + void tick().then(() => + document.getElementById(`job-${focusedJob}`)?.scrollIntoView({ block: 'start' }) + ); } }); let checkedAt = $state(null); - const tabs = [{ id: 'overview', label: 'Overview' }, { id: 'policies', label: 'Policies & schedules' }, { id: 'activity', label: 'Activity' }]; - const policies: { id: MaintenancePolicy; title: string; description: string }[] = [ - { id: 'health', title: 'Health recovery', description: 'Restart unhealthy servers, then rebuild once if recovery fails. Confirmed missing containers go directly to rebuild. Rebuilding deletes instance files and installs the latest promoted setup.' }, - { id: 'drift', title: 'Configuration drift', description: 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' }, - { id: 'updates', title: 'Automatic Plutonium updates', description: 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' } + const tabs = [ + { id: 'overview', label: 'Overview' }, + { id: 'policies', label: 'Policies & schedules' }, + { id: 'activity', label: 'Activity' } ]; - const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []); - const outdated = $derived(versions.filter(v => v.update_status === 'outdated')); - const unknown = $derived(versions.filter(v => v.update_status === 'unknown')); - const activeJobs = $derived(data?.runtime.jobs.filter(j => !j.server_deleted_at && ['queued', 'restarting'].includes(j.state)) ?? []); - const scanErrors = $derived(groupMaintenanceScanErrors(data?.recovery.scanError, data?.runtime.servers.map(version => version.server) ?? [])); + const policies: { id: MaintenancePolicy; title: string; description: string }[] = [ + { + id: 'health', + title: 'Health recovery', + description: + 'Restart unhealthy servers, then rebuild once if recovery fails. Confirmed missing containers go directly to rebuild. Rebuilding deletes instance files and installs the latest promoted setup.' + }, + { + id: 'drift', + title: 'Configuration drift', + description: + 'Rebuild servers whose managed files differ from the promoted setup. Record the differences in a linked Campaign. Rebuilding deletes instance files.' + }, + { + id: 'updates', + title: 'Automatic Plutonium updates', + description: + 'Update outdated running servers on restart. One automatic attempt per published revision; failed verification requires inspection. Disabled by default.' + } + ]; + const versions = $derived( + data?.runtime.servers.filter( + (v) => + !server || + `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase()) + ) ?? [] + ); + const outdated = $derived(versions.filter((v) => v.update_status === 'outdated')); + const unknown = $derived(versions.filter((v) => v.update_status === 'unknown')); + const activeJobs = $derived( + data?.runtime.jobs.filter( + (j) => !j.server_deleted_at && ['queued', 'restarting'].includes(j.state) + ) ?? [] + ); + const scanErrors = $derived( + groupMaintenanceScanErrors( + data?.recovery.scanError, + data?.runtime.servers.map((version) => version.server) ?? [] + ) + ); const versionGroups = $derived(groupMaintenanceVersions(versions, activeJobs)); - const updatableServers = $derived(versionGroups.flatMap(group => group.updatable)); - const activities = $derived([ - ...(data?.recovery.incidents ?? []).map(i => ({ - key: i.id, kind: i.kind, server: i.server, server_deleted_at: i.server_deleted_at, at: i.first_seen_at, - title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift', - state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done', - label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null - })), - ...(data?.runtime.jobs ?? []).map(j => ({ - key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, server_deleted_at: j.server_deleted_at, at: j.requested_at, - title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance', - state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done', - label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued', - detail: ['queued', 'restarting'].includes(j.state) ? j.operation_stage ?? j.detail : j.detail, - href: j.provisioning_id ? `/provisioning/${encodeURIComponent(j.provisioning_id)}` : '', job: j - })) - ].sort((a, b) => b.at.localeCompare(a.at))); - const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id); - const attention = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'attention' && isLatest(a)).length); - const waiting = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'active').length); - const matching = $derived(activities.filter(a => focusedJob ? a.key === `job-${focusedJob}` : - (kind === 'all' || a.kind === kind) && - (!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) && - (view === 'overview' ? !a.server_deleted_at && a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status) - )); + const updatableServers = $derived(versionGroups.flatMap((group) => group.updatable)); + const activities = $derived( + [ + ...(data?.recovery.incidents ?? []).map((i) => ({ + key: i.id, + kind: i.kind, + server: i.server, + server_deleted_at: i.server_deleted_at, + at: i.first_seen_at, + title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift', + state: ['open', 'recovering'].includes(i.state) + ? 'active' + : i.state === 'attention' + ? 'attention' + : 'done', + label: watchdogStateLabels[i.state], + detail: i.state === 'recovering' ? (i.operation_stage ?? i.summary) : i.summary, + href: `/maintenance/${encodeURIComponent(i.id)}`, + job: null + })), + ...(data?.runtime.jobs ?? []).map((j) => ({ + key: `job-${j.id}`, + kind: j.kind === 'schedule' ? 'schedule' : 'updates', + server: j.server, + server_deleted_at: j.server_deleted_at, + at: j.requested_at, + title: + j.kind === 'schedule' + ? 'Scheduled restart' + : j.kind === 'update' + ? 'Plutonium update' + : 'Legacy maintenance', + state: ['queued', 'restarting'].includes(j.state) + ? 'active' + : j.state === 'failed' + ? 'attention' + : 'done', + label: + j.state === 'failed' + ? 'Needs attention' + : j.state === 'done' + ? 'Completed' + : j.state === 'cancelled' + ? 'Cancelled' + : j.state === 'restarting' + ? 'Recovering' + : 'Queued', + detail: ['queued', 'restarting'].includes(j.state) + ? (j.operation_stage ?? j.detail) + : j.detail, + href: j.provisioning_id + ? `/provisioning/${encodeURIComponent(j.provisioning_id)}` + : '', + job: j + })) + ].sort((a, b) => b.at.localeCompare(a.at)) + ); + const isLatest = (item: (typeof activities)[number]) => + !item.job || + !data?.runtime.jobs.some( + (j) => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id + ); + const attention = $derived( + activities.filter((a) => !a.server_deleted_at && a.state === 'attention' && isLatest(a)) + .length + ); + const waiting = $derived( + activities.filter((a) => !a.server_deleted_at && a.state === 'active').length + ); + const matching = $derived( + activities.filter((a) => + focusedJob + ? a.key === `job-${focusedJob}` + : (kind === 'all' || a.kind === kind) && + (!server || + `${a.server} ${serverTitle(a.server)}` + .toLowerCase() + .includes(server.toLowerCase())) && + (view === 'overview' + ? !a.server_deleted_at && a.state !== 'done' && isLatest(a) + : status === 'all' || a.state === status) + ) + ); $effect(() => { const params = $page.url.searchParams; - view = ['overview', 'policies', 'activity'].includes(params.get('view') ?? '') ? params.get('view')! : 'overview'; - kind = ['health', 'drift', 'updates', 'schedule'].includes(params.get('kind') ?? '') ? params.get('kind')! : 'all'; + view = ['overview', 'policies', 'activity'].includes(params.get('view') ?? '') + ? params.get('view')! + : 'overview'; + kind = ['health', 'drift', 'updates', 'schedule'].includes(params.get('kind') ?? '') + ? params.get('kind')! + : 'all'; server = params.get('server') ?? ''; - status = ['active', 'attention', 'done'].includes(params.get('status') ?? '') ? params.get('status')! : 'all'; + status = ['active', 'attention', 'done'].includes(params.get('status') ?? '') + ? params.get('status')! + : 'all'; limit = Math.max(25, Number(params.get('limit')) || 25); }); $effect(() => { @@ -105,191 +218,711 @@ function saveContext(reset = true) { if (reset) limit = 25; const url = new URL($page.url); - for (const [key, value] of Object.entries({ view: view === 'overview' ? '' : view, kind: kind === 'all' ? '' : kind, server, status: status === 'all' ? '' : status, limit: limit > 25 ? String(limit) : '' })) { - if (value) url.searchParams.set(key, value); else url.searchParams.delete(key); + for (const [key, value] of Object.entries({ + view: view === 'overview' ? '' : view, + kind: kind === 'all' ? '' : kind, + server, + status: status === 'all' ? '' : status, + limit: limit > 25 ? String(limit) : '' + })) { + if (value) url.searchParams.set(key, value); + else url.searchParams.delete(key); } void goto(url, { replaceState: true, noScroll: true, keepFocus: true }); } function receive(body: MaintenanceOverview) { data = body; checkedAt = Date.now(); - if (!scheduleLoaded) { scheduleEnabled = body.runtime.schedule.enabled; scheduleLoaded = true; } + if (!scheduleLoaded) { + scheduleEnabled = body.runtime.schedule.enabled; + scheduleLoaded = true; + } } async function load(job = focusedJob) { if (busy || refreshing) return; refreshing = true; const current = ++request; try { - const response = await fetch(`/api/maintenance${job ? `?job=${encodeURIComponent(job)}` : ''}`); + const response = await fetch( + `/api/maintenance${job ? `?job=${encodeURIComponent(job)}` : ''}` + ); if (!response.ok) throw new Error('Could not refresh maintenance.'); const body = await response.json(); - if (current === request) { receive(body); error = ''; } - } catch (cause) { if (current === request) error = (cause as Error).message; } - finally { refreshing = false; } + if (current === request) { + receive(body); + error = ''; + } + } catch (cause) { + if (current === request) error = (cause as Error).message; + } finally { + refreshing = false; + } } async function act(body: Record) { if (busy) return; - busy = true; notice = ''; error = ''; request++; + busy = true; + notice = ''; + error = ''; + request++; try { - const response = await fetch('/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const response = await fetch('/api/maintenance', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); const result = await response.json(); if (!response.ok) throw new Error(result.error ?? 'Could not update maintenance.'); - receive(result); notice = result.notice; - } catch (cause) { error = (cause as Error).message; } - finally { busy = false; } + receive(result); + notice = result.notice; + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } } - $effect(() => { const job = focusedJob; return untrack(() => pollVisible(() => load(job), 5000)); }); + $effect(() => { + const job = focusedJob; + return untrack(() => pollVisible(() => load(job), 5000)); + }); Maintenance | Electric Cherry
- + {#snippet actions()} - {#if data}{/if} + {#if data}{/if} {/snippet} - {#if error}{/if} + {#if error}{/if} {#if !error}{/if} {#if notice}

{notice}

{/if} - {#if data?.policies.paused}
All maintenance is paused. Checks and verification continue. New actions, scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, betas, and manual server operations continue.
{/if} + {#if data?.policies.paused}
+ All maintenance is paused. Checks and verification continue. New actions, + scheduled runs, and queued maintenance are held. Submitted operations finish recovery. Campaigns, + betas, and manual server operations continue. +
{/if} {#if $page.url.searchParams.has('returnTo')}{/if} - saveContext()} ariaLabel="Maintenance views" idPrefix="maintenance-tabs" panelId="maintenance-panel" /> -
- {#if data} - {#if view === 'policies'} -

Policies apply to the current fleet and respect intentionally stopped servers. Disabling a recovery or automatic update policy holds its queued actions. Already submitted operations continue verification.

-
{#each policies as policy} -
-

{policy.title}

{data.policies[policy.id] ? 'Enabled' : 'Disabled'}
-

{policy.description}

- + saveContext()} + ariaLabel="Maintenance views" + idPrefix="maintenance-tabs" + panelId="maintenance-panel" + /> +
+ {#if data} + {#if view === 'policies'} +

+ Policies apply to the current fleet and respect intentionally stopped servers. + Disabling a recovery or automatic update policy holds its queued actions. + Already submitted operations continue verification. +

+
+ {#each policies as policy} +
+
+

{policy.title}

+ {data.policies[policy.id] ? 'Enabled' : 'Disabled'} +
+

{policy.description}

+ +
+ {/each} +
+
+

Scheduled restarts

+

+ Restart the running fleet daily, even without a new release. Intentionally + stopped servers are excluded. Busy servers wait until empty and may restart + later. +

+
{ + event.preventDefault(); + void act({ + action: 'schedule', + enabled: scheduleEnabled, + time: scheduleTime, + timeZone: $timeZone + }); + }} + > + + + +
+

+ Saving follows daylight saving changes in your selected timezone. Skipped + times run at the next valid minute; repeated times run once. Missed days + become one run on resume. +

+ {#if data.runtime.schedule.enabled && data.runtime.schedule.timeZone !== $timeZone}

+ Saved for {data.runtime.schedule.time} in {data.runtime.schedule + .timeZone}. Save to use {$timeZone}. +

{/if} + {#if data.runtime.schedule.nextAt}

+ Next run: {$timeFormat.dateTime(data.runtime.schedule.nextAt)}{data + .policies.paused + ? ' · Held while maintenance is paused' + : ''} +

{/if} +

+ Disabling the schedule stops future runs. Existing requests remain in + Activity, where unsubmitted restarts can be cancelled. +

- {/each}
-
-

Scheduled restarts

-

Restart the running fleet daily, even without a new release. Intentionally stopped servers are excluded. Busy servers wait until empty and may restart later.

-
{ event.preventDefault(); void act({ action: 'schedule', enabled: scheduleEnabled, time: scheduleTime, timeZone: $timeZone }); }}> - - - -
-

Saving follows daylight saving changes in your selected timezone. Skipped times run at the next valid minute; repeated times run once. Missed days become one run on resume.

- {#if data.runtime.schedule.enabled && data.runtime.schedule.timeZone !== $timeZone}

Saved for {data.runtime.schedule.time} in {data.runtime.schedule.timeZone}. Save to use {$timeZone}.

{/if} - {#if data.runtime.schedule.nextAt}

Next run: {$timeFormat.dateTime(data.runtime.schedule.nextAt)}{data.policies.paused ? ' · Held while maintenance is paused' : ''}

{/if} -

Disabling the schedule stops future runs. Existing requests remain in Activity, where unsubmitted restarts can be cancelled.

-
- {:else if view === 'activity'} - -

Use active work and attention to cancel waiting maintenance or retry a failed verification after inspection.

- {:else} - {#if view === 'overview'} -
{attention} {plural(attention, 'needs', 'need')} attention{waiting} active{outdated.length} outdated{unknown.length} {plural(unknown.length, 'version')} unverified
- {#if data.runtime.schedule.nextAt}

Next scheduled restart: {$timeFormat.dateTime(data.runtime.schedule.nextAt)} · Manage schedule

{/if} - {#if data.recovery.lastScanAt}

Health and drift checked

{/if} - {#if scanErrors.length} -
-

Checks need attention

-
    {#each scanErrors as group (group.key)} -

  • {group.title}{group.servers.length ? ` · ${countLabel(group.servers.length, 'server')}` : ''}

    -

    {group.description}

    - {#if group.servers.length}{/if} -
    Technical details
      {#each group.details as detail}
    • {#if detail.server}{serverTitle(detail.server)}{/if}
      {detail.message}
      • {/each}
    -
    • - {/each}
+ {:else if view === 'activity'} + +

+ Use active work and attention to + cancel waiting maintenance or retry a failed verification after inspection. +

+ {:else} + {#if view === 'overview'} +
+ {attention} + {plural(attention, 'needs', 'need')} attention{waiting} active{outdated.length} outdated{unknown.length} + {plural(unknown.length, 'version')} unverified +
+ {#if data.runtime.schedule.nextAt}

+ Next scheduled restart: {$timeFormat.dateTime( + data.runtime.schedule.nextAt + )} · Manage schedule +

{/if} + {#if data.recovery.lastScanAt}

+ Health and drift checked +

{/if} + {#if scanErrors.length} +
+
+

Checks need attention

+ +
+
    + {#each scanErrors as group (group.key)} +
  • +

    + {group.title}{group.servers + .length + ? ` · ${countLabel(group.servers.length, 'server')}` + : ''} +

    +

    {group.description}

    + {#if group.servers.length}{/if} +
    + Technical details +
      + {#each group.details as detail}
    • + {#if detail.server}{serverTitle( + detail.server + )}{/if} + 
      {detail.message}
      +
      • {/each} +
    +
    +
    • + {/each} +
+
+ {/if} + {/if} +
+ + + {#if view === 'activity'}{/if} +
+ {#if view === 'overview' && ['all', 'updates'].includes(kind)} +
+
+

Plutonium updates

+ {#if updatableServers.length}{/if} +
+

+ Published revision: {data.runtime.release.revision + ? `r${data.runtime.release.revision}` + : 'Unknown'} + · + Production manifest + · + Changelog +

+ {#if data.runtime.release.checkedAt}

+ Checked +

{/if} + {#if data.runtime.release.error}{/if} +

+ Plutainer applies the available release at startup when auto-update is + enabled. Maintenance verifies the running revision after recovery. + Unknown versions require a successful check before automatic updates. +

+
+ {#each versionGroups as group (group.key)} +
+
+
+ {countLabel(group.versions.length, 'server')} · {group.revision + ? `reported r${group.revision}` + : 'revision unknown'}{group.label} +
+
+ {#if group.pending.length}{countLabel(group.pending.length, 'restart')} pending{/if} + {#if group.status === 'unknown'} + {:else if group.updatable.length}{/if} +
+
+
+ {group.versions.length === 1 + ? 'Server and check details' + : `Show ${group.versions.length} servers and check details`} +
    + {#each group.versions as version (version.server)} + {@const pending = group.pending.find( + (job) => job.server === version.server + )} +
  • +
    + + {#if pending}{#if pending.provisioning_id}View restart{:else}View pending restart{/if}{/if} +
    + {#if version.checked_at}Last checked {/if} + {#if version.error}
    {version.error}
    {/if} +
    • + {/each} +
+
+
+ {:else}{/each} +
{/if} - {/if} -
- - - {#if view === 'activity'}{/if} -
- {#if view === 'overview' && ['all', 'updates'].includes(kind)} -
-

Plutonium updates

{#if updatableServers.length}{/if}
-

Published revision: {data.runtime.release.revision ? `r${data.runtime.release.revision}` : 'Unknown'} · Production manifest · Changelog

- {#if data.runtime.release.checkedAt}

Checked

{/if} - {#if data.runtime.release.error}{/if} -

Plutainer applies the available release at startup when auto-update is enabled. Maintenance verifies the running revision after recovery. Unknown versions require a successful check before automatic updates.

-
{#each versionGroups as group (group.key)} -
-
-
{countLabel(group.versions.length, 'server')} · {group.revision ? `reported r${group.revision}` : 'revision unknown'}{group.label}
-
- {#if group.pending.length}{countLabel(group.pending.length, 'restart')} pending{/if} - {#if group.status === 'unknown'} - {:else if group.updatable.length}{/if} +
+

+ {focusedJob + ? `Maintenance request #${focusedJob}` + : view === 'overview' + ? 'Active work & attention' + : 'Maintenance activity'} +

+ {#if focusedJob}

+ Selected retained request. Show active work and attention +

{/if} +
+ {#each matching.slice(0, limit) as item (item.key)} +
+
+

+ {item.title} · +

+ {item.label}
-
-
{group.versions.length === 1 ? 'Server and check details' : `Show ${group.versions.length} servers and check details`} -
    {#each group.versions as version (version.server)} - {@const pending = group.pending.find(job => job.server === version.server)} -
  • - {#if pending}{#if pending.provisioning_id}View restart{:else}View pending restart{/if}{/if} -
    - {#if version.checked_at}Last checked {/if} - {#if version.error}
    {version.error}
    {/if} -
    • - {/each}
-
-
- {:else}{/each}
+ {#if item.detail}

{item.detail}

{/if} +

+ {item.job?.target_revision + ? ` · Expected r${item.job.target_revision} or newer` + : ''} +

+
+ {#if item.href}{item.job + ? 'View restart operation' + : 'View evidence and recovery'}{/if} + {#if item.job?.state === 'queued' && !item.server_deleted_at}{/if} + {#if item.job?.state === 'failed' && !item.server_deleted_at && item.job.operation_state === 'done'}{/if} +
+ + {:else}{/each} +
+ saveContext(false)} + label="activities" + />
{/if} -
-

{focusedJob ? `Maintenance request #${focusedJob}` : view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}

- {#if focusedJob}

Selected retained request. Show active work and attention

{/if} -
{#each matching.slice(0, limit) as item (item.key)} -
-

{item.title} ·

{item.label}
- {#if item.detail}

{item.detail}

{/if} -

{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}

-
{#if item.href}{item.job ? 'View restart operation' : 'View evidence and recovery'}{/if} - {#if item.job?.state === 'queued' && !item.server_deleted_at}{/if} - {#if item.job?.state === 'failed' && !item.server_deleted_at && item.job.operation_state === 'done'}{/if} -
-
- {:else}{/each}
- saveContext(false)} label="activities" /> -
- {/if} - {:else if !error}

Loading maintenance…

{/if} + {:else if !error}

Loading maintenance…

{/if} diff --git a/src/routes/maintenance/[id]/+page.svelte b/src/routes/maintenance/[id]/+page.svelte index 2611129..98b7c12 100644 --- a/src/routes/maintenance/[id]/+page.svelte +++ b/src/routes/maintenance/[id]/+page.svelte @@ -29,22 +29,44 @@ try { const value = JSON.parse(incident?.details ?? '{}'); return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; - } catch { return {}; } + } catch { + return {}; + } }); const files = $derived(evidence.files ?? []); const actions = $derived(detail?.actions.slice().reverse() ?? []); const events = $derived(detail?.events.slice().reverse() ?? []); - const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? ''))); - const canRetry = $derived(!incident?.server_deleted_at && (incident?.state === 'attention' || incident?.state === 'dismissed')); - const canDismiss = $derived(!incident?.server_deleted_at && (incident?.state === 'open' || incident?.state === 'attention')); + const activeOperation = $derived( + actions.some((action) => + ['queued', 'waiting', 'running', 'awaiting-health'].includes( + action.operation_state ?? '' + ) + ) + ); + const canRetry = $derived( + !incident?.server_deleted_at && + (incident?.state === 'attention' || incident?.state === 'dismissed') + ); + const canDismiss = $derived( + !incident?.server_deleted_at && + (incident?.state === 'open' || incident?.state === 'attention') + ); const explanation = $derived.by(() => { if (!incident) return ''; - if (incident.server_deleted_at) return 'This server was deleted. Its recorded finding, evidence, and recovery results are preserved below.'; - if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.'; - if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.'; - if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.'; - if (incident.state === 'recovering') return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.'; - if (incident.kind === 'health' && evidence.containerState?.toLowerCase() === 'missing') return 'Maintenance confirmed that the container is missing. When health recovery is enabled and Maintenance is not paused, three observations over at least two minutes trigger a direct rebuild without a player-count check. Capacity limits and protected work still apply.'; + if (incident.server_deleted_at) + return 'This server was deleted. Its recorded finding, evidence, and recovery results are preserved below.'; + if (incident.state === 'resolved') + return incident.kind === 'drift' + ? 'The observed file drift has cleared.' + : 'The server passed the checks for this incident.'; + if (incident.state === 'dismissed') + return 'This incident was dismissed. Retry to allow automatic recovery again.'; + if (incident.state === 'attention') + return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.'; + if (incident.state === 'recovering') + return 'Recovery is in progress. Follow the provisioning operations below for eligibility checks, progress, and health verification.'; + if (incident.kind === 'health' && evidence.containerState?.toLowerCase() === 'missing') + return 'Maintenance confirmed that the container is missing. When health recovery is enabled and Maintenance is not paused, three observations over at least two minutes trigger a direct rebuild without a player-count check. Capacity limits and protected work still apply.'; return 'Maintenance detected this problem. Automatic recovery requires a confirmed stopped container or a fresh zero-player count from RCON or telemetry before interrupting the server. Offline hosts and unknown container states wait for another check.'; }); @@ -54,10 +76,16 @@ const current = ++request; try { const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`); - if (!response.ok) throw new Error(response.status === 404 ? 'Maintenance finding not found.' : 'Could not refresh this incident.'); - const body = await response.json() as WatchdogDetail; + if (!response.ok) + throw new Error( + response.status === 404 + ? 'Maintenance finding not found.' + : 'Could not refresh this incident.' + ); + const body = (await response.json()) as WatchdogDetail; if (current !== request || id !== $page.params.id) return; - detail = body; loadError = ''; + detail = body; + loadError = ''; } catch (error) { if (current === request && id === $page.params.id) loadError = (error as Error).message; } @@ -67,97 +95,260 @@ if (busy || !incident) return; const id = incident.id; const current = ++request; - busy = action; actionError = ''; notice = ''; + busy = action; + actionError = ''; + notice = ''; try { const response = await fetch(`/api/maintenance/${encodeURIComponent(id)}`, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action }) + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action }) }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not update this incident.'); if (current !== request || id !== $page.params.id) return; - detail = body; loadError = ''; - notice = action === 'retry' ? 'Recovery retry requested. Maintenance will recheck the server before taking action.' : 'Incident dismissed.'; + detail = body; + loadError = ''; + notice = + action === 'retry' + ? 'Recovery retry requested. Maintenance will recheck the server before taking action.' + : 'Incident dismissed.'; } catch (error) { - if (current === request && id === $page.params.id) actionError = (error as Error).message; - } finally { if (current === request) busy = null; } + if (current === request && id === $page.params.id) + actionError = (error as Error).message; + } finally { + if (current === request) busy = null; + } } $effect(() => { - $page.params.id; - request++; detail = null; loadError = ''; actionError = ''; notice = ''; busy = null; - eventLimit = 25; fileLimit = 25; - untrack(() => { void load(); }); + void $page.params.id; + request++; + detail = null; + loadError = ''; + actionError = ''; + notice = ''; + busy = null; + eventLimit = 25; + fileLimit = 25; + untrack(() => { + void load(); + }); }); onMount(() => { const stop = pollVisible(load, 5000, false); - return () => { stop(); request++; }; + return () => { + stop(); + request++; + }; }); -{incident ? `${serverTitle(incident.server)} · Maintenance` : 'Maintenance finding'} | Electric Cherry +{incident ? `${serverTitle(incident.server)} · Maintenance` : 'Maintenance finding'} | Electric + Cherry
- - {#snippet crumbs()}{/snippet} + + {#snippet crumbs()}{/snippet} {#snippet actions()} - {#if canRetry}{/if} - {#if canDismiss}{/if} + {#if canRetry}{/if} + {#if canDismiss}{/if} {/snippet} - {#if loadError}{/if} + {#if loadError}{/if} {#if actionError}{/if} {#if notice}

{notice}

{/if} {#if incident && detail}
-
{#if incident.server_deleted_at}Deleted{:else}View server{/if}
+
+ {#if incident.server_deleted_at}Deleted{:else}View server{/if} +

{incident.summary}

{explanation}

- {#if activeOperation && (canRetry || canDismiss)}

Complete or cancel the active provisioning operation before retrying or dismissing this incident.

{/if} - {#if !incident.server_deleted_at}{#if incident.kind === 'drift'}

Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup. The repair is recorded as a campaign.

{:else if incident.state !== 'resolved'}

Automatic health recovery restarts an existing container, then rebuilds if needed. A confirmed missing container goes directly to rebuild. Rebuilding deletes any remaining instance files and installs the latest promoted setup.

{/if}{/if} + {#if activeOperation && (canRetry || canDismiss)}

+ Complete or cancel the active provisioning operation before retrying or + dismissing this incident. +

{/if} + {#if !incident.server_deleted_at}{#if incident.kind === 'drift'}

+ Drift repair deletes the existing instance and all of its files, then + rebuilds it with the latest promoted setup. The repair is recorded as a campaign. +

{:else if incident.state !== 'resolved'}

+ Automatic health recovery restarts an existing container, then rebuilds if + needed. A confirmed missing container goes directly to rebuild. Rebuilding + deletes any remaining instance files and installs the latest promoted setup. +

{/if}{/if}
-
First detected
{$timeFormat.dateTime(incident.first_seen_at)}
-
Last observed
{$timeFormat.dateTime(incident.last_seen_at)}
- {#if incident.finished_at}
Finished
{$timeFormat.dateTime(incident.finished_at)}
{/if} - {#if evidence.containerState}
Observed container state
{evidence.containerState}
{/if} +
+
First detected
+
{$timeFormat.dateTime(incident.first_seen_at)}
+
+
+
Last observed
+
{$timeFormat.dateTime(incident.last_seen_at)}
+
+ {#if incident.finished_at}
+
Finished
+
{$timeFormat.dateTime(incident.finished_at)}
+
{/if} + {#if evidence.containerState}
+
Observed container state
+
{evidence.containerState}
+
{/if}
- {#if evidence.reason && evidence.reason !== incident.summary}

Observed reason: {evidence.reason}

{/if} - {#if !incident.server_deleted_at}
Diagnose connectionOpen server logs
{/if} + {#if evidence.reason && evidence.reason !== incident.summary}

+ Observed reason: {evidence.reason} +

{/if} + {#if !incident.server_deleted_at}
+ Diagnose connectionOpen server logs +
{/if}

Recovery operations

- {#if !actions.length} + {#if !actions.length} {:else}
- {#each actions as action (action.id)} -
-

{action.action === 'restart' ? 'Restart server' : 'Rebuild server'}

{#if action.operation_state}{/if}
- {#if action.operation_stage}

{action.operation_stage}

{/if} - {#if action.operation_error}

{action.operation_error}

{/if} - -
View provisioning{#if action.campaign_id}View repair campaign{/if}
-
- {/each} -
{/if} + {#each actions as action (action.id)} +
+
+

+ {action.action === 'restart' + ? 'Restart server' + : 'Rebuild server'} +

+ {#if action.operation_state}{/if} +
+ {#if action.operation_stage}

{action.operation_stage}

{/if} + {#if action.operation_error}

+ {action.operation_error} +

{/if} + +
+ View provisioning{#if action.campaign_id}View repair campaign{/if} +
+
+ {/each} +
{/if}
{#if incident.kind === 'drift'}
-

Observed file drift {files.length}

-

These differences describe the observed problem. Recovery rebuilds the whole server instance.

+

+ Observed file drift {files.length} +

+

+ These differences describe the observed problem. Recovery rebuilds the whole + server instance. +

{#if !files.length} {:else}
- {#each files.slice(0, fileLimit) as file (`${file.kind}:${file.driftKind}:${file.relPath}`)} -
-
{file.driftKind === 'replace' ? 'Modified' : file.driftKind === 'missing' ? 'Missing' : 'Untracked'}{file.relPath}{file.kind}
-
Observed and expected file
-
Observed
{file.beforeSize === null ? 'Absent' : `${file.beforeSize} bytes`}{#if file.beforeSha256}{file.beforeSha256}{/if}
-
Expected
{file.afterSize === null ? 'Absent' : `${file.afterSize} bytes`}{#if file.afterSha256}{file.afterSha256}{/if}
-
-
- {/each} -
{/if} + {#each files.slice(0, fileLimit) as file (`${file.kind}:${file.driftKind}:${file.relPath}`)} +
+
+ {file.driftKind === 'replace' + ? 'Modified' + : file.driftKind === 'missing' + ? 'Missing' + : 'Untracked'}{file.relPath}{file.kind} +
+
+ Observed and expected file +
+
+
Observed
+
+ {file.beforeSize === null + ? 'Absent' + : `${file.beforeSize} bytes`}{#if file.beforeSha256}{file.beforeSha256}{/if} +
+
+
+
Expected
+
+ {file.afterSize === null + ? 'Absent' + : `${file.afterSize} bytes`}{#if file.afterSha256}{file.afterSha256}{/if} +
+
+
+
+
+ {/each} + + {/if}
{/if} @@ -166,44 +357,164 @@

Newest first · updates automatically

{#if !events.length} {:else}
    - {#each events.slice(0, eventLimit) as event (event.id)} -

  1. {event.message}

    2. - {/each} -
{/if} + {#each events.slice(0, eventLimit) as event (event.id)} +
  • + +

    {event.message}

    +
    • + {/each} + + {/if}
    {:else if !loadError}

    Loading incident…

    {/if} diff --git a/src/routes/matches/+page.server.ts b/src/routes/matches/+page.server.ts index 860a06c..20b3d62 100644 --- a/src/routes/matches/+page.server.ts +++ b/src/routes/matches/+page.server.ts @@ -3,7 +3,10 @@ import { serverMatchHistory } from '$lib/server/fleet/match-history.js'; import { matchApiParams, matchPageSize, readMatchQuery } from '$lib/match-history-query.js'; export function load({ url }: { url: URL }) { let query; - try { query = readMatchQuery(matchApiParams(url)); } - catch { error(400, 'Invalid match history filters.'); } + try { + query = readMatchQuery(matchApiParams(url)); + } catch { + error(400, 'Invalid match history filters.'); + } return { history: serverMatchHistory(undefined, matchPageSize, query) }; } diff --git a/src/routes/matches/+page.svelte b/src/routes/matches/+page.svelte index 2ab1060..c2c6f9d 100644 --- a/src/routes/matches/+page.svelte +++ b/src/routes/matches/+page.svelte @@ -12,5 +12,9 @@ diff --git a/src/routes/matches/[id]/+page.svelte b/src/routes/matches/[id]/+page.svelte index 69dcabd..d65820b 100644 --- a/src/routes/matches/[id]/+page.svelte +++ b/src/routes/matches/[id]/+page.svelte @@ -16,7 +16,13 @@ let { data }: { data: { match: MatchDetail } } = $props(); const match = $derived(data.match); - const gameName = $derived(match.game === 't5' ? 'Black Ops I' : match.game === 't6' ? 'Black Ops II' : match.game.toUpperCase()); + const gameName = $derived( + match.game === 't5' + ? 'Black Ops I' + : match.game === 't6' + ? 'Black Ops II' + : match.game.toUpperCase() + ); const number = new Intl.NumberFormat('en-US'); const stats = [ { key: 'highestRound', label: 'Highest round' }, @@ -32,50 +38,133 @@ {match.map} · Match #{match.id} · Electric Cherry
    - - {#snippet crumbs()}{/snippet} + + {#snippet crumbs()}{/snippet} {#snippet actions()}Completed{/snippet}
    - - - - + + + +

    Match details

    -
    Server
    -
    {match.startObserved ? 'Started' : 'First observed'}
    -
    Ended
    {#if match.endedAt}{:else}—{/if}
    -
    Players at start
    {match.startPlayers ?? '—'}
    +
    +
    Server
    +
    + +
    +
    +
    +
    {match.startObserved ? 'Started' : 'First observed'}
    +
    +
    +
    +
    Ended
    +
    + {#if match.endedAt}{:else}—{/if} +
    +
    +
    +
    Players at start
    +
    {match.startPlayers ?? '—'}
    +
    - {#if !match.startObserved}

    The match start was not recorded. The first observation is shown above; the full game time is unknown.

    {/if} + {#if !match.startObserved}

    + The match start was not recorded. The first observation is shown above; the full + game time is unknown. +

    {/if}

    Player results

    -

    Every recorded player, including those absent from the final report. Stats are the highest reported counters for this match.

    +

    + Every recorded player, including those absent from the final report. Stats are the + highest reported counters for this match. +

    {#if match.players.length === 0} - + {:else}
    - {#each stats as stat}{/each} + {#each stats as stat}{/each} {#each match.players as player (player.guid)} - - {#each stats as stat}{/each} + + {#each stats as stat}{/each} {/each} @@ -89,39 +178,167 @@
    About these results -

    Kills, headshots, revives, downs, and score retain each player's highest reported value. Highest reported score is a points balance, not total points earned or necessarily the final score.

    -

    Highest round includes late joins. Rounds survived counts only observed round progression while the player was present. “At match end” means the player appeared in the final report; other players keep their earlier stats. A dash means no value was recorded.

    -

    Player names use the latest recorded identity. Times reflect when reports were received and use your selected timezone. These results remain available after the server is deleted.

    +

    + Kills, headshots, revives, downs, and score retain each player's highest reported value. + Highest reported score is a points balance, not total points earned or necessarily the + final score. +

    +

    + Highest round includes late joins. Rounds survived counts only observed round + progression while the player was present. “At match end” means the player appeared in + the final report; other players keep their earlier stats. A dash means no value was + recorded. +

    +

    + Player names use the latest recorded identity. Times reflect when reports were received + and use your selected timezone. These results remain available after the server is + deleted. +

    diff --git a/src/routes/metrics/+page.svelte b/src/routes/metrics/+page.svelte index e479098..264d58a 100644 --- a/src/routes/metrics/+page.svelte +++ b/src/routes/metrics/+page.svelte @@ -36,62 +36,154 @@ const report = $derived(snapshot ?? data); const format = $derived(createTimeFormatter(report.scope.timeZone)); const captured = $derived(report.summary.observed + report.summary.unknown); - const contextSearch = $derived((page.state as { metricsTableSearch?: string }).metricsTableSearch ?? page.url.search); - const view = $derived(new URLSearchParams(contextSearch).get('view') === 'instances' ? 'instances' : new URLSearchParams(contextSearch).get('view') === 'insights' ? 'insights' : 'maps'); + const contextSearch = $derived( + (page.state as { metricsTableSearch?: string }).metricsTableSearch ?? page.url.search + ); + const view = $derived( + new URLSearchParams(contextSearch).get('view') === 'instances' + ? 'instances' + : new URLSearchParams(contextSearch).get('view') === 'insights' + ? 'insights' + : 'maps' + ); function changeView(value: string) { - const url = new URL(page.url); url.search = contextSearch; url.searchParams.set('view', value); + const url = new URL(page.url); + url.search = contextSearch; + url.searchParams.set('view', value); replaceState(url, { ...page.state, metricsTableSearch: url.search }); } - const gameOptions = $derived([{ value: '', label: 'All games' }, ...[...new Set([...report.scopeOptions.games, ...(report.scope.game ? [report.scope.game] : [])])].map(game => ({ value: game, label: gameName(game) }))]); + const gameOptions = $derived([ + { value: '', label: 'All games' }, + ...[ + ...new Set([ + ...report.scopeOptions.games, + ...(report.scope.game ? [report.scope.game] : []) + ]) + ].map((game) => ({ value: game, label: gameName(game) })) + ]); const mapOptions = $derived([ { value: '', label: 'All maps' }, - ...report.scopeOptions.maps.filter(map => !report.scope.game || map.game === report.scope.game).map(map => ({ value: JSON.stringify([map.game, map.map]), label: `${map.map}${report.scope.game ? '' : ` · ${gameName(map.game)}`}` })), - ...(report.scope.map && !report.scopeOptions.maps.some(map => map.game === report.scope.game && map.map === report.scope.map) ? [{ value: JSON.stringify([report.scope.game, report.scope.map]), label: report.scope.map }] : []) + ...report.scopeOptions.maps + .filter((map) => !report.scope.game || map.game === report.scope.game) + .map((map) => ({ + value: JSON.stringify([map.game, map.map]), + label: `${map.map}${report.scope.game ? '' : ` · ${gameName(map.game)}`}` + })), + ...(report.scope.map && + !report.scopeOptions.maps.some( + (map) => map.game === report.scope.game && map.map === report.scope.map + ) + ? [ + { + value: JSON.stringify([report.scope.game, report.scope.map]), + label: report.scope.map + } + ] + : []) ]); - const mapValue = $derived(report.scope.map ? JSON.stringify([report.scope.game, report.scope.map]) : ''); - const scopeLabel = $derived(report.scope.map ? `${report.scope.map} · ${gameName(report.scope.game!)}` : report.scope.game ? gameName(report.scope.game) : 'All games and maps'); + const mapValue = $derived( + report.scope.map ? JSON.stringify([report.scope.game, report.scope.map]) : '' + ); + const scopeLabel = $derived( + report.scope.map + ? `${report.scope.map} · ${gameName(report.scope.game!)}` + : report.scope.game + ? gameName(report.scope.game) + : 'All games and maps' + ); const refresh = createMetricsRefresh({ getQuery: () => { - const params = new URLSearchParams({ days: String(data.days), timeZone: data.scope.timeZone }); + const params = new URLSearchParams({ + days: String(data.days), + timeZone: data.scope.timeZone + }); if (data.scope.game) params.set('game', data.scope.game); if (data.scope.map) params.set('map', data.scope.map); return params.toString(); }, - onData: value => { snapshot = value; checkedAt = Date.now(); }, - onError: value => { refreshError = value; }, - onLoading: value => { refreshing = value; } + onData: (value) => { + snapshot = value; + checkedAt = Date.now(); + }, + onError: (value) => { + refreshError = value; + }, + onLoading: (value) => { + refreshing = value; + } + }); + beforeNavigate(() => { + refresh.cancel(); + scopeNavigating = true; + }); + afterNavigate(() => { + scopeNavigating = false; + attemptedPreference = null; }); - beforeNavigate(() => { refresh.cancel(); scopeNavigating = true; }); - afterNavigate(() => { scopeNavigating = false; attemptedPreference = null; }); $effect(() => { // Navigation owns the new report; an older background request cannot replace it. void data; - untrack(() => { refresh.cancel(); snapshot = null; refreshError = null; checkedAt = Date.now(); }); + untrack(() => { + refresh.cancel(); + snapshot = null; + refreshError = null; + checkedAt = Date.now(); + }); }); $effect(() => { // Layout initializes the saved preference on mount; react when it arrives. const preferredZone = $timeZone; - if (mounted && !scopeNavigating && !new URLSearchParams(contextSearch).has('timeZone') && preferredZone !== data.scope.timeZone && preferredZone !== attemptedPreference) { - untrack(() => { attemptedPreference = preferredZone; void changeScope({ timeZone: preferredZone }, true); }); + if ( + mounted && + !scopeNavigating && + !new URLSearchParams(contextSearch).has('timeZone') && + preferredZone !== data.scope.timeZone && + preferredZone !== attemptedPreference + ) { + untrack(() => { + attemptedPreference = preferredZone; + void changeScope({ timeZone: preferredZone }, true); + }); } }); onMount(() => { mounted = true; - const stop = pollVisible(() => scopeNavigating ? undefined : refresh.refresh(), 60_000, false); - return () => { stop(); refresh.cancel(); }; + const stop = pollVisible( + () => (scopeNavigating ? undefined : refresh.refresh()), + 60_000, + false + ); + return () => { + stop(); + refresh.cancel(); + }; }); - function gameName(game: string) { return game === 't5' ? 'Black Ops I' : game === 't6' ? 'Black Ops II' : game.toUpperCase(); } + function gameName(game: string) { + return game === 't5' ? 'Black Ops I' : game === 't6' ? 'Black Ops II' : game.toUpperCase(); + } async function changeScope(changes: Record, replace = false) { const params = new URLSearchParams(contextSearch); if (!params.has('timeZone')) params.set('timeZone', report.scope.timeZone); - for (const [key, value] of Object.entries(changes)) { if (value) params.set(key, value); else params.delete(key); } - params.delete('mapPage'); params.delete('serverPage'); + for (const [key, value] of Object.entries(changes)) { + if (value) params.set(key, value); + else params.delete(key); + } + params.delete('mapPage'); + params.delete('serverPage'); refresh.cancel(); scopeError = ''; scopeNavigating = true; - try { await goto(`/metrics?${params}`, { keepFocus: true, noScroll: true, replaceState: replace }); } - catch { scopeError = 'Could not change the report scope. The previous report is still shown.'; } - finally { scopeNavigating = false; } + try { + await goto(`/metrics?${params}`, { + keepFocus: true, + noScroll: true, + replaceState: replace + }); + } catch { + scopeError = 'Could not change the report scope. The previous report is still shown.'; + } finally { + scopeNavigating = false; + } } function changeMap(value: string) { if (!value) return changeScope({ map: '' }); @@ -101,7 +193,8 @@ function rangeHref(days: number): string { const params = new URLSearchParams(contextSearch); params.set('days', String(days)); - params.delete('mapPage'); params.delete('serverPage'); + params.delete('mapPage'); + params.delete('serverPage'); params.set('timeZone', report.scope.timeZone); return `/metrics?${params}`; } @@ -114,18 +207,50 @@
    - changeMap((event.currentTarget as HTMLSelectElement).value)} /> -
    {#if scopeNavigating}Updating report…{:else if !refreshError}{/if}
    + changeMap((event.currentTarget as HTMLSelectElement).value)} + /> +
    + {#if scopeNavigating}Updating report…{:else if !refreshError}{/if} +
    {#if scopeError}{/if} {#if refreshError} {/if} @@ -133,68 +258,254 @@ {#if captured > 0}{/if} {#if captured === 0} - 0 ? 'Waiting for observed time' : 'No captured time in this scope'} + 0 + ? 'Waiting for observed time' + : 'No captured time in this scope'} description={report.latestSampleAt === null - ? report.registeredServerCount > 0 ? 'Leave the manager running to collect server status and build a history.' : 'Register a server to begin collecting utilization history.' - : report.servers.length > 0 ? 'Captured samples have no elapsed time in this window. Time builds between status checks.' - : 'There are no elapsed observations for the selected game, map, and time window. Try another scope or a wider range.'}> + ? report.registeredServerCount > 0 + ? 'Leave the manager running to collect server status and build a history.' + : 'Register a server to begin collecting utilization history.' + : report.servers.length > 0 + ? 'Captured samples have no elapsed time in this window. Time builds between status checks.' + : 'There are no elapsed observations for the selected game, map, and time window. Try another scope or a wider range.'} + > {#snippet actions()} - {#if report.latestSampleAt !== null && report.days < 30}{/if} - {#if report.scope.game || report.scope.map}{/if} - {#if report.registeredServerCount === 0}{/if} + {#if report.latestSampleAt !== null && report.days < 30}{/if} + {#if report.scope.game || report.scope.map}{/if} + {#if report.registeredServerCount === 0}{/if} {/snippet} {:else} {#if report.registeredServerCount === 0} -

    No servers are currently registered. This report shows retained history; new samples resume when servers are registered.

    +

    + No servers are currently registered. This report shows retained history; new samples + resume when servers are registered. +

    {:else if report.latestSampleAt !== null && report.now - report.latestSampleAt > 5 * 60_000} -

    No new collection samples in the last five minutes. This report still includes the retained observations shown below.

    +

    + No new collection samples in the last five minutes. This report still includes the + retained observations shown below. +

    {/if} - {#if report.summary.observed === 0}

    All captured server status is unknown. A dash means occupancy could not be calculated; it does not mean servers were empty.

    {/if} + {#if report.summary.observed === 0}

    + All captured server status is unknown. A dash means occupancy could not be + calculated; it does not mean servers were empty. +

    {/if} {/if} {#if captured > 0} - changeScope({ timeZone: value })} /> - + changeScope({ timeZone: value })} + /> +
    - {#if view === 'insights'}{:else}{/if} + {#if view === 'insights'}{:else}{/if}
    {/if}
    About this report -
    - {#if captured > 0}{metricDuration(report.summary.observed)} known server time{metricDuration(report.summary.unknown)} unknown status{/if} - Scope last collected {report.scopeLatestSampleAt === null ? 'No retained samples' : format.dateTime(report.scopeLatestSampleAt)} - Fleet last collected {report.latestSampleAt === null ? 'No retained samples' : format.dateTime(report.latestSampleAt)} - Report refreshed {format.time(report.now)} · refreshes every minute while visible - {#if report.scopeFirstSampleAt !== null}Scope retained samples from {format.dateTime(report.scopeFirstSampleAt)}{/if} -
    -

    Previous window: {format.period(report.previous.since)} – {format.period(report.previous.until)}. Known time: {metricDuration(report.previous.summary.observed)} previous / {metricDuration(report.summary.observed)} current. Percentage points show the difference between two percentages. Equal windows do not guarantee equal observation coverage.

    -

    Time with players is occupied server time divided by known server time. Known time includes occupied and idle observations; unknown status and collection gaps are excluded from that percentage.

    -

    Each periodic observation covers at most {ONLINE_MAX_MS / 1000} seconds. Longer collection gaps are excluded. History is retained for 31 days. Player-hours include all players reported by the server and are estimates, not individual session records.

    -

    Timeline buckets are aligned to UTC, with labels shown in {report.scope.timeZone.replaceAll('_', ' ')}. Daily buckets are 24-hour periods, not local calendar days. The first and latest buckets can be partial. Percentage bars divide occupied, idle, and unknown time by captured time; the headline occupancy percentage uses known time only.

    -

    Historical server rows are grouped by name and may include multiple registrations using that name. Map groups preserve the recorded game and map label.

    +
    + {#if captured > 0}{metricDuration(report.summary.observed)} known server time{metricDuration(report.summary.unknown)} unknown status{/if} + Scope last collected {report.scopeLatestSampleAt === null + ? 'No retained samples' + : format.dateTime(report.scopeLatestSampleAt)} + Fleet last collected {report.latestSampleAt === null + ? 'No retained samples' + : format.dateTime(report.latestSampleAt)} + Report refreshed {format.time(report.now)} · refreshes every minute while visible + {#if report.scopeFirstSampleAt !== null}Scope retained samples from {format.dateTime(report.scopeFirstSampleAt)}{/if} +
    +

    + Previous window: {format.period(report.previous.since)} – {format.period( + report.previous.until + )}. Known time: {metricDuration(report.previous.summary.observed)} previous / {metricDuration( + report.summary.observed + )} current. Percentage points show the difference between two percentages. Equal windows do + not guarantee equal observation coverage. +

    +

    + Time with players is occupied server time divided by known server time. Known time + includes occupied and idle observations; unknown status and collection gaps are excluded + from that percentage. +

    +

    + Each periodic observation covers at most {ONLINE_MAX_MS / 1000} seconds. Longer collection + gaps are excluded. History is retained for 31 days. Player-hours include all players reported + by the server and are estimates, not individual session records. +

    +

    + Timeline buckets are aligned to UTC, with labels shown in {report.scope.timeZone.replaceAll( + '_', + ' ' + )}. Daily buckets are 24-hour periods, not local calendar days. The first and latest + buckets can be partial. Percentage bars divide occupied, idle, and unknown time by + captured time; the headline occupancy percentage uses known time only. +

    +

    + Historical server rows are grouped by name and may include multiple registrations using + that name. Map groups preserve the recorded game and map label. +

    diff --git a/src/routes/operations/+page.server.ts b/src/routes/operations/+page.server.ts index 444d176..dd3e894 100644 --- a/src/routes/operations/+page.server.ts +++ b/src/routes/operations/+page.server.ts @@ -1,3 +1,5 @@ import { redirect } from '@sveltejs/kit'; -export function load() { redirect(308, '/activity'); } +export function load() { + redirect(308, '/activity'); +} diff --git a/src/routes/plugins/+page.svelte b/src/routes/plugins/+page.svelte index 3f12e3a..c44388b 100644 --- a/src/routes/plugins/+page.svelte +++ b/src/routes/plugins/+page.svelte @@ -32,19 +32,30 @@ currentVersion: { id: number; version: number; release_version: string | null } | null; kind: ArtifactKind; latestVersion: { - id: number; version: number; sha256: string; size: number; + id: number; + version: number; + sha256: string; + size: number; release_version: string | null; - state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string; + state: 'untested' | 'tested' | 'test-failed'; + uploaded_at: string; } | null; versionCount: number; servers: string[]; runningBetaId: number | null; } - interface ScopeOptions { games: string[]; mapsByGame: Record; } + interface ScopeOptions { + games: string[]; + mapsByGame: Record; + } let addOpen = $state(false); let githubBusy = $state(false); - let importComplete = $state<{ pluginId: number; betaUrl: string; alreadyImported: boolean } | null>(null); + let importComplete = $state<{ + pluginId: number; + betaUrl: string; + alreadyImported: boolean; + } | null>(null); let method = $state('upload'); let updatesOpen = $state(false); let updates = $state([]); @@ -54,20 +65,59 @@ let sort = $state('name'); let limit = $state(25); let mounted = $state(false); - afterNavigate(() => { mounted = true; }); + afterNavigate(() => { + mounted = true; + }); let lastFilters = ''; - $effect(() => { const key = JSON.stringify([search, game, status, sort]); if (!mounted) return; if (lastFilters && lastFilters !== key) limit = 25; lastFilters = key; }); - $effect(() => { if (mounted) saveLibraryContext({ q: search, game, status, sort, limit }, { sort: 'name', limit: 25 }); }); - const visible = $derived.by(() => plugins.filter(a => (a.name + ' ' + a.target_rel_path).toLowerCase().includes(search.toLowerCase()) - && (!game || !a.scope_type || (a.scope_type === 'game' ? a.scope_value === game : (scopeOptions.mapsByGame[game] ?? []).includes(a.scope_value ?? ''))) - && (!status || stateLabel(a) === status)).sort((a,b) => sort === 'recent' ? (b.latestVersion?.uploaded_at ?? '').localeCompare(a.latestVersion?.uploaded_at ?? '') : a.name.localeCompare(b.name))); + $effect(() => { + const key = JSON.stringify([search, game, status, sort]); + if (!mounted) return; + if (lastFilters && lastFilters !== key) limit = 25; + lastFilters = key; + }); + $effect(() => { + if (mounted) + saveLibraryContext( + { q: search, game, status, sort, limit }, + { sort: 'name', limit: 25 } + ); + }); + const visible = $derived.by(() => + plugins + .filter( + (a) => + (a.name + ' ' + a.target_rel_path) + .toLowerCase() + .includes(search.toLowerCase()) && + (!game || + !a.scope_type || + (a.scope_type === 'game' + ? a.scope_value === game + : (scopeOptions.mapsByGame[game] ?? []).includes( + a.scope_value ?? '' + ))) && + (!status || stateLabel(a) === status) + ) + .sort((a, b) => + sort === 'recent' + ? (b.latestVersion?.uploaded_at ?? '').localeCompare( + a.latestVersion?.uploaded_at ?? '' + ) + : a.name.localeCompare(b.name) + ) + ); let plugins = $state([]); let importCount = $state(0); let loading = $state(true); let flash = $state(null); let uploading = $state(false); let scopeOptions = $state({ games: [], mapsByGame: {} }); - let successBanner = $state<{ name: string; artifactId: number; version: number; release_version: string | null } | null>(null); + let successBanner = $state<{ + name: string; + artifactId: number; + version: number; + release_version: string | null; + } | null>(null); let newName = $state(''); let newReleaseVersion = $state(''); @@ -90,13 +140,18 @@ fetch('/api/scope-options') ]); if (!pluginRes.ok) { - throw new Error('Could not refresh plugins. Your last loaded results are still shown.'); + throw new Error( + 'Could not refresh plugins. Your last loaded results are still shown.' + ); } else { plugins = ((await pluginRes.json()) as { scripts?: ArtifactRow[] }).scripts ?? []; } flash = null; if (scopeRes.ok) { - scopeOptions = ((await scopeRes.json()) as ScopeOptions | undefined) ?? { games: [], mapsByGame: {} }; + scopeOptions = ((await scopeRes.json()) as ScopeOptions | undefined) ?? { + games: [], + mapsByGame: {} + }; } } catch (err) { flash = `Failed to load plugins: ${(err as Error).message}`; @@ -104,7 +159,15 @@ loading = false; } } - onMount(() => { const params = new URLSearchParams(window.location.search); search = params.get('q') ?? ''; game = params.get('game') ?? ''; status = params.get('status') ?? ''; sort = params.get('sort') === 'recent' ? 'recent' : 'name'; limit = libraryLimit(params); void load(); }); + onMount(() => { + const params = new URLSearchParams(window.location.search); + search = params.get('q') ?? ''; + game = params.get('game') ?? ''; + status = params.get('status') ?? ''; + sort = params.get('sort') === 'recent' ? 'recent' : 'name'; + limit = libraryLimit(params); + void load(); + }); function pickFile(e: Event): void { const t = e.target as HTMLInputElement; @@ -115,8 +178,12 @@ async function createPlugin(e: Event): Promise { e.preventDefault(); if (uploading || !newFile || !newName.trim()) return; - if (newScopeType !== 'none' && !scopeValue) { flash = `Pick a ${newScopeType}`; return; } - uploading = true; flash = null; + if (newScopeType !== 'none' && !scopeValue) { + flash = `Pick a ${newScopeType}`; + return; + } + uploading = true; + flash = null; const form = new FormData(); form.append('name', newName.trim()); form.append('kind', 'plugin'); @@ -127,23 +194,48 @@ form.append('content', newFile); try { const res = await fetch('/api/plugins', { method: 'POST', body: form }); - const body = (await res.json()) as { ok: boolean; error?: string; script?: { id: number; name: string } }; - if (!res.ok || !body.ok) { flash = body.error ?? `HTTP ${res.status}`; return; } + const body = (await res.json()) as { + ok: boolean; + error?: string; + script?: { id: number; name: string }; + }; + if (!res.ok || !body.ok) { + flash = body.error ?? `HTTP ${res.status}`; + return; + } successBanner = body.script - ? { name: body.script.name, artifactId: body.script.id, version: 1, release_version: newReleaseVersion.trim() || null } + ? { + name: body.script.name, + artifactId: body.script.id, + version: 1, + release_version: newReleaseVersion.trim() || null + } : null; - newName = ''; newScopeType = 'none'; newScopeGame = ''; newScopeMap = ''; newNotes = ''; newFile = null; + newName = ''; + newScopeType = 'none'; + newScopeGame = ''; + newScopeMap = ''; + newNotes = ''; + newFile = null; newReleaseVersion = ''; addOpen = false; await load(); - } catch (error) { flash = (error as Error).message; } finally { + } catch (error) { + flash = (error as Error).message; + } finally { uploading = false; } } function scopeLabel(a: ArtifactRow): string { if (!a.scope_type) return 'unscoped'; - return a.scope_type === 'map' ? mapLabel(a.scope_value ?? '') : a.scope_value === 't5' ? 'Black Ops I' : a.scope_value === 't6' ? 'Black Ops II' : a.scope_value ?? 'Unscoped'; + return a.scope_type === 'map' + ? mapLabel(a.scope_value ?? '') + : a.scope_value === 't5' + ? 'Black Ops I' + : a.scope_value === 't6' + ? 'Black Ops II' + : (a.scope_value ?? 'Unscoped'); } function stateLabel(a: ArtifactRow): string { if (a.runningBetaId) return 'testing'; @@ -163,115 +255,305 @@ let scopeTypeOptions = [ { value: 'none', label: 'Unscoped' }, { value: 'game', label: 'Game' }, - { value: 'map', label: 'Map' } + { value: 'map', label: 'Map' } ]; Plugins · Electric Cherry
    - - {#snippet actions()}{#if plugins.length}{/if}{#if loading || plugins.length || flash}{/if}{/snippet} + + {#snippet actions()}{#if plugins.length}{/if}{#if loading || plugins.length || flash}{/if}{/snippet} - {#if importComplete}{/if} - {#if successBanner} - - {/if} + {#if importComplete}{/if} + {#if successBanner} + + {/if} - {#if flash && !addOpen}{/if} + {#if flash && !addOpen}{/if} {#if loading || plugins.length || flash}
    - - - - -
    {/if} - {#if game}

    Game results include unscoped plugins that can use explicitly selected beta servers.

    {/if} - {#if updatesOpen}
    { if (!event.currentTarget.open) updatesOpen = false; }}>Update check details{#key importCount} updates = results} />{/key}
    {/if} + + + + +
    {/if} + {#if game}

    + Game results include unscoped plugins that can use explicitly selected beta servers. +

    {/if} + {#if updatesOpen}
    { + if (!event.currentTarget.open) updatesOpen = false; + }} + > + Update check details{#key importCount} (updates = results)} + />{/key} +
    {/if}
    -
    {visible.length} of {plugins.length} plugins{#if visible.length && (search || game || status)}{/if}
    +
    + {visible.length} of {plugins.length} plugins{#if visible.length && (search || game || status)}{/if} +
    {#if loading}
    Loading…
    {:else if visible.length === 0 && !flash} - - {#snippet actions()}{#if plugins.length}{:else}{/if}{/snippet} + + {#snippet actions()}{#if plugins.length}{:else}{/if}{/snippet} {:else} -
    PlayerParticipation{stat.label}Last seen
    PlayerParticipation{stat.label}Last seen
    - {#if player.canViewProfile}{player.name}{:else}{player.name}{/if} + {#if player.canViewProfile}{player.name}{:else}{player.name}{/if} {player.guid} {player.presentAtEnd ? 'At match end' : 'Not at match end'}{#if !player.hasStats}No stats recorded{/if}{player[stat.key] === null ? '—' : number.format(player[stat.key]!)}{player.presentAtEnd + ? 'At match end' + : 'Not at match end'}{#if !player.hasStats}No stats recorded{/if}{player[stat.key] === null + ? '—' + : number.format(player[stat.key]!)}
    - - {#each visible.slice(0,limit) as a (a.id)} - {@const update = updates.find(update => update.pluginId === a.id && update.status === 'available')} - - - - - - - - {/each} -
    PluginGame / scopeVersion / statusEligible serversNext action
    {a.name}{#if a.target_rel_path !== `${a.name}.dll`}{a.target_rel_path}{/if} - {#if update}{update.release?.tag} available{/if}{scopeLabel(a)}{a.scope_type ? a.servers.length : 'Explicit targets'}{#if update}Review update - {:else if a.runningBetaId !== null}View beta - {:else if a.latestVersion && a.latestVersion.id !== a.current_version_id}Review version{/if}
    +
    + + + {#each visible.slice(0, limit) as a (a.id)} + {@const update = updates.find( + (update) => + update.pluginId === a.id && update.status === 'available' + )} + + + + + + + + {/each} +
    PluginGame / scopeVersion / statusEligible serversNext action
    {a.name}{#if a.target_rel_path !== `${a.name}.dll`}{a.target_rel_path}{/if} + {#if update}{update.release?.tag} available{/if}{scopeLabel(a)}{a.scope_type ? a.servers.length : 'Explicit targets'}{#if update}Review update + {:else if a.runningBetaId !== null}View beta + {:else if a.latestVersion && a.latestVersion.id !== a.current_version_id}Review version{/if}
    +
    {/if}
    - +
    - {#if method === 'github'} { importComplete = result; updates = updates.filter(update => update.pluginId !== result.pluginId); importCount++; addOpen = false; await load(); }} /> - {:else}
    - - - - - - - - - - -
    - newScopeMap = ''} options={gameOptions} disabled={uploading} /> - {/if} - {#if newScopeType === 'map'} - - -
    - - {#if flash}{flash}{/if} -
    - -{/if} + + + + + + +
    + (newScopeMap = '')} + options={gameOptions} + disabled={uploading} + /> + {/if} + {#if newScopeType === 'map'} + + +
    + + {#if flash}{flash}{/if} +
    + + {/if}
    - diff --git a/src/routes/provisioning/+page.svelte b/src/routes/provisioning/+page.svelte index 9eb4e7a..7619a7c 100644 --- a/src/routes/provisioning/+page.svelte +++ b/src/routes/provisioning/+page.svelte @@ -16,7 +16,18 @@ import RelativeTime from '$ui/components/RelativeTime.svelte'; import LifecycleDialog from '$ui/components/LifecycleDialog.svelte'; - import { provisioningListStatuses, provisioningListSources, provisioningSourceLabels, provisioningSourceLabel, provisioningMatchesSource, provisioningPageSize, provisioningPageOffset, type ProvisioningListSource, type ProvisioningListRow as Deployment, type ProvisioningListPage } from '$lib/provisioning-list.js'; + import { + provisioningListStatuses, + provisioningListSources, + provisioningSourceLabels, + provisioningSourceLabel, + provisioningMatchesSource, + provisioningPageSize, + provisioningPageOffset, + type ProvisioningListSource, + type ProvisioningListRow as Deployment, + type ProvisioningListPage + } from '$lib/provisioning-list.js'; import { provisioningActions, actionLabels } from '$lib/provisioning.js'; import { contextHref } from '$lib/navigation-context.js'; @@ -52,10 +63,16 @@ let cancellationNotice = $state(''); $effect(() => { const params = $page.url.searchParams; - tab = ['active', 'history'].includes(params.get('view') ?? '') ? params.get('view')! : 'active'; - action = provisioningActions.some(value => value === params.get('action')) ? params.get('action')! : 'all'; - historyState = provisioningListStatuses.some(value => value === params.get('status')) ? params.get('status')! : 'all'; - source = provisioningListSources.find(value => value === params.get('source')) ?? 'all'; + tab = ['active', 'history'].includes(params.get('view') ?? '') + ? params.get('view')! + : 'active'; + action = provisioningActions.some((value) => value === params.get('action')) + ? params.get('action')! + : 'all'; + historyState = provisioningListStatuses.some((value) => value === params.get('status')) + ? params.get('status')! + : 'all'; + source = provisioningListSources.find((value) => value === params.get('source')) ?? 'all'; search = params.get('server') ?? ''; limit = provisioningPageSize(Number(params.get('limit') ?? 25)); offset = provisioningPageOffset(Number(params.get('offset') ?? 0)); @@ -63,26 +80,56 @@ function saveContext(reset = false) { if (reset) offset = 0; const url = new URL($page.url); - for (const [key, value] of Object.entries({ view: tab === 'active' ? '' : tab, action: action === 'all' ? '' : action, status: historyState === 'all' ? '' : historyState, source: source === 'all' ? '' : source, server: search, limit: limit !== 25 ? String(limit) : '', offset: offset ? String(offset) : '' })) { - if (value) url.searchParams.set(key, value); else url.searchParams.delete(key); + for (const [key, value] of Object.entries({ + view: tab === 'active' ? '' : tab, + action: action === 'all' ? '' : action, + status: historyState === 'all' ? '' : historyState, + source: source === 'all' ? '' : source, + server: search, + limit: limit !== 25 ? String(limit) : '', + offset: offset ? String(offset) : '' + })) { + if (value) url.searchParams.set(key, value); + else url.searchParams.delete(key); } void goto(url, { replaceState: true, noScroll: true, keepFocus: true }); } async function cancelRequests(rows: Deployment[], bulk = false) { if (cancelling || !rows.length) return; - const ids = rows.map(row => row.id); - if (!await confirmAction({ title: bulk ? `Cancel all ${ids.length} active operations?` : 'Cancel operation?', - message: `${bulk ? `This affects all ${ids.length} cancellable active operations across the fleet, regardless of the current filters.` : `Cancel ${actionLabels[rows[0].action].toLowerCase()} on ${serverTitle(rows[0].server)}.`} Cancelling stops remaining work and health checks. Changes already made are not undone; commands already delivered to a host may still finish.`, - confirmLabel: bulk ? 'Cancel all operations' : 'Cancel operation', danger: true })) return; - cancelling = true; cancellationNotice = ''; let cancelled = 0; const unavailable: string[] = []; + const ids = rows.map((row) => row.id); + if ( + !(await confirmAction({ + title: bulk ? `Cancel all ${ids.length} active operations?` : 'Cancel operation?', + message: `${bulk ? `This affects all ${ids.length} cancellable active operations across the fleet, regardless of the current filters.` : `Cancel ${actionLabels[rows[0].action].toLowerCase()} on ${serverTitle(rows[0].server)}.`} Cancelling stops remaining work and health checks. Changes already made are not undone; commands already delivered to a host may still finish.`, + confirmLabel: bulk ? 'Cancel all operations' : 'Cancel operation', + danger: true + })) + ) + return; + cancelling = true; + cancellationNotice = ''; + let cancelled = 0; + const unavailable: string[] = []; for (const id of ids) { - try { const response = await fetch(`/api/provisioning/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); if (response.ok) cancelled++; else unavailable.push(id); } - catch { unavailable.push(id); } + try { + const response = await fetch(`/api/provisioning/${encodeURIComponent(id)}/cancel`, { + method: 'POST' + }); + if (response.ok) cancelled++; + else unavailable.push(id); + } catch { + unavailable.push(id); + } } cancellationNotice = `${cancelled} operation${cancelled === 1 ? '' : 's'} cancelled.${unavailable.length ? ` ${unavailable.length} could not be cancelled; refresh to see their current state.` : ''}`; - cancelling = false; await load(); historyRefresh++; + cancelling = false; + await load(); + historyRefresh++; } - const tabs = [{id:'active',label:'Active'},{id:'history',label:'History'}]; + const tabs = [ + { id: 'active', label: 'Active' }, + { id: 'history', label: 'History' } + ]; let action = $state('all'); let search = $state(''); let refreshing = $state(false); @@ -94,167 +141,623 @@ if (!res.ok) throw new Error('Could not load provisioning activity.'); const result: ProvisioningListPage = await res.json(); if (signal?.aborted) return; - active = result.deployments; now = Date.now(); updatedAt = now; + active = result.deployments; + now = Date.now(); + updatedAt = now; needsReview = result.needsReview; error = ''; - } catch (err) { if (!signal?.aborted) error = (err as Error).message; return false; } - finally { loading = false; refreshing = false; } + } catch (err) { + if (!signal?.aborted) error = (err as Error).message; + return false; + } finally { + loading = false; + refreshing = false; + } } // History has its own visible-page poll; filter changes discard incompatible // rows, while refreshes keep the current page available until replacement. $effect(() => { if (tab !== 'history') return; - const params = new URLSearchParams({ view: 'history', action, status: historyState, source, - server: search, limit: String(limit), offset: String(offset) }); + const params = new URLSearchParams({ + view: 'history', + action, + status: historyState, + source, + server: search, + limit: String(limit), + offset: String(offset) + }); void historyRefresh; return untrack(() => { if (previousHistoryQuery !== params.toString()) { - history = []; historyTotal = 0; historyUpdatedAt = null; historyError = ''; + history = []; + historyTotal = 0; + historyUpdatedAt = null; + historyError = ''; previousHistoryQuery = params.toString(); } let disposed = false; let first = true; - const stop = pollVisible(async signal => { + const stop = pollVisible(async (signal) => { // A new record can shift this page. Keep keyboard focus stable // until it leaves the results; an explicit Retry still runs. if (!first && historyResults?.contains(document.activeElement)) return; first = false; historyLoading = true; try { - const response = await fetch(`/api/provisioning?${params}`, { signal, cache: 'no-store' }); + const response = await fetch(`/api/provisioning?${params}`, { + signal, + cache: 'no-store' + }); if (!response.ok) throw new Error('Could not load provisioning history.'); const result: ProvisioningListPage = await response.json(); if (!signal.aborted) { - history = result.deployments; historyTotal = result.total; - historyUpdatedAt = Date.now(); now = Date.now(); historyError = ''; + history = result.deployments; + historyTotal = result.total; + historyUpdatedAt = Date.now(); + now = Date.now(); + historyError = ''; } - } catch (err) { if (!signal.aborted) historyError = (err as Error).message; return false; } - finally { if (!disposed) historyLoading = false; } + } catch (err) { + if (!signal.aborted) historyError = (err as Error).message; + return false; + } finally { + if (!disposed) historyLoading = false; + } }, 15000); - return () => { disposed = true; stop(); }; + return () => { + disposed = true; + stop(); + }; }); }); - onMount(() => { const stop = pollVisible(load, 5000); const clock = setInterval(() => now = Date.now(), 30000); return () => { stop(); clearInterval(clock); }; }); - function fmt(value: string) { return $timeFormat.dateTime(value); } - const filtered = $derived(action !== 'all' || source !== 'all' || historyState !== 'all' || Boolean(search)); - const visibleActive = $derived(active.filter(row => { - if (action !== 'all' && row.action !== action) return false; - if (historyState !== 'all' && (historyState === 'pending' ? !['queued','waiting'].includes(row.state) : row.state !== historyState)) return false; - if (search && !`${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())) return false; - if (source !== 'all' && !provisioningMatchesSource(row, source)) return false; - return true; - })); + onMount(() => { + const stop = pollVisible(load, 5000); + const clock = setInterval(() => (now = Date.now()), 30000); + return () => { + stop(); + clearInterval(clock); + }; + }); + function fmt(value: string) { + return $timeFormat.dateTime(value); + } + const filtered = $derived( + action !== 'all' || source !== 'all' || historyState !== 'all' || Boolean(search) + ); + const visibleActive = $derived( + active.filter((row) => { + if (action !== 'all' && row.action !== action) return false; + if ( + historyState !== 'all' && + (historyState === 'pending' + ? !['queued', 'waiting'].includes(row.state) + : row.state !== historyState) + ) + return false; + if ( + search && + !`${row.server} ${serverTitle(row.server)}` + .toLowerCase() + .includes(search.toLowerCase()) + ) + return false; + if (source !== 'all' && !provisioningMatchesSource(row, source)) return false; + return true; + }) + ); const queueSummary = $derived.by(() => { const groups = { queued: 0, waiting: 0, verifying: 0, running: 0 }; for (const row of visibleActive) { const group = provisioningQueueStatus(row).group; if (group in groups) groups[group as keyof typeof groups]++; } - return Object.entries(groups).filter(([, count]) => count).map(([label, count]) => `${count} ${label === 'verifying' ? 'verifying health' : label === 'running' ? 'in progress' : label}`).join(' · '); + return Object.entries(groups) + .filter(([, count]) => count) + .map( + ([label, count]) => + `${count} ${label === 'verifying' ? 'verifying health' : label === 'running' ? 'in progress' : label}` + ) + .join(' · '); }); - function clearFilters() { action = 'all'; source = 'all'; historyState = 'all'; search = ''; saveContext(true); } - function reviewAttention() { tab = 'history'; action = 'all'; source = 'all'; search = ''; historyState = 'attention'; saveContext(true); } + function clearFilters() { + action = 'all'; + source = 'all'; + historyState = 'all'; + search = ''; + saveContext(true); + } + function reviewAttention() { + tab = 'history'; + action = 'all'; + source = 'all'; + search = ''; + historyState = 'attention'; + saveContext(true); + } Provisioning | Electric Cherry
    {#snippet actions()} - - + + {/snippet} {#if $page.url.searchParams.has('returnTo')}{/if}
    -
    saveContext(true)} ariaLabel="Provisioning views" idPrefix="deployment-tabs" panelId="deployment-panel" />
    - - - - +
    + saveContext(true)} + ariaLabel="Provisioning views" + idPrefix="deployment-tabs" + panelId="deployment-panel" + /> +
    + + + + {#if filtered}{/if}
    - { fleetNotice = `${actionLabels[action]} queued for ${count} server${count === 1 ? '' : 's'}.`; tab = 'active'; clearFilters(); void load(); }} /> + { + fleetNotice = `${actionLabels[action]} queued for ${count} server${count === 1 ? '' : 's'}.`; + tab = 'active'; + clearFilters(); + void load(); + }} + /> {#if fleetNotice}

    {fleetNotice}

    {/if} {#if cancellationNotice}

    {cancellationNotice}

    {/if} -
    - {#if error && tab === 'active'}{/if} +
    + {#if error && tab === 'active'}{/if}
    - {#if tab === 'active'}

    {filtered ? `${visibleActive.length} of ${active.length}` : active.length} active{#if queueSummary} · {queueSummary}{/if}

    + {#if tab === 'active'}

    + {filtered ? `${visibleActive.length} of ${active.length}` : active.length} active{#if queueSummary} · {queueSummary}{/if} +

    {:else}

    History {historyTotal}

    {/if}
    - {#if tab === 'active' && active.some(row => row.canCancel)}{/if} - {#if needsReview}{/if} - {#if tab === 'active'} void load()} retryDisabled={refreshing} /> - {:else} historyRefresh++} retryDisabled={historyLoading} />{/if} + {#if tab === 'active' && active.some((row) => row.canCancel)}{/if} + {#if needsReview}{/if} + {#if tab === 'active'} void load()} + retryDisabled={refreshing} + /> + {:else} historyRefresh++} + retryDisabled={historyLoading} + />{/if}
    {#if tab === 'active'} {#if loading}

    Loading provisioning activity…

    - {:else if !visibleActive.length} + {:else if !visibleActive.length} {:else}{@render operationTable(visibleActive)}{/if} {:else}
    - {#if historyError}{/if} - {#if historyLoading && !historyUpdatedAt}

    Loading provisioning history…

    - {:else if !historyError && !history.length} - {:else}{@render operationTable(history)}{/if} - + {#if historyError}{/if} + {#if historyLoading && !historyUpdatedAt}

    + Loading provisioning history… +

    + {:else if !historyError && !history.length} + {:else}{@render operationTable(history)}{/if} +
    {/if}
    {#snippet operationTable(rows: Deployment[])} -
    - - {#each rows as row (row.id)} - {@const status = provisioningQueueStatus(row)} - {@const primarySource = row.sources?.[0]} - {@const operationHref = contextHref(`/provisioning/${encodeURIComponent(row.id)}`, $page.url)} - {@const isExpanded = expanded[row.id] ?? false} - - - - - - - - - - - {/each} -
    ServerOperationStatusSourceRequesterStartedDetails
    {actionLabels[row.action]}{status.label}{#if ['queued','waiting'].includes(status.group) && row.phaseStartedAt && now - Date.parse(row.phaseStartedAt) >= 600000}In this stage {elapsedTime(row.phaseStartedAt, now)}{/if}{#if primarySource?.href}{provisioningSourceLabel(row)}{:else}{provisioningSourceLabel(row)}{/if}{#if (row.sources?.length ?? 0) > 1}+{row.sources!.length - 1} more{/if}{provisioningRequester(row)}
    +
    + + + {#each rows as row (row.id)} + {@const status = provisioningQueueStatus(row)} + {@const primarySource = row.sources?.[0]} + {@const operationHref = contextHref( + `/provisioning/${encodeURIComponent(row.id)}`, + $page.url + )} + {@const isExpanded = expanded[row.id] ?? false} + + + + + + + + + + + {/each} +
    ServerOperationStatusSourceRequesterStartedDetails
    {actionLabels[row.action]}{status.label}{#if ['queued', 'waiting'].includes(status.group) && row.phaseStartedAt && now - Date.parse(row.phaseStartedAt) >= 600000}In this stage {elapsedTime(row.phaseStartedAt, now)}{/if}{#if primarySource?.href}{provisioningSourceLabel(row)}{:else}{provisioningSourceLabel(row)}{/if}{#if (row.sources?.length ?? 0) > 1}+{row.sources!.length - 1} more{/if}{provisioningRequester(row)}
    +
    {/snippet} diff --git a/src/routes/provisioning/[id]/+page.svelte b/src/routes/provisioning/[id]/+page.svelte index 11f51ca..476f80c 100644 --- a/src/routes/provisioning/[id]/+page.svelte +++ b/src/routes/provisioning/[id]/+page.svelte @@ -7,9 +7,18 @@ import { actionLabels } from '$lib/provisioning.js'; import { pollVisible } from '$ui/poll.js'; import { serverTitle } from '$ui/server-labels.js'; - import { createProvisioningRefresh, type ProvisioningRefreshState } from '$ui/provisioning-refresh.js'; + import { + createProvisioningRefresh, + type ProvisioningRefreshState + } from '$ui/provisioning-refresh.js'; import { confirmAction } from '$ui/confirm.js'; - import { elapsedTime, isProvisioningDetail, provisioningNeedsRefresh, provisioningOutcome, provisioningRequester } from '$ui/provisioning-presentation.js'; + import { + elapsedTime, + isProvisioningDetail, + provisioningNeedsRefresh, + provisioningOutcome, + provisioningRequester + } from '$ui/provisioning-presentation.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte'; import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; @@ -28,12 +37,23 @@ let { data }: { data: PageData } = $props(); const initial = untrack(() => data.detail); let view = $state>({ - id: initial.deployment.id, detail: initial, refreshing: false, busy: null, - loadError: '', actionError: '', notice: '', refreshedAt: initial.generatedAt + id: initial.deployment.id, + detail: initial, + refreshing: false, + busy: null, + loadError: '', + actionError: '', + notice: '', + refreshedAt: initial.generatedAt }); let now = $state(Date.parse(initial.generatedAt)); let copyStatus = $state(''); - const controller = createProvisioningRefresh({ validate: isProvisioningDetail, onChange: value => { view = value; } }); + const controller = createProvisioningRefresh({ + validate: isProvisioningDetail, + onChange: (value) => { + view = value; + } + }); const detail = $derived(view.detail); const deployment = $derived(detail?.deployment); const startedAt = $derived(timestampISO(deployment?.started_at)); @@ -45,10 +65,24 @@ const outcome = $derived(detail ? provisioningOutcome(detail) : null); const active = $derived(outcome?.active ?? false); const needsRefresh = $derived(detail ? provisioningNeedsRefresh(detail) : false); - const duration = $derived(finishedAt || active ? elapsedTime(startedAt, finishedAt ? Date.parse(finishedAt) : now) : 'Not recorded'); - const serverHref = $derived(deployment && deployment.server_exists && !deployment.server_deleted_at ? `/servers/${encodeURIComponent(deployment.server)}` : null); + const duration = $derived( + finishedAt || active + ? elapsedTime(startedAt, finishedAt ? Date.parse(finishedAt) : now) + : 'Not recorded' + ); + const serverHref = $derived( + deployment && deployment.server_exists && !deployment.server_deleted_at + ? `/servers/${encodeURIComponent(deployment.server)}` + : null + ); const recipeStatus = $derived(detail?.progress?.recipeStatus ?? null); - const recipeSelection = $derived(recipeStatus === 'queued' ? 'Captured when queued' : recipeStatus === 'attempted' ? 'Selected when creation began' : 'Recorded setup; selection time unknown'); + const recipeSelection = $derived( + recipeStatus === 'queued' + ? 'Captured when queued' + : recipeStatus === 'attempted' + ? 'Selected when creation began' + : 'Recorded setup; selection time unknown' + ); $effect(() => { const snapshot = data.detail; untrack(() => { @@ -59,7 +93,14 @@ }); $effect(() => { if (!needsRefresh) return; - return pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false); + return pollVisible( + () => { + now = Date.now(); + return controller.refresh(); + }, + 5000, + false + ); }); onDestroy(() => controller.dispose()); async function retryWithRebuild() { @@ -68,23 +109,33 @@ const confirmed = await confirmAction({ title: 'Retry creation with a rebuild?', message: `Rebuild ${serverTitle(deployment.server)} (${deployment.server}) using the latest promoted setup. This keeps the same lobby and port, deletes its container and all instance files, and starts a new provisioning operation.\n\nPlayer safety, capacity limits, and protected work checks still apply. The original failure remains in this operation’s history.`, - confirmLabel: 'Retry with rebuild', danger: true + confirmLabel: 'Retry with rebuild', + danger: true }); - if (confirmed && view.id === id && detail?.canRetryWithRebuild) await controller.act('retry-rebuild', - 'Rebuild retry requested. Follow the linked recovery operation below for progress.'); + if (confirmed && view.id === id && detail?.canRetryWithRebuild) + await controller.act( + 'retry-rebuild', + 'Rebuild retry requested. Follow the linked recovery operation below for progress.' + ); } async function overrideSafety() { - if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return; + if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) + return; const id = deployment.id; const action = actionLabels[deployment.action]; const destructive = ['rebuild', 'destroy'].includes(deployment.action); const confirmed = await confirmAction({ title: `Admin override: ${action.toLowerCase()} server?`, message: `${action} ${serverTitle(deployment.server)} (${deployment.server}) without waiting for a fresh player count or an empty server. This can disconnect active players and interrupt their game.${destructive ? '\n\nThis action deletes the container and all instance files.' : ''}\n\nThis override applies only to this operation and records your administrator identity. Capacity, operation conflicts, file validation, and post-action verification still apply.`, - confirmLabel: `Override and ${action.toLowerCase()}`, danger: true + confirmLabel: `Override and ${action.toLowerCase()}`, + danger: true }); - if (confirmed && view.id === id && canOverride) await controller.act('override', - 'Admin override enabled for this operation. Provisioning will proceed without player safety checks when the remaining requirements are met.', { confirmed: true }); + if (confirmed && view.id === id && canOverride) + await controller.act( + 'override', + 'Admin override enabled for this operation. Provisioning will proceed without player safety checks when the remaining requirements are met.', + { confirmed: true } + ); } async function cancelOperation() { if (!deployment || !detail?.canCancel || view.busy) return; @@ -92,10 +143,14 @@ const confirmed = await confirmAction({ title: 'Cancel operation?', message: `Stop remaining work and health checks for ${serverTitle(deployment.server)} (${deployment.server}). Changes already made are not undone. Commands already sent to the host may still finish.`, - confirmLabel: 'Cancel operation', danger: true + confirmLabel: 'Cancel operation', + danger: true }); - if (confirmed && view.id === id && detail?.canCancel) await controller.act('cancel', - 'Operation cancelled. Already-submitted commands may finish; no further steps will be started.'); + if (confirmed && view.id === id && detail?.canCancel) + await controller.act( + 'cancel', + 'Operation cancelled. Already-submitted commands may finish; no further steps will be started.' + ); } async function cancelCreation() { if (!deployment || !detail?.canCancelCreation || view.busy) return; @@ -103,82 +158,271 @@ const confirmed = await confirmAction({ title: 'Cancel creation and remove server?', message: `Stop this launch and delete the container and instance files for ${serverTitle(deployment.server)} (${deployment.server}), even if player status is unknown or players have joined. Shared game files are kept. The key returns to its game pool after removal is confirmed.`, - confirmLabel: 'Cancel creation and remove server', danger: true + confirmLabel: 'Cancel creation and remove server', + danger: true }); - if (confirmed && view.id === id && detail?.canCancelCreation) await controller.act('cancel-creation', - 'Creation cancelled. Follow cleanup below until removal is confirmed.'); + if (confirmed && view.id === id && detail?.canCancelCreation) + await controller.act( + 'cancel-creation', + 'Creation cancelled. Follow cleanup below until removal is confirmed.' + ); } async function copyLink() { copyStatus = ''; try { const url = new URL($page.url); - url.search = ''; url.hash = ''; + url.search = ''; + url.hash = ''; await navigator.clipboard.writeText(url.toString()); copyStatus = 'Operation link copied.'; - } catch { copyStatus = 'Copy unavailable. Copy this page’s address from your browser.'; } + } catch { + copyStatus = 'Copy unavailable. Copy this page’s address from your browser.'; + } } {#snippet dateTime(value: Timestamp)} {@const iso = timestampISO(value)} - {#if iso}{:else}Not recorded{/if} + {#if iso}{:else}Not recorded{/if} {/snippet} -{deployment ? `${actionLabels[deployment.action]} ${deployment.server}` : 'Provisioning'} | Electric Cherry +{deployment ? `${actionLabels[deployment.action]} ${deployment.server}` : 'Provisioning'} | Electric + Cherry
    - - {#snippet crumbs()}{/snippet} - {#snippet actions()}{#if serverHref}{/if}{/snippet} + + {#snippet crumbs()}{/snippet} + {#snippet actions()}{#if serverHref}{/if}{/snippet} {#if deployment && outcome}

    Requested by {provisioningRequester(deployment)}{' · '} - {#if startedAt}{:else}Start: Not recorded{/if}{' · '} - {finishedAt ? `took ${duration}` : active ? `${duration} elapsed` : 'Duration: Not recorded'} + {#if startedAt}{:else}Start: Not recorded{/if}{' · '} + {finishedAt + ? `took ${duration}` + : active + ? `${duration} elapsed` + : 'Duration: Not recorded'}

    {/if}
    {#if active} - void controller.forceRefresh()} retryDisabled={view.refreshing || !!view.busy} /> + void controller.forceRefresh()} + retryDisabled={view.refreshing || !!view.busy} + /> {/if} - {#if view.loadError}{/if} + {#if view.loadError}{/if} {#if deployment && detail && outcome} -
    -
    {#if deployment.server_deleted_at}Deleted identity{/if}{#if detail.progress?.recovery?.state === 'held'}Recovery capacity held{/if}
    -
    -

    {outcome.headline}

    {#if outcome.description}

    {outcome.description}

    {/if} - {#if outcome.checksSummary}

    {' '}{outcome.checksSummary}

    {/if} +
    +
    + {#if deployment.server_deleted_at}Deleted identity{/if}{#if detail.progress?.recovery?.state === 'held'}Recovery capacity held{/if}
    - {#if deployment.error}{/if} - {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}

    Last failure event · {@render dateTime(detail.activity.failureExcerpt.created_at)}{detail.activity.failureExcerpt.message}

    {/if} - {#if detail.warningMessages.length}

    Recorded warnings

      {#each detail.warningMessages as warning}
    • {warning}
      • {/each}
    {/if} +
    +

    {outcome.headline}

    + {#if outcome.description}

    {outcome.description}

    {/if} + {#if outcome.checksSummary}

    + {' '}{outcome.checksSummary} +

    {/if} +
    + {#if deployment.error}{/if} + {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}

    + Last failure event · {@render dateTime( + detail.activity.failureExcerpt.created_at + )}{detail.activity.failureExcerpt.message} +

    {/if} + {#if detail.warningMessages.length}
    +

    Recorded warnings

    +
      + {#each detail.warningMessages as warning}
    • {warning}
      • {/each} +
    +
    {/if} - {#if detail.safetyOverride}

    Admin override enabled

    Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · {@render dateTime(detail.safetyOverride.confirmedAt)}.

    {/if} + {#if detail.safetyOverride}
    +

    Admin override enabled

    +

    + Player safety checks bypassed for this operation. Confirmed by {detail + .safetyOverride.actorName} · {@render dateTime( + detail.safetyOverride.confirmedAt + )}. +

    +
    {/if} {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
    - {#if canOverride}
    {#if detail.safetyOverrideUnavailableReason}

    {detail.safetyOverrideUnavailableReason}

    {/if}
    {/if} - {#if detail.canResumeVerification}{/if} - {#if detail.canRetryWithRebuild}{/if} - {#if detail.canResumeRebuild}{/if} - {#if detail.canResumeCleanup}{/if} - {#if detail.canCancel}{/if} - {#if detail.canCancelCreation}{/if} + {#if canOverride}
    + {#if detail.safetyOverrideUnavailableReason}

    + {detail.safetyOverrideUnavailableReason} +

    {/if} +
    {/if} + {#if detail.canResumeVerification}{/if} + {#if detail.canRetryWithRebuild}{/if} + {#if detail.canResumeRebuild}{/if} + {#if detail.canResumeCleanup}{/if} + {#if detail.canCancel}{/if} + {#if detail.canCancelCreation}{/if}
    {/if} - {#if detail.cancellation}
    {detail.cancellation.state === 'done' ? 'Cancellation cleanup completed' : 'Cancellation cleanup'}

    {detail.cancellation.error ?? detail.cancellation.stage}

    View cleanup progress →
    {/if} - {#if view.actionError}{/if} + {#if detail.cancellation}
    + {detail.cancellation.state === 'done' + ? 'Cancellation cleanup completed' + : 'Cancellation cleanup'} +

    {detail.cancellation.error ?? detail.cancellation.stage}

    + View cleanup progress → +
    {/if} + {#if view.actionError}{/if} {#if view.notice}

    {view.notice}

    {/if} - {#if detail.progress?.blocker?.href}{detail.progress.blocker.label ?? 'View blocking work'} →{/if} - {#if detail.progress?.recovery}
    {detail.progress.recovery.state === 'held' ? 'Recovery remains in progress' : detail.progress.recovery.state === 'released' ? 'Recovery capacity released' : detail.progress.recovery.state === 'transferred' ? 'Recovery transferred' : 'Recovery state not confirmed'}

    {detail.progress.recovery.message}

    Recorded {@render dateTime(detail.progress.recovery.at)}{#if detail.progress.recovery.operationId && detail.progress.recovery.operationId !== deployment.id}View recovery operation →{/if}
    {/if} - {#if serverHref && !['done', 'cancelled'].includes(deployment.state)}
    Open server logsDiagnose connection{#if detail.progress?.phase === 'verify-map' || detail.progress?.checks.some(check => check.key === 'map' && check.result !== 'pass')}Review map configuration{/if}
    {/if} + {#if detail.progress?.blocker?.href}{detail.progress.blocker.label ?? 'View blocking work'} →{/if} + {#if detail.progress?.recovery}
    + {detail.progress.recovery.state === 'held' + ? 'Recovery remains in progress' + : detail.progress.recovery.state === 'released' + ? 'Recovery capacity released' + : detail.progress.recovery.state === 'transferred' + ? 'Recovery transferred' + : 'Recovery state not confirmed'} +

    {detail.progress.recovery.message}

    + Recorded {@render dateTime(detail.progress.recovery.at)}{#if detail.progress.recovery.operationId && detail.progress.recovery.operationId !== deployment.id}View recovery operation →{/if} +
    {/if} + {#if serverHref && !['done', 'cancelled'].includes(deployment.state)}
    + Open server logsDiagnose connection{#if detail.progress?.phase === 'verify-map' || detail.progress?.checks.some((check) => check.key === 'map' && check.result !== 'pass')}Review map configuration{/if} +
    {/if} {#if (active && phaseStartedAt) || observedAt}
    - {#if active && phaseStartedAt}
    Time in phase{elapsedTime(phaseStartedAt, now)}
    {/if} - {#if observedAt}
    Last evaluated{@render dateTime(observedAt)}
    {/if} + {#if active && phaseStartedAt}
    + Time in phase{elapsedTime(phaseStartedAt, now)} +
    {/if} + {#if observedAt}
    + Last evaluated{@render dateTime(observedAt)} +
    {/if}
    {/if}
    @@ -186,49 +430,177 @@ {#key deployment.id}
    - +
    @@ -238,33 +610,356 @@
    diff --git a/src/routes/scripts/+page.svelte b/src/routes/scripts/+page.svelte index 9a9210f..f96b723 100644 --- a/src/routes/scripts/+page.svelte +++ b/src/routes/scripts/+page.svelte @@ -10,16 +10,37 @@ import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; import ScriptUploadDrawer from '$ui/components/ScriptUploadDrawer.svelte'; - import { betaSelectionHref, canTest, mapLabel, scopeLabel, scriptStatus, type LibraryScript, type ScriptStatus } from '$lib/script-library.js'; + import { + betaSelectionHref, + canTest, + mapLabel, + scopeLabel, + scriptStatus, + type LibraryScript, + type ScriptStatus + } from '$lib/script-library.js'; import { zombieMaps } from '$lib/map-aliases.js'; import { libraryLimit, saveLibraryContext } from '$ui/library-context.js'; let limit = $state(25); let mounted = $state(false); - afterNavigate(() => { mounted = true; }); + afterNavigate(() => { + mounted = true; + }); let lastFilters = ''; - $effect(() => { const key = JSON.stringify([search, status, game, map, sort]); if (!mounted) return; if (lastFilters && lastFilters !== key) limit = 25; lastFilters = key; }); - $effect(() => { if (mounted) saveLibraryContext({ q: search, status, game, map, sort, limit, selected: selected.join(',') }, { status: 'all', sort: 'attention', limit: 25 }); }); + $effect(() => { + const key = JSON.stringify([search, status, game, map, sort]); + if (!mounted) return; + if (lastFilters && lastFilters !== key) limit = 25; + lastFilters = key; + }); + $effect(() => { + if (mounted) + saveLibraryContext( + { q: search, status, game, map, sort, limit, selected: selected.join(',') }, + { status: 'all', sort: 'attention', limit: 25 } + ); + }); let scripts = $state([]); let loading = $state(true); let error = $state(''); @@ -32,141 +53,463 @@ let uploader: ScriptUploadDrawer; let dragDepth = $state(0); const tabs: Array<{ value: ScriptStatus | 'all'; label: string }> = [ - { value: 'all', label: 'All scripts' }, { value: 'untested', label: 'Needs testing' }, - { value: 'testing', label: 'Beta active' }, { value: 'failed', label: 'Test failed' }, - { value: 'tested', label: 'Test passed' }, { value: 'promoted', label: 'Promoted' } + { value: 'all', label: 'All scripts' }, + { value: 'untested', label: 'Needs testing' }, + { value: 'testing', label: 'Beta active' }, + { value: 'failed', label: 'Test failed' }, + { value: 'tested', label: 'Test passed' }, + { value: 'promoted', label: 'Promoted' } ]; - const failedCount = $derived(scripts.filter(script => scriptStatus(script) === 'failed').length); + const failedCount = $derived( + scripts.filter((script) => scriptStatus(script) === 'failed').length + ); const priority = { failed: 0, untested: 1, testing: 2, tested: 3, promoted: 4, empty: 5 }; - const maps = $derived([...new Set(scripts.filter(s => s.scope_type === 'map' && (!game || zombieMaps(game).some(m => m.canonical === s.scope_value))).map(s => s.scope_value!))].sort()); - const visible = $derived(scripts.filter(s => { - const haystack = `${s.name} ${s.target_rel_path} ${s.scope_value ?? ''} ${scopeLabel(s)}`.toLowerCase(); - return search.toLowerCase().trim().split(/\s+/).every(term => haystack.includes(term)) - && (status === 'all' || scriptStatus(s) === status) - && (!game || !s.scope_type || (s.scope_type === 'game' ? s.scope_value === game : s.scope_type === 'map' && zombieMaps(game).some(m => m.canonical === s.scope_value))) - && (!map || s.scope_type === 'map' && s.scope_value === map); - }).sort((a, b) => { - if (sort === 'recent') return (b.latestVersion?.uploaded_at ?? '').localeCompare(a.latestVersion?.uploaded_at ?? '') || a.name.localeCompare(b.name); - if (sort === 'attention') return priority[scriptStatus(a)] - priority[scriptStatus(b)] || a.name.localeCompare(b.name); - return a.name.localeCompare(b.name); - })); - const selectedScripts = $derived(scripts.filter(s => selected.includes(s.id) && canTest(s))); - const hiddenSelected = $derived(selectedScripts.filter(s => !visible.slice(0, limit).some(row => row.id === s.id)).length); + const maps = $derived( + [ + ...new Set( + scripts + .filter( + (s) => + s.scope_type === 'map' && + (!game || zombieMaps(game).some((m) => m.canonical === s.scope_value)) + ) + .map((s) => s.scope_value!) + ) + ].sort() + ); + const visible = $derived( + scripts + .filter((s) => { + const haystack = + `${s.name} ${s.target_rel_path} ${s.scope_value ?? ''} ${scopeLabel(s)}`.toLowerCase(); + return ( + search + .toLowerCase() + .trim() + .split(/\s+/) + .every((term) => haystack.includes(term)) && + (status === 'all' || scriptStatus(s) === status) && + (!game || + !s.scope_type || + (s.scope_type === 'game' + ? s.scope_value === game + : s.scope_type === 'map' && + zombieMaps(game).some((m) => m.canonical === s.scope_value))) && + (!map || (s.scope_type === 'map' && s.scope_value === map)) + ); + }) + .sort((a, b) => { + if (sort === 'recent') + return ( + (b.latestVersion?.uploaded_at ?? '').localeCompare( + a.latestVersion?.uploaded_at ?? '' + ) || a.name.localeCompare(b.name) + ); + if (sort === 'attention') + return ( + priority[scriptStatus(a)] - priority[scriptStatus(b)] || + a.name.localeCompare(b.name) + ); + return a.name.localeCompare(b.name); + }) + ); + const selectedScripts = $derived(scripts.filter((s) => selected.includes(s.id) && canTest(s))); + const hiddenSelected = $derived( + selectedScripts.filter((s) => !visible.slice(0, limit).some((row) => row.id === s.id)) + .length + ); let reviewSelection = $state(false); - const selectable = $derived(visible.slice(0,limit).filter(canTest)); - const allSelected = $derived(selectable.length > 0 && selectable.every(s => selected.includes(s.id))); + const selectable = $derived(visible.slice(0, limit).filter(canTest)); + const allSelected = $derived( + selectable.length > 0 && selectable.every((s) => selected.includes(s.id)) + ); async function load() { error = ''; try { const res = await fetch('/api/scripts?kind=script'); if (!res.ok) throw new Error('Could not load scripts. Try again.'); scripts = (await res.json()).scripts; - selected = selected.filter(id => scripts.some(s => s.id === id && canTest(s))); - } catch (e) { error = (e as Error).message; } - finally { loading = false; } + selected = selected.filter((id) => scripts.some((s) => s.id === id && canTest(s))); + } catch (e) { + error = (e as Error).message; + } finally { + loading = false; + } } onMount(() => { const params = new URLSearchParams(window.location.search); - search = params.get('q') ?? ''; game = params.get('game') ?? ''; map = params.get('map') ?? ''; - status = [...tabs, { value: 'empty' }].some(t => t.value === params.get('status')) ? params.get('status') as ScriptStatus : 'all'; - sort = ['attention', 'recent', 'name'].includes(params.get('sort') ?? '') ? params.get('sort')! : 'attention'; - limit = libraryLimit(params); selected = (params.get('selected') ?? '').split(',').map(Number).filter(id => Number.isSafeInteger(id) && id > 0).slice(0, 500); + search = params.get('q') ?? ''; + game = params.get('game') ?? ''; + map = params.get('map') ?? ''; + status = [...tabs, { value: 'empty' }].some((t) => t.value === params.get('status')) + ? (params.get('status') as ScriptStatus) + : 'all'; + sort = ['attention', 'recent', 'name'].includes(params.get('sort') ?? '') + ? params.get('sort')! + : 'attention'; + limit = libraryLimit(params); + selected = (params.get('selected') ?? '') + .split(',') + .map(Number) + .filter((id) => Number.isSafeInteger(id) && id > 0) + .slice(0, 500); void load(); }); - function toggle(id: number) { selected = selected.includes(id) ? selected.filter(n => n !== id) : [...selected, id]; } - function selectVisible() { - const ids = selectable.map(s => s.id); - selected = allSelected ? selected.filter(id => !ids.includes(id)) : [...new Set([...selected, ...ids])]; + function toggle(id: number) { + selected = selected.includes(id) ? selected.filter((n) => n !== id) : [...selected, id]; + } + function selectVisible() { + const ids = selectable.map((s) => s.id); + selected = allSelected + ? selected.filter((id) => !ids.includes(id)) + : [...new Set([...selected, ...ids])]; + } + function betaHref(rows: LibraryScript[]) { + return betaSelectionHref( + rows.map((s) => ({ scriptId: s.id, versionId: s.latestVersion!.id })) + ); + } + function resetFilters() { + search = ''; + status = 'all'; + game = ''; + map = ''; + } + function onDrag(event: DragEvent) { + if (event.dataTransfer?.types.includes('Files')) event.preventDefault(); } - function betaHref(rows: LibraryScript[]) { return betaSelectionHref(rows.map(s => ({ scriptId: s.id, versionId: s.latestVersion!.id }))); } - function resetFilters() { search = ''; status = 'all'; game = ''; map = ''; } - function onDrag(event: DragEvent) { if (event.dataTransfer?.types.includes('Files')) event.preventDefault(); } function onDrop(event: DragEvent) { if (!event.dataTransfer?.types.includes('Files')) return; - event.preventDefault(); dragDepth = 0; + event.preventDefault(); + dragDepth = 0; void uploader.addFiles([...event.dataTransfer.files]); } Scripts · Electric Cherry - { if (event.dataTransfer?.types.includes('Files')) { event.preventDefault(); dragDepth++; } }} - ondragleave={() => dragDepth = Math.max(0, dragDepth - 1)} ondragend={() => dragDepth = 0} /> + { + if (event.dataTransfer?.types.includes('Files')) { + event.preventDefault(); + dragDepth++; + } + }} + ondragleave={() => (dragDepth = Math.max(0, dragDepth - 1))} + ondragend={() => (dragDepth = 0)} +/>
    - - {#snippet actions()}{#if loading || scripts.length || error}{/if}{/snippet} + + {#snippet actions()}{#if loading || scripts.length || error}{/if}{/snippet}
    {#if loading || scripts.length || error}
    - - - - - -
    -
    - {visible.length} of {scripts.length} scripts - {#if failedCount && status !== 'failed'}{/if} - {#if visible.length && (search || status !== 'all' || game || map)}{/if} -
    {/if} - {#if game}

    Game results include unscoped scripts that can use explicitly selected beta servers.

    {/if} - {#if error}{/if} + + + + + +
    +
    + {visible.length} of {scripts.length} scripts + {#if failedCount && status !== 'failed'}{/if} + {#if visible.length && (search || status !== 'all' || game || map)}{/if} +
    {/if} + {#if game}

    + Game results include unscoped scripts that can use explicitly selected beta servers. +

    {/if} + {#if error}{/if} {#if loading}

    Loading your library…

    {:else if !scripts.length && !error} - - {#snippet actions()}{/snippet} + + {#snippet actions()}{/snippet} {:else if !visible.length && !error} - + {#snippet actions()}{/snippet} {:else} -
    - - {#each visible.slice(0,limit) as s (s.id)} - - - - - - - - - {/each} -
    ScriptGame / scopeVersion / statusEligible serversNext action
    toggle(s.id)} />{s.name}{#if s.target_rel_path !== `${s.name}.gsc`}{s.target_rel_path}{/if}{scopeLabel(s)}{s.scope_type ? s.servers.length : 'Explicit targets'}{#if s.runningBetaId !== null}View beta - {:else if scriptStatus(s) === 'failed'}Review failure - {:else if canTest(s)}Create beta{/if}
    +
    + + + {#each visible.slice(0, limit) as s (s.id)} + + + + + + + + + {/each} +
    ScriptGame / scopeVersion / statusEligible serversNext action
    toggle(s.id)} + />{s.name}{#if s.target_rel_path !== `${s.name}.gsc`}{s.target_rel_path}{/if}{scopeLabel(s)}{s.scope_type ? s.servers.length : 'Explicit targets'}{#if s.runningBetaId !== null}View beta + {:else if scriptStatus(s) === 'failed'}Review failure + {:else if canTest(s)}Create beta{/if}
    +
    {/if} -
    {#if selectedScripts.length}
    - {selectedScripts.length} selected{hiddenSelected ? ` · ${hiddenSelected} hidden` : ''}Latest versions selected - - - - {#if reviewSelection}
      {#each selectedScripts as script}
    • {script.name} · v{script.latestVersion?.version}
      • {/each}
    {/if} + {selectedScripts.length} selected{hiddenSelected + ? ` · ${hiddenSelected} hidden` + : ''}Latest versions selected + + + + {#if reviewSelection}
      + {#each selectedScripts as script}
    • + {script.name} · v{script.latestVersion?.version} +
      • {/each} +
    {/if}
    {/if}
    -{#if dragDepth > 0}
    Drop your scripts hereWe’ll recognize existing scripts and check for duplicate versions.
    {/if} +{#if dragDepth > 0}
    + Drop your scripts hereWe’ll recognize existing scripts and check for duplicate versions. +
    {/if} diff --git a/src/routes/server-keys/+page.svelte b/src/routes/server-keys/+page.svelte index c2d2bfa..2e55d60 100644 --- a/src/routes/server-keys/+page.svelte +++ b/src/routes/server-keys/+page.svelte @@ -15,7 +15,13 @@ import Field from '$lib/ui/components/primitives/Field.svelte'; import Input from '$lib/ui/components/primitives/Input.svelte'; - interface SavedKey { id: string; game: ZombieGame; label: string; assigned_servers: string[]; updated_at: string } + interface SavedKey { + id: string; + game: ZombieGame; + label: string; + assigned_servers: string[]; + updated_at: string; + } const games: ZombieGame[] = ['t5', 't6']; let open = $state(false); let search = $state(''); @@ -34,13 +40,29 @@ gameFilter = $page.url.searchParams.get('game') ?? ''; limit = Math.max(25, Number($page.url.searchParams.get('limit')) || 25); }); - const filtered = $derived(keys.filter(key => (!gameFilter || key.game === gameFilter) - && [key.game, key.label, ...key.assigned_servers.flatMap(name => [name, serverTitle(name)])].join(' ').toLowerCase().includes(search.toLowerCase()))); + const filtered = $derived( + keys.filter( + (key) => + (!gameFilter || key.game === gameFilter) && + [ + key.game, + key.label, + ...key.assigned_servers.flatMap((name) => [name, serverTitle(name)]) + ] + .join(' ') + .toLowerCase() + .includes(search.toLowerCase()) + ) + ); function saveSearch() { - const url = new URL($page.url); url.searchParams.delete('server'); - if (search) url.searchParams.set('search', search); else url.searchParams.delete('search'); - if (gameFilter) url.searchParams.set('game', gameFilter); else url.searchParams.delete('game'); - if (limit > 25) url.searchParams.set('limit', String(limit)); else url.searchParams.delete('limit'); + const url = new URL($page.url); + url.searchParams.delete('server'); + if (search) url.searchParams.set('search', search); + else url.searchParams.delete('search'); + if (gameFilter) url.searchParams.set('game', gameFilter); + else url.searchParams.delete('game'); + if (limit > 25) url.searchParams.set('limit', String(limit)); + else url.searchParams.delete('limit'); void goto(url, { replaceState: true, keepFocus: true, noScroll: true }); } async function load() { @@ -48,110 +70,312 @@ if (!res.ok) throw new Error('Could not load server keys'); keys = (await res.json()).keys; } - async function reload() { try { await load(); error = ''; } catch (e) { error = (e as Error).message; } finally { loading = false; } } - onMount(() => { void reload(); }); + async function reload() { + try { + await load(); + error = ''; + } catch (e) { + error = (e as Error).message; + } finally { + loading = false; + } + } + onMount(() => { + void reload(); + }); function addKey() { - plutoKey = ''; label = ''; error = ''; + plutoKey = ''; + label = ''; + error = ''; if (gameFilter === 't5' || gameFilter === 't6') game = gameFilter; open = true; } async function save(e: SubmitEvent) { e.preventDefault(); if (!plutoKey.trim() || busy) return; - busy = true; error = ''; notice = ''; + busy = true; + error = ''; + notice = ''; try { const res = await fetch('/api/server-keys', { - method: 'PUT', headers: { 'Content-Type': 'application/json' }, + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, label, plutoKey }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not save key'); - plutoKey = ''; label = ''; open = false; notice = 'Key saved to the game pool.'; await load(); - } catch (e) { error = (e as Error).message; } - finally { busy = false; } + plutoKey = ''; + label = ''; + open = false; + notice = 'Key saved to the game pool.'; + await load(); + } catch (e) { + error = (e as Error).message; + } finally { + busy = false; + } } async function remove(key: SavedKey) { if (key.assigned_servers.length) return; - if (!await confirmAction({ title: 'Remove saved key?', message: `Remove ${key.label || 'this key'} from the ${key.game.toUpperCase()} pool?`, confirmLabel: 'Remove key', danger: true })) return; - busy = true; error = ''; notice = ''; + if ( + !(await confirmAction({ + title: 'Remove saved key?', + message: `Remove ${key.label || 'this key'} from the ${key.game.toUpperCase()} pool?`, + confirmLabel: 'Remove key', + danger: true + })) + ) + return; + busy = true; + error = ''; + notice = ''; try { const res = await fetch('/api/server-keys', { - method: 'DELETE', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: key.id }) + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: key.id }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not remove key'); - notice = 'Saved key removed.'; await load(); - } catch (e) { error = (e as Error).message; } - finally { busy = false; } + notice = 'Saved key removed.'; + await load(); + } catch (e) { + error = (e as Error).message; + } finally { + busy = false; + } } Keys · Electric Cherry
    - - {#snippet actions()}{/snippet} - -

    A key stays assigned while its server exists, including when stopped, and becomes available after removal is confirmed. Rebuilds keep the same key. Use the optional label to record its Plutonium hostname; saving a label does not change that hostname.

    -{#if !loading}
    {#each games as poolGame} - {@const pool = keys.filter(key => key.game === poolGame)} -
    {poolGame === 't5' ? 'Black Ops I' : 'Black Ops II'}{pool.filter(key => !key.assigned_servers.length).length} available · {pool.filter(key => key.assigned_servers.length).length} assigned · {pool.length} total
    -{/each}
    {/if} -
    - { search = event.currentTarget.value; limit = 25; saveSearch(); }} placeholder="Search labels or assigned servers…" /> - + + {#snippet actions()}{/snippet} + +

    + A key stays assigned while its server exists, including when stopped, and becomes available + after removal is confirmed. Rebuilds keep the same key. Use the optional label to record its + Plutonium hostname; saving a label does not change that hostname. +

    + {#if !loading}
    + {#each games as poolGame} + {@const pool = keys.filter((key) => key.game === poolGame)} +
    + {poolGame === 't5' ? 'Black Ops I' : 'Black Ops II'}{pool.filter((key) => !key.assigned_servers.length).length} available · {pool.filter( + (key) => key.assigned_servers.length + ).length} assigned · {pool.length} total +
    + {/each} +
    {/if} +
    + { + search = event.currentTarget.value; + limit = 25; + saveSearch(); + }} + placeholder="Search labels or assigned servers…" + /> + +
    + {#if error && !open}{/if} + {#if notice}

    {notice}

    {/if} +

    Saved keys

    + {#if loading}

    Loading keys…

    + {:else if !filtered.length} + {:else} +
    + + + {#each filtered.slice(0, limit) as key (key.id)} + + + + + + + {/each} +
    Game / labelAvailabilityLast savedActions
    {key.game.toUpperCase()}{key.label || 'Unlabeled key'}{#if key.assigned_servers.length}Assigned{#each key.assigned_servers as name}
    + {name} +
    {/each}{:else}Available{/if}    		{#if key.assigned_servers.length}In use{/if}
    +
    + {/if} +
    -{#if error && !open}{/if} -{#if notice}

    {notice}

    {/if} -

    Saved keys

    -{#if loading}

    Loading keys…

    -{:else if !filtered.length} -{:else} -
    - - {#each filtered.slice(0, limit) as key (key.id)} - - - - - - - {/each} -
    Game / labelAvailabilityLast savedActions
    {key.game.toUpperCase()}{key.label || 'Unlabeled key'}{#if key.assigned_servers.length}Assigned{#each key.assigned_servers as name}
    {name}
    {/each}{:else}Available{/if}    		{#if key.assigned_servers.length}In use{/if}
    -{/if} - -
    - + {#if error}{/if}
    - - + + - - + + -
    +
    + +
    + diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index 6ffd84d..c591c1d 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -49,7 +49,12 @@ reason?: RconFailure; detail?: string; rawReply?: string; - status?: { players: number; map: string | null; gametype: string | null; roster: RosterPlayer[] }; + status?: { + players: number; + map: string | null; + gametype: string | null; + roster: RosterPlayer[]; + }; } interface Server { plutonium?: import('$lib/plutonium.js').PlutoniumVersion; @@ -58,24 +63,47 @@ rcon_status?: import('$lib/server-observation.js').GameObservation; telemetry?: import('$lib/telemetry.js').GameTelemetry; name: string; - native_instance_id?: string | null; host_id?: string | null; instance_generation?: number | null; game_library_id?: string | null; + native_instance_id?: string | null; + host_id?: string | null; + instance_generation?: number | null; + game_library_id?: string | null; host?: import('$lib/hosts.js').Host | null; - network?: { bindAddress: string; localPort: number; publicAddress: string | null; publicPort: number | null } | null; - game_port: number | null; rcon_host: string | null; - game: string | null; map: string | null; - last_players: number | null; last_checked_at: string | null; - last_map: string | null; last_gametype: string | null; - last_seen_at: string | null; last_error: string | null; + network?: { + bindAddress: string; + localPort: number; + publicAddress: string | null; + publicPort: number | null; + } | null; + game_port: number | null; + rcon_host: string | null; + game: string | null; + map: string | null; + last_players: number | null; + last_checked_at: string | null; + last_map: string | null; + last_gametype: string | null; + last_seen_at: string | null; + last_error: string | null; roster: RosterPlayer[] | null; - health: Health; endpoint: RconEndpoint | null; + health: Health; + endpoint: RconEndpoint | null; held: Array<{ target: string; betaId: number }>; - active_since: string | null; inactive_since: string | null; - game_started_at: string | null; game_ended_at: string | null; + active_since: string | null; + inactive_since: string | null; + game_started_at: string | null; + game_ended_at: string | null; } interface InventoryEntry { - relPath: string; kind?: string; scriptVersionId: number | null; - artifactId?: number | null; version?: number | null; releaseVersion?: string | null; - sha256: string; size: number; updatedAt: string; source: string | null; + relPath: string; + kind?: string; + scriptVersionId: number | null; + artifactId?: number | null; + version?: number | null; + releaseVersion?: string | null; + sha256: string; + size: number; + updatedAt: string; + source: string | null; } let name = $derived($page.params.name ?? ''); @@ -86,21 +114,46 @@ return detail.startFleetMonitor(); }); const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.serverSnapshot.servers); - const serverContext = $derived(visibleFleet.find(s => s.name === name) ?? null); - let server = $derived((serverContext as Server | null)); + const serverContext = $derived(visibleFleet.find((s) => s.name === name) ?? null); + let server = $derived(serverContext as Server | null); const health = $derived(serverContext ? serverHealth(serverContext, $uiNow) : null); - const canOperate = $derived($page.data?.user?.role === 'admin' || $page.data?.user?.role === 'operator'); - const availableActions = $derived(serverContext ? applicableRuntimeActions(serverContext, $uiNow) : []); - const primaryAction = $derived(availableActions.includes('restart') ? 'restart' : availableActions.includes('start') ? 'start' : null); - function openAction(action: RuntimeAction) { if (canOperate && availableActions.includes(action)) { lifecycleAction = action; lifecycleOpen = true; } } + const canOperate = $derived( + $page.data?.user?.role === 'admin' || $page.data?.user?.role === 'operator' + ); + const availableActions = $derived( + serverContext ? applicableRuntimeActions(serverContext, $uiNow) : [] + ); + const primaryAction = $derived( + availableActions.includes('restart') + ? 'restart' + : availableActions.includes('start') + ? 'start' + : null + ); + function openAction(action: RuntimeAction) { + if (canOperate && availableActions.includes(action)) { + lifecycleAction = action; + lifecycleOpen = true; + } + } let consumedAction = ''; $effect(() => { const requested = $page.url.searchParams.get('action'); const key = `${name}:${requested}`; - if (requested !== 'start') { consumedAction = ''; return; } - if (requested === 'start' && canOperate && availableActions.includes('start') && consumedAction !== key) { - consumedAction = key; openAction('start'); - const url = new URL($page.url); url.searchParams.delete('action'); + if (requested !== 'start') { + consumedAction = ''; + return; + } + if ( + requested === 'start' && + canOperate && + availableActions.includes('start') && + consumedAction !== key + ) { + consumedAction = key; + openAction('start'); + const url = new URL($page.url); + url.searchParams.delete('action'); void goto(url, { replaceState: true, noScroll: true, keepFocus: true }); } }); @@ -111,8 +164,23 @@ let savedRconSnippet = $state(null); let probingRcon = $state(false); let destroyOpen = $state(false); - const sectionIds = ['live', 'scoreboard', 'matches', 'chat', 'logs', 'configuration', 'config', 'inventory', 'diagnostics', 'history']; - const activeTab = $derived(sectionIds.includes($page.url.searchParams.get('tab') ?? '') ? $page.url.searchParams.get('tab')! : 'live'); + const sectionIds = [ + 'live', + 'scoreboard', + 'matches', + 'chat', + 'logs', + 'configuration', + 'config', + 'inventory', + 'diagnostics', + 'history' + ]; + const activeTab = $derived( + sectionIds.includes($page.url.searchParams.get('tab') ?? '') + ? $page.url.searchParams.get('tab')! + : 'live' + ); const activeSection = $derived(serverSection(activeTab)); const loadedServerName = $derived(server?.name); $effect(() => { @@ -120,8 +188,12 @@ const currentName = loadedServerName; if (!anchor || !currentName) return; let cancelled = false; - void tick().then(() => { if (!cancelled) document.getElementById(anchor)?.scrollIntoView({ block: 'start' }); }); - return () => { cancelled = true; }; + void tick().then(() => { + if (!cancelled) document.getElementById(anchor)?.scrollIntoView({ block: 'start' }); + }); + return () => { + cancelled = true; + }; }); function changeTab(tab: string) { const url = new URL($page.url); @@ -138,10 +210,26 @@ if (tab !== 'configuration' && (tab !== 'diagnostics' || !canOperate)) return; const sequence = ++tabSequence; try { - const response = await fetch('/api/servers/' + encodeURIComponent(currentName) + (tab === 'diagnostics' ? '/rcon-diagnostics' : '/scripts'), { signal }); - if (!response.ok) throw new Error(tab === 'diagnostics' ? 'Could not refresh RCON diagnostics.' : 'Could not refresh inventory.'); + const response = await fetch( + '/api/servers/' + + encodeURIComponent(currentName) + + (tab === 'diagnostics' ? '/rcon-diagnostics' : '/scripts'), + { signal } + ); + if (!response.ok) + throw new Error( + tab === 'diagnostics' + ? 'Could not refresh RCON diagnostics.' + : 'Could not refresh inventory.' + ); const body = await response.json(); - if (sequence !== tabSequence || name !== currentName || activeSection !== tab || signal?.aborted) return; + if ( + sequence !== tabSequence || + name !== currentName || + activeSection !== tab || + signal?.aborted + ) + return; if (tab === 'diagnostics') { if (!canOperate) return; savedRconSnippet = body.replySnippet; @@ -149,46 +237,94 @@ tabError = ''; return true; } catch (error) { - if (sequence === tabSequence && name === currentName && activeSection === tab && !signal?.aborted) tabError = (error as Error).message; + if ( + sequence === tabSequence && + name === currentName && + activeSection === tab && + !signal?.aborted + ) + tabError = (error as Error).message; return false; - } finally { if (sequence === tabSequence && name === currentName && activeSection === tab && !signal?.aborted) tabLoading = false; } + } finally { + if ( + sequence === tabSequence && + name === currentName && + activeSection === tab && + !signal?.aborted + ) + tabLoading = false; + } } $effect(() => { const currentName = name; const tab = activeSection; - tabError = ''; inventory = []; inventoryLimit = 25; savedRconSnippet = null; + tabError = ''; + inventory = []; + inventoryLimit = 25; + savedRconSnippet = null; tabLoading = tab === 'configuration' || (tab === 'diagnostics' && canOperate); if (tab !== 'configuration' && (tab !== 'diagnostics' || !canOperate)) return; - return pollVisible(signal => loadTab(currentName, tab, signal), 10000); + return pollVisible((signal) => loadTab(currentName, tab, signal), 10000); + }); + $effect(() => { + void name; + void canOperate; + rconProbe = null; }); - $effect(() => { name; canOperate; rconProbe = null; }); const fresh = $derived(server ? freshServer(server, $uiNow) : false); - let effectivePlayers = $derived(fresh && health?.state === 'online' ? server?.last_players ?? null : null); + let effectivePlayers = $derived( + fresh && health?.state === 'online' ? (server?.last_players ?? null) : null + ); const scoreboard = $derived(server?.telemetry ?? $telemetryByServer.get(name)); - let effectiveHealth = $derived(server?.health === 'online' && !fresh ? 'stale' : server?.health ?? 'offline'); - let effectiveSeen = $derived(server?.last_seen_at ?? null); - let effectiveErr = $derived(effectiveHealth === 'starting' || effectiveHealth === 'provisioning' - ? null : server?.rcon_status?.last_error ?? server?.last_error ?? null); + let effectiveHealth = $derived( + server?.health === 'online' && !fresh ? 'stale' : (server?.health ?? 'offline') + ); + let effectiveErr = $derived( + effectiveHealth === 'starting' || effectiveHealth === 'provisioning' + ? null + : (server?.rcon_status?.last_error ?? server?.last_error ?? null) + ); let effectiveEndpoint = $derived(server?.endpoint ?? null); let decoded = $derived(decodeRconError(effectiveErr)); let effectiveReason = $derived(decoded?.reason ?? null); let errorDetail = $derived(decoded?.detail ?? effectiveErr ?? ''); let effectiveSnippet = $derived( - canOperate ? rconProbe ? rconProbe.rawReply ?? null : savedRconSnippet : null + canOperate ? (rconProbe ? (rconProbe.rawReply ?? null) : savedRconSnippet) : null ); let effectiveActiveSince = $derived(server?.active_since ?? null); let effectiveInactiveSince = $derived(server?.inactive_since ?? null); let effectiveGameStartedAt = $derived(server?.game_started_at ?? null); let effectiveGameEndedAt = $derived(server?.game_ended_at ?? null); - let resetElapsed = $derived(fresh && effectiveGameEndedAt && server?.last_players !== 0 ? elapsedSince(effectiveGameEndedAt, $uiNow) : null); - let activeElapsed = $derived(fresh && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null); - let inactiveElapsed = $derived(fresh && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null); - let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince); - let gameElapsed = $derived(fresh && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null); + let resetElapsed = $derived( + fresh && effectiveGameEndedAt && server?.last_players !== 0 + ? elapsedSince(effectiveGameEndedAt, $uiNow) + : null + ); + let activeElapsed = $derived( + fresh && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null + ); + let inactiveElapsed = $derived( + fresh && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null + ); + let gameInProgress = $derived( + !!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince + ); + let gameElapsed = $derived( + fresh && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null + ); - const currentRound = $derived(effectivePlayers !== null && scoreboard && 'round' in scoreboard && freshTelemetry(scoreboard, $uiNow, server?.telemetry_after) ? scoreboard.round : null); - let hasDiagnosticIssue = $derived(!!health && !['online', 'starting', 'provisioning', 'stopped'].includes(health.state)); + const currentRound = $derived( + effectivePlayers !== null && + scoreboard && + 'round' in scoreboard && + freshTelemetry(scoreboard, $uiNow, server?.telemetry_after) + ? scoreboard.round + : null + ); + let hasDiagnosticIssue = $derived( + !!health && !['online', 'starting', 'provisioning', 'stopped'].includes(health.state) + ); function ago(iso: string | null, now: number): string { if (!iso) return 'never'; @@ -201,196 +337,564 @@ async function probeRcon(): Promise { if (!canOperate || probingRcon || !name) return; const currentName = name; - probingRcon = true; rconProbe = null; + probingRcon = true; + rconProbe = null; try { - const res = await fetch(`/api/servers/${encodeURIComponent(currentName)}/probe-rcon`, { method: 'POST' }); + const res = await fetch(`/api/servers/${encodeURIComponent(currentName)}/probe-rcon`, { + method: 'POST' + }); const body = await res.json().catch(() => null); - if (!body || typeof body.ok !== 'boolean') throw new Error(`HTTP ${res.status}: Expected a JSON connection result.`); + if (!body || typeof body.ok !== 'boolean') + throw new Error(`HTTP ${res.status}: Expected a JSON connection result.`); if (!canOperate || name !== currentName) return; rconProbe = body as RconProbeResult; await refreshFleet(); } catch (err) { - if (name === currentName) rconProbe = { ok: false, reason: 'send-error', detail: (err as Error).message }; - } finally { probingRcon = false; } + if (name === currentName) + rconProbe = { ok: false, reason: 'send-error', detail: (err as Error).message }; + } finally { + probingRcon = false; + } } // Tab config — Diagnostics gets a warning badge when there's an issue. let tabs = $derived([ { id: 'live', label: 'Live' }, { id: 'matches', label: 'Matches' }, { id: 'configuration', label: 'Configuration' }, - { id: 'diagnostics', label: 'Diagnostics', badge: hasDiagnosticIssue || effectiveErr ? '!' : null }, + { + id: 'diagnostics', + label: 'Diagnostics', + badge: hasDiagnosticIssue || effectiveErr ? '!' : null + }, { id: 'history', label: 'History' } ]); -{serverTitle(name)} · {activeTab.charAt(0).toUpperCase() + activeTab.slice(1)} · Electric Cherry - - +{serverTitle(name)} · {activeTab.charAt(0).toUpperCase() + activeTab.slice(1)} · Electric Cherry + + {#if server} -
    +
    {#snippet actions()} {#if canOperate} - {#if primaryAction}{/if} -
    More actions
    {#each availableActions.filter(action => action !== primaryAction) as action}{/each}
    + {#if primaryAction}{/if} +
    + More actions +
    + {#each availableActions.filter((action) => action !== primaryAction) as action}{/each} +
    +
    {/if} {/snippet}
    - {#if server.host_id}{server.host?.name ?? server.host_id}{/if} -
    Details
    + {#if server.host_id}{server.host?.name ?? server.host_id}{/if} +
    + Details +
    + +
    +
    - {#if $fleetError}{/if} + {#if $fleetError}{/if} {#if hasDiagnosticIssue && health}
    -
    {health.label}

    {health.reason} {health.since ? `Last evidence ${ago(health.since, $uiNow)}.` : 'No current report is available.'}

    -
    {#if health.state === 'host-offline' && server.host_id}{:else}{/if}
    +
    + {health.label} +

    + {health.reason} + {health.since + ? `Last evidence ${ago(health.since, $uiNow)}.` + : 'No current report is available.'} +

    +
    +
    + {#if health.state === 'host-offline' && server.host_id}{:else}{/if} +
    {/if}
    - +
    -
    - {#if tabError}{/if} +
    + {#if tabError}{/if} {#if activeSection === 'live'}
    - {#if !scoreboard || !fresh || !freshTelemetry(scoreboard, $uiNow, server.telemetry_after)}

    Players: {effectivePlayers ?? '—'} · {fleetIdentity(server).map}{!fresh ? ' · Current count unavailable' : ''}

    {/if} - - {#key name}{/key} - + {#if !scoreboard || !fresh || !freshTelemetry(scoreboard, $uiNow, server.telemetry_after)}

    + Players: {effectivePlayers ?? '—'} · {fleetIdentity(server).map}{!fresh + ? ' · Current count unavailable' + : ''} +

    {/if} + + {#key name}{/key} +
    - - {#key `${name}:${scoreboard?.timing?.matchId ?? ''}:${effectiveGameStartedAt ?? ''}`}{/key} - + + {#key `${name}:${scoreboard?.timing?.matchId ?? ''}:${effectiveGameStartedAt ?? ''}`}{/key} +
    {:else if activeSection === 'configuration'}
    - {#key name}{/key} + {#key name}{/key}
    - -
    -

    Installed files

    - {inventory.length} -
    - {#if tabLoading}{:else if inventory.length === 0 && !tabError} - - {:else} -
    - {#each inventory.slice(0, inventoryLimit) as f} - - {/each} -
    FileTypeRecorded versionDetails
    {#if f.artifactId}{f.relPath}{:else}{f.relPath}{/if}{f.kind ?? 'file'}{f.releaseVersion ?? (f.version ? `v${f.version}` : '—')}
    {f.relPath} Details
    Size
    {new Intl.NumberFormat().format(f.size)} B
    SHA-256
    {f.sha256}
    Recorded
    Source
    {#if f.source?.startsWith('promote:') || f.source?.startsWith('rollback:')}View campaign{:else}{f.source === 'native-create' ? 'Instance creation' : f.source ?? 'Unknown'}{/if}
    - {/if} -     - + +
    +

    Installed files

    + {inventory.length} +
    + {#if tabLoading}{:else if inventory.length === 0 && !tabError} + + {:else} +
    + + + {#each inventory.slice(0, inventoryLimit) as f} + + {/each} + +
    FileTypeRecorded versionDetails
    {#if f.artifactId}{f.relPath}{:else}{f.relPath}{/if}{f.kind ?? 'file'}{f.releaseVersion ?? + (f.version ? `v${f.version}` : '—')}
    + {f.relPath} + Details +
    +
    Size
    +
    + {new Intl.NumberFormat().format( + f.size + )} B +
    +
    SHA-256
    +
    {f.sha256}
    +
    Recorded
    +
    + +
    +
    Source
    +
    + {#if f.source?.startsWith('promote:') || f.source?.startsWith('rollback:')}View campaign{:else}{f.source === + 'native-create' + ? 'Instance creation' + : (f.source ?? + 'Unknown')}{/if} +
    +
    +
    +
    + {/if} +     +
    {:else if activeSection === 'diagnostics'}
    - -
    -

    Connection health

    -
    -
    - {#if native}
    Host agent{server.host?.connected && freshStatusTimestamp(server.host.lastHeartbeat, $uiNow) ? 'Online' : 'Unavailable'}{server.host?.lastHeartbeat ? `Last heartbeat ${ago(server.host.lastHeartbeat, $uiNow)}` : 'No heartbeat received'}{#if server.host_id}Inspect host{/if}
    {/if} -
    Game connection · RCON{effectiveReason ? 'Issue' : server.rcon_status?.last_seen_at && freshStatusTimestamp(server.rcon_status.last_seen_at, $uiNow) ? 'Responding' : 'No fresh response'}{server.rcon_status?.last_seen_at ? `Last response ${ago(server.rcon_status.last_seen_at, $uiNow)}` : 'No response received'}{#if canOperate && (effectiveEndpoint || native)}{/if}
    -
    Game telemetry{fresh && freshTelemetry(scoreboard, $uiNow, server.telemetry_after) ? 'Reporting' : scoreboard ? 'Last report outdated' : 'No report'}{scoreboard ? `Last report ${ago(scoreboard.receivedAt, $uiNow)}` : 'Waiting for game data'}
    -
    -

    Player count source: {fresh ? server.status_source === 'telemetry' ? 'game telemetry' : server.status_source === 'rcon' ? 'RCON' : 'unavailable' : 'unavailable'}.

    - {#if effectiveReason} -
    - {effectiveReason} - {errorDetail} + +
    +

    Connection health

    -
    {native ? 'Inspect the host agent connection and game logs. RCON is sent locally by the agent; public game-port forwarding does not affect this check.' : rconFixHint(effectiveReason)}
    - {:else if effectiveErr} -
    - Error - {effectiveErr} +
    + {#if native}
    + Host agent{server.host?.connected && + freshStatusTimestamp(server.host.lastHeartbeat, $uiNow) + ? 'Online' + : 'Unavailable'}{server.host?.lastHeartbeat + ? `Last heartbeat ${ago(server.host.lastHeartbeat, $uiNow)}` + : 'No heartbeat received'}{#if server.host_id}Inspect host{/if} +
    {/if} +
    + Game connection · RCON{effectiveReason + ? 'Issue' + : server.rcon_status?.last_seen_at && + freshStatusTimestamp( + server.rcon_status.last_seen_at, + $uiNow + ) + ? 'Responding' + : 'No fresh response'}{server.rcon_status?.last_seen_at + ? `Last response ${ago(server.rcon_status.last_seen_at, $uiNow)}` + : 'No response received'}{#if canOperate && (effectiveEndpoint || native)}{/if} +
    +
    + Game telemetry{fresh && + freshTelemetry(scoreboard, $uiNow, server.telemetry_after) + ? 'Reporting' + : scoreboard + ? 'Last report outdated' + : 'No report'}{scoreboard + ? `Last report ${ago(scoreboard.receivedAt, $uiNow)}` + : 'Waiting for game data'} +
    - {/if} +

    + Player count source: {fresh + ? server.status_source === 'telemetry' + ? 'game telemetry' + : server.status_source === 'rcon' + ? 'RCON' + : 'unavailable' + : 'unavailable'}. +

    + {#if effectiveReason} +
    + {effectiveReason} + {errorDetail} +
    +
    + {native + ? 'Inspect the host agent connection and game logs. RCON is sent locally by the agent; public game-port forwarding does not affect this check.' + : rconFixHint(effectiveReason)} +
    + {:else if effectiveErr} +
    + Error + {effectiveErr} +
    + {/if} - {#if canOperate && effectiveSnippet} -
    - Failure reply (addresses redacted) - 
    {effectiveSnippet}
    + {#if canOperate && effectiveSnippet} +
    + Failure reply (addresses redacted) + 
    {effectiveSnippet}
    +
    + {/if} + + {#if rconProbe?.ok} +
    + Connected + + RCON: {rconProbe.status?.players ?? '?'} players + {#if rconProbe.status?.map} + · map {rconProbe.status.map}{/if} + {#if rconProbe.status?.gametype} + · {rconProbe.status.gametype}{/if} + +
    + {/if} + + {#if rconProbe && !rconProbe.ok}{/if} +
    + Connection details +
    + {#if native}
    + Host{server.host?.name ?? server.host_id} +
    +
    + Instance{server.native_instance_id} +
    +
    + Generation{server.instance_generation} +
    +
    + Game-file version{server.game_library_id} +
    + {#if server.network}
    + Local UDP binding{server.network.bindAddress}:{server.network + .localPort} +
    +
    + Public / NAT endpoint{server.network.publicAddress || + server.host?.publicAddress || + server.host?.hostname || + 'Not configured'}:{server.network.publicPort ?? + server.network.localPort} +
    {/if} + {/if} +
    + Game port{server.game_port ?? '(unresolved)'} +
    +
    + Game{server.game ?? '(unset)'} +
    +
    + Map{server.map ?? '(unset)'} +
    + {#if server.held.length > 0} +
    + Held by beta{server.held + .map((h) => `${h.target} (beta ${h.betaId})`) + .join(', ')} +
    + {/if} +
    - {/if} - - {#if rconProbe?.ok} -
    - Connected - - RCON: {rconProbe.status?.players ?? '?'} players - {#if rconProbe.status?.map} · map {rconProbe.status.map}{/if} - {#if rconProbe.status?.gametype} · {rconProbe.status.gametype}{/if} - -
    - {/if} - - {#if rconProbe && !rconProbe.ok}{/if} -
    Connection details -
    - {#if native}
    Host{server.host?.name ?? server.host_id}
    Instance{server.native_instance_id}
    Generation{server.instance_generation}
    Game-file version{server.game_library_id}
    {#if server.network}
    Local UDP binding{server.network.bindAddress}:{server.network.localPort}
    Public / NAT endpoint{server.network.publicAddress || server.host?.publicAddress || server.host?.hostname || 'Not configured'}:{server.network.publicPort ?? server.network.localPort}
    {/if} - {/if} -
    Game port{server.game_port ?? '(unresolved)'}
    -
    Game{server.game ?? '(unset)'}
    -
    Map{server.map ?? '(unset)'}
    - {#if server.held.length > 0} -
    Held by beta{server.held.map((h) => `${h.target} (beta ${h.betaId})`).join(', ')}
    - {/if} -
    + +
    + Telemetry requests{#if telemetryOpen}
    + {#key name}{/key} +
    {/if}
    - -
    Telemetry requests{#if telemetryOpen}
    {#key name}{/key}
    {/if}
    - - - + + +
    {:else if activeSection === 'matches'} {#key name}{/key} {:else if activeSection === 'history'} -
    Open in Activity
    +
    + Open in Activity +
    {#key name}{/key} {/if}
    {:else if $fleetError} - + {:else if $fleet.receivedAt || data.serverSnapshot} - + {:else}
    Loading…
    {/if} diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 31ee991..380eb0f 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -7,28 +7,46 @@ let settings = $state(null); let loading = $state(true); let error = $state(''); - onMount(() => { void loadSettings(); }); + onMount(() => { + void loadSettings(); + }); async function loadSettings() { - loading = true; error = ''; + loading = true; + error = ''; try { const response = await fetch('/api/settings'); if (!response.ok) throw new Error('Could not load fleet settings.'); settings = validateFleetSettings(await response.json()); - } catch (cause) { error = cause instanceof Error ? cause.message : 'Could not load fleet settings.'; } - finally { loading = false; } + } catch (cause) { + error = cause instanceof Error ? cause.message : 'Could not load fleet settings.'; + } finally { + loading = false; + } } Settings · Electric Cherry
    - + {#if loading}

    Loading fleet settings…

    {:else if settings} - {:else}
    {/if} + {:else}
    + + +
    {/if}
    diff --git a/src/routes/settings/users/+page.svelte b/src/routes/settings/users/+page.svelte index bbd87cd..0855657 100644 --- a/src/routes/settings/users/+page.svelte +++ b/src/routes/settings/users/+page.svelte @@ -8,148 +8,732 @@ import Button from '$ui/components/primitives/Button.svelte'; import Dialog from '$ui/components/primitives/Dialog.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; - let users = $state([]); let loading = $state(true); let busy = $state(false); - let error = $state(''); let notice = $state(''); let formError = $state(''); - let createOpen = $state(false); let editOpen = $state(false); let editing = $state(null); - let username = $state(''); let displayName = $state(''); let password = $state(''); - let role = $state('viewer'); let enabled = $state(true); let showPassword = $state(false); - interface AuditEvent { id: number; actorId: string | null; actor: string | null; action: string; targetId: string | null; details: string; createdAt: number } - let events = $state([]); let auditLoading = $state(true); let auditError = $state(''); + let users = $state([]); + let loading = $state(true); + let busy = $state(false); + let error = $state(''); + let notice = $state(''); + let formError = $state(''); + let createOpen = $state(false); + let editOpen = $state(false); + let editing = $state(null); + let username = $state(''); + let displayName = $state(''); + let password = $state(''); + let role = $state('viewer'); + let enabled = $state(true); + let showPassword = $state(false); + interface AuditEvent { + id: number; + actorId: string | null; + actor: string | null; + action: string; + targetId: string | null; + details: string; + createdAt: number; + } + let events = $state([]); + let auditLoading = $state(true); + let auditError = $state(''); const roles: { value: UserRole; label: string; description: string }[] = [ - { value: 'viewer', label: 'Viewer', description: 'Read fleet status and operation summaries.' }, - { value: 'operator', label: 'Operator', description: 'Operate servers, campaigns, scripts, and configs.' }, - { value: 'admin', label: 'Administrator', description: 'Manage users, access, settings, hosts, and the fleet.' } + { + value: 'viewer', + label: 'Viewer', + description: 'Read fleet status and operation summaries.' + }, + { + value: 'operator', + label: 'Operator', + description: 'Operate servers, campaigns, scripts, and configs.' + }, + { + value: 'admin', + label: 'Administrator', + description: 'Manage users, access, settings, hosts, and the fleet.' + } ]; - const roleDescription = $derived(roles.find(item => item.value === role)?.description); + const roleDescription = $derived(roles.find((item) => item.value === role)?.description); const administrator = $derived($page.data.user?.role === 'admin'); async function load() { loading = true; try { - const response = await fetch('/api/users'); const body = await response.json(); + const response = await fetch('/api/users'); + const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load users.'); users = body.users; - } catch (cause) { error = (cause as Error).message; } - finally { loading = false; } + } catch (cause) { + error = (cause as Error).message; + } finally { + loading = false; + } } async function loadAudit() { - auditLoading = true; auditError = ''; + auditLoading = true; + auditError = ''; try { - const response = await fetch('/api/users/audit'); const body = await response.json(); + const response = await fetch('/api/users/audit'); + const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load account activity.'); events = body.events; - } catch (cause) { auditError = (cause as Error).message; } - finally { auditLoading = false; } + } catch (cause) { + auditError = (cause as Error).message; + } finally { + auditLoading = false; + } } - const actionLabels: Record = { setup: 'Administrator created', login: 'Signed in', 'login.failed': 'Sign-in failed', logout: 'Signed out', 'sessions.revoked': 'Sessions ended', 'user.created': 'User created', 'user.updated': 'Access updated', 'password.changed': 'Password changed' }; - function auditDetails(value: string): string { try { const parsed = JSON.parse(value); return Object.keys(parsed).length ? JSON.stringify(parsed, null, 2) : ''; } catch { return value; } } - onMount(() => { if (administrator) void Promise.all([load(), loadAudit()]); else { loading = false; auditLoading = false; } }); + const actionLabels: Record = { + setup: 'Administrator created', + login: 'Signed in', + 'login.failed': 'Sign-in failed', + logout: 'Signed out', + 'sessions.revoked': 'Sessions ended', + 'user.created': 'User created', + 'user.updated': 'Access updated', + 'password.changed': 'Password changed' + }; + function auditDetails(value: string): string { + try { + const parsed = JSON.parse(value); + return Object.keys(parsed).length ? JSON.stringify(parsed, null, 2) : ''; + } catch { + return value; + } + } + onMount(() => { + if (administrator) void Promise.all([load(), loadAudit()]); + else { + loading = false; + auditLoading = false; + } + }); function openCreate() { - username = ''; displayName = ''; password = ''; role = 'viewer'; enabled = true; showPassword = false; - formError = ''; editing = null; createOpen = true; + username = ''; + displayName = ''; + password = ''; + role = 'viewer'; + enabled = true; + showPassword = false; + formError = ''; + editing = null; + createOpen = true; } function openEdit(user: AuthUser) { - editing = user; username = user.username; displayName = user.displayName; password = ''; role = user.role; - enabled = !user.disabled; showPassword = false; formError = ''; editOpen = true; + editing = user; + username = user.username; + displayName = user.displayName; + password = ''; + role = user.role; + enabled = !user.disabled; + showPassword = false; + formError = ''; + editOpen = true; } async function create(event: SubmitEvent) { - event.preventDefault(); if (busy) return; busy = true; formError = ''; error = ''; notice = ''; + event.preventDefault(); + if (busy) return; + busy = true; + formError = ''; + error = ''; + notice = ''; try { - const response = await fetch('/api/users', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ username, displayName, password, role }) }); - const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not create this user.'); - password = ''; createOpen = false; notice = `${body.user.displayName || body.user.username} was added. They must change the temporary password at sign-in.`; + const response = await fetch('/api/users', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username, displayName, password, role }) + }); + const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not create this user.'); + password = ''; + createOpen = false; + notice = `${body.user.displayName || body.user.username} was added. They must change the temporary password at sign-in.`; await Promise.all([load(), loadAudit()]); - } catch (cause) { formError = (cause as Error).message; } finally { busy = false; } + } catch (cause) { + formError = (cause as Error).message; + } finally { + busy = false; + } } async function save(event: SubmitEvent) { - event.preventDefault(); if (busy || !editing) return; busy = true; formError = ''; error = ''; notice = ''; - const ownAccessChanged = editing.id === $page.data.user?.id && (role !== editing.role || enabled === editing.disabled || !!password); + event.preventDefault(); + if (busy || !editing) return; + busy = true; + formError = ''; + error = ''; + notice = ''; + const ownAccessChanged = + editing.id === $page.data.user?.id && + (role !== editing.role || enabled === editing.disabled || !!password); try { - const response = await fetch(`/api/users/${encodeURIComponent(editing.id)}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ displayName, role, disabled: !enabled, ...(password ? { password } : {}) }) }); - const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not save user changes.'); - password = ''; editOpen = false; - if (ownAccessChanged) { window.location.assign('/login'); return; } - notice = `Changes saved for ${body.user.displayName || body.user.username}.`; await Promise.all([load(), loadAudit()]); await invalidateAll(); - } catch (cause) { formError = (cause as Error).message; } finally { busy = false; } + const response = await fetch(`/api/users/${encodeURIComponent(editing.id)}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + displayName, + role, + disabled: !enabled, + ...(password ? { password } : {}) + }) + }); + const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not save user changes.'); + password = ''; + editOpen = false; + if (ownAccessChanged) { + window.location.assign('/login'); + return; + } + notice = `Changes saved for ${body.user.displayName || body.user.username}.`; + await Promise.all([load(), loadAudit()]); + await invalidateAll(); + } catch (cause) { + formError = (cause as Error).message; + } finally { + busy = false; + } } async function revoke(user: AuthUser) { const own = user.id === $page.data.user?.id; - if (!await confirmAction({ title: `Sign out ${user.displayName || user.username}?`, message: own ? 'This will end all of your sessions, including this one. You can sign in again with your password.' : 'This ends all active sessions for this user. Their account and password remain available for a new sign-in.', confirmLabel: 'Sign out sessions', danger: true })) return; - busy = true; error = ''; notice = ''; + if ( + !(await confirmAction({ + title: `Sign out ${user.displayName || user.username}?`, + message: own + ? 'This will end all of your sessions, including this one. You can sign in again with your password.' + : 'This ends all active sessions for this user. Their account and password remain available for a new sign-in.', + confirmLabel: 'Sign out sessions', + danger: true + })) + ) + return; + busy = true; + error = ''; + notice = ''; try { - const response = await fetch(`/api/users/${encodeURIComponent(user.id)}/revoke-sessions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' }); - const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not end these sessions.'); - if (own) { window.location.assign('/login'); return; } + const response = await fetch( + `/api/users/${encodeURIComponent(user.id)}/revoke-sessions`, + { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' } + ); + const body = await response.json(); + if (!response.ok) throw new Error(body.error ?? 'Could not end these sessions.'); + if (own) { + window.location.assign('/login'); + return; + } notice = `All sessions ended for ${user.displayName || user.username}.`; await loadAudit(); - } catch (cause) { error = (cause as Error).message; } finally { busy = false; } + } catch (cause) { + error = (cause as Error).message; + } finally { + busy = false; + } } Users · Electric Cherry
    - {#snippet actions()}{#if administrator}{/if}{/snippet} + {#snippet actions()}{#if administrator}{/if}{/snippet} - {#if !administrator}Go to your account + {#if !administrator} + Go to your account {:else} - {#if error}{/if}{#if notice}

    {notice}

    {/if} -
    {#each roles as item}

    {item.label}

    {item.description}

    {/each}
    + {#if error}{/if}{#if notice}

    + {notice} +

    {/if} +
    + {#each roles as item}
    +

    {item.label}

    +

    {item.description}

    +
    {/each} +
    {#if loading}

    Loading users…

    - {:else if !users.length}

    No users could be loaded.

    + {:else if !users.length}
    +

    No users could be loaded.

    + +
    {:else} -
    {#each users as user (user.id)}
    -

    {user.displayName} {#if user.id === $page.data.user?.id}You{/if}

    @{user.username}

    -
    {user.role === 'admin' ? 'Administrator' : user.role}{user.disabled ? 'Disabled' : 'Enabled'}{#if user.mustChangePassword && !user.disabled}Password change required{/if}
    - -
    -
    {/each}
    +
    + {#each users as user (user.id)}
    +
    +

    + {user.displayName} + {#if user.id === $page.data.user?.id}You{/if} +

    +

    @{user.username}

    +
    +
    + {user.role === 'admin' ? 'Administrator' : user.role}{user.disabled ? 'Disabled' : 'Enabled'}{#if user.mustChangePassword && !user.disabled}Password change required{/if} +
    + +
    + +
    +
    {/each} +
    {/if}
    -

    Account activity

    The latest 200 sign-in and account events.

    +
    +
    +

    Account activity

    +

    The latest 200 sign-in and account events.

    +
    + +
    {#if auditError}{/if} - {#if auditLoading && !events.length}

    Loading account activity…

    + {#if auditLoading && !events.length}

    + Loading account activity… +

    {:else if !events.length}

    No account events recorded.

    - {:else}
    - {#each events as event (event.id)}{@const details = auditDetails(event.details)}{/each} -
    TimeEventByAccountDetails
    {actionLabels[event.action] ?? event.action}{event.actor ? `@${event.actor}` : 'Unauthenticated'}{event.targetId ? users.find(user => user.id === event.targetId)?.username ?? event.targetId : '—'}{#if details}
    View details
    {details}
    {:else}—{/if}
    {/if} + {:else}
    + + + {#each events as event (event.id)}{@const details = auditDetails( + event.details + )}{/each} + +
    TimeEventByAccountDetails
    {actionLabels[event.action] ?? event.action}{event.actor ? `@${event.actor}` : 'Unauthenticated'}{event.targetId + ? (users.find((user) => user.id === event.targetId) + ?.username ?? event.targetId) + : '—'}{#if details}
    + View details + 
    {details}
    +
    {:else}—{/if}
    +
    {/if}
    {/if}
    - -
    - {#if formError}{/if} - -

    3–64 characters: letters, numbers, periods, underscores, and hyphens. Start with a letter or number.

    - -

    {roleDescription}

    - -

    8–128 characters. Save it before adding the user; it will not be displayed again.

    - -
    -
    + +
    +
    + {#if formError}{/if} + +

    + 3–64 characters: letters, numbers, periods, underscores, and hyphens. Start with a + letter or number. +

    + + +

    {roleDescription}

    + +

    + 8–128 characters. Save it before adding the user; it will not be displayed again. +

    + +
    + +
    +
    +
    - -
    - {#if formError}{/if} - -

    {roleDescription}

    -

    Disabled users cannot sign in. Running game servers continue operating.

    - {#if editing?.id === $page.data.user?.id}

    Use My account to change your own password. Changes to your access can sign you out.

    - {:else}

    Leave blank to keep the current password. A reset requires the user to choose a new password at their next sign-in.

    {/if} -
    -
    + +
    +
    + {#if formError}{/if} + + +

    {roleDescription}

    + +

    + Disabled users cannot sign in. Running game servers continue operating. +

    + {#if editing?.id === $page.data.user?.id}

    + Use My account to change your own password. Changes to your + access can sign you out. +

    + {:else} +

    + Leave blank to keep the current password. A reset requires the user to choose a + new password at their next sign-in. +

    + {/if} +
    + +
    +
    +
    diff --git a/src/routes/setup/+page.svelte b/src/routes/setup/+page.svelte index 92216a4..0265c4e 100644 --- a/src/routes/setup/+page.svelte +++ b/src/routes/setup/+page.svelte @@ -1,43 +1,196 @@ -Initial setup · Electric Cherry +Initial setup · Electric Cherry
    - -

    Use the setup token configured by the controller owner. This step is available only until the first administrator is created.

    + +

    + Use the setup token configured by the controller owner. This step is available only until + the first administrator is created. +

    {#if error}{/if} -
    - -

    Provided through the controller’s EC_SETUP_TOKEN setting.

    - -

    3–64 characters. Start with a letter or number; periods, underscores, and hyphens are allowed.

    - - -

    Use 8–128 characters. A long, unique passphrase works well.

    - - -
    +
    +
    + +

    + Provided through the controller’s EC_SETUP_TOKEN setting. +

    + +

    + 3–64 characters. Start with a letter or number; periods, underscores, and hyphens + are allowed. +

    + + +

    + Use 8–128 characters. A long, unique passphrase works well. +

    + + +
    +
    diff --git a/src/routes/watchdog/+page.server.ts b/src/routes/watchdog/+page.server.ts index ae10622..c1ab536 100644 --- a/src/routes/watchdog/+page.server.ts +++ b/src/routes/watchdog/+page.server.ts @@ -1,2 +1,4 @@ import { redirect } from '@sveltejs/kit'; -export function load({ url }: { url: URL }) { redirect(308, `/maintenance${url.search}`); } +export function load({ url }: { url: URL }) { + redirect(308, `/maintenance${url.search}`); +} diff --git a/src/routes/watchdog/[id]/+page.server.ts b/src/routes/watchdog/[id]/+page.server.ts index 15b5ed2..00b2c37 100644 --- a/src/routes/watchdog/[id]/+page.server.ts +++ b/src/routes/watchdog/[id]/+page.server.ts @@ -1,2 +1,4 @@ import { redirect } from '@sveltejs/kit'; -export function load({ params, url }: { params: { id: string }; url: URL }) { redirect(308, `/maintenance/${encodeURIComponent(params.id)}${url.search}`); } +export function load({ params, url }: { params: { id: string }; url: URL }) { + redirect(308, `/maintenance/${encodeURIComponent(params.id)}${url.search}`); +} diff --git a/tests/helpers/agent-files.ts b/tests/helpers/agent-files.ts index a2911ad..7103406 100644 --- a/tests/helpers/agent-files.ts +++ b/tests/helpers/agent-files.ts @@ -6,18 +6,40 @@ import { vi } from 'vitest'; export async function mockAgentFiles() { const backend = await import('../../src/lib/server/backends/native.js'); const { serverDir } = await import('../../src/lib/server/config.js'); - return vi.spyOn(backend, 'nativeRequest').mockImplementation(async (instance, kind, payload = {}) => { - const path = join(serverDir(instance.server), String(payload.path ?? '.')); - let result: unknown; - if (kind === 'files.read') { - const content = await fs.readFile(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); - result = { contentBase64: content?.toString('base64') ?? null }; - } else if (kind === 'files.stat') { - const stat = await fs.stat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); - result = stat ? { exists: true, size: stat.size, mtime: stat.mtime.toISOString(), isDirectory: stat.isDirectory(), isFile: stat.isFile() } : null; - } else if (kind === 'files.list') { - result = { entries: (await fs.readdir(path, { withFileTypes: true })).map(entry => ({ name: entry.name, isDirectory: entry.isDirectory(), isFile: entry.isFile() })) }; - } else throw new Error(`Unexpected agent command: ${kind}`); - return result as never; - }); + return vi + .spyOn(backend, 'nativeRequest') + .mockImplementation(async (instance, kind, payload = {}) => { + const path = join(serverDir(instance.server), String(payload.path ?? '.')); + let result: unknown; + if (kind === 'files.read') { + const content = await fs.readFile(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return null; + throw error; + }); + result = { contentBase64: content?.toString('base64') ?? null }; + } else if (kind === 'files.stat') { + const stat = await fs.stat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return null; + throw error; + }); + result = stat + ? { + exists: true, + size: stat.size, + mtime: stat.mtime.toISOString(), + isDirectory: stat.isDirectory(), + isFile: stat.isFile() + } + : null; + } else if (kind === 'files.list') { + result = { + entries: (await fs.readdir(path, { withFileTypes: true })).map((entry) => ({ + name: entry.name, + isDirectory: entry.isDirectory(), + isFile: entry.isFile() + })) + }; + } else throw new Error(`Unexpected agent command: ${kind}`); + return result as never; + }); } diff --git a/tests/helpers/fixture.ts b/tests/helpers/fixture.ts index e0b2e1a..3e12b41 100644 --- a/tests/helpers/fixture.ts +++ b/tests/helpers/fixture.ts @@ -27,8 +27,12 @@ export async function newFixture(servers: string[] = ['t5_zombie_theater_1']): P await mkdir(betaArchive, { recursive: true }); for (const s of servers) { - await mkdir(s.startsWith('t6_') ? join(t6, s, 'runtime/plutonium/storage/t6/scripts/zm') - : join(t5, s, 'runtime/plutonium/storage/t5/raw/scripts/sp'), { recursive: true }); + await mkdir( + s.startsWith('t6_') + ? join(t6, s, 'runtime/plutonium/storage/t6/scripts/zm') + : join(t5, s, 'runtime/plutonium/storage/t5/raw/scripts/sp'), + { recursive: true } + ); } process.env.SERVERS_ROOT = t5; @@ -49,7 +53,8 @@ export async function newFixture(servers: string[] = ['t5_zombie_theater_1']): P try { const { closeReports } = await import('../../src/lib/server/reports/index.js'); await closeReports(); - const { closeLaunchPreparation } = await import('../../src/lib/server/provisioning/launch-preparation.js'); + const { closeLaunchPreparation } = + await import('../../src/lib/server/provisioning/launch-preparation.js'); await closeLaunchPreparation(); const mod = (await import('../../src/lib/server/db/index.js')) as { closeDb: () => void; diff --git a/tests/helpers/native-host.ts b/tests/helpers/native-host.ts index 98497db..b3f298f 100644 --- a/tests/helpers/native-host.ts +++ b/tests/helpers/native-host.ts @@ -7,27 +7,60 @@ export async function prepareNativeHost(servers: string[] = []) { const libraries = await import('../../src/lib/server/game-files/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const host = hosts.createHost({ name: 'Fixture host', portMin: 28960, portMax: 29060 }); - const inventory = { protocolVersion: 1, cpuCount: 64, cpuUsagePercent: 0, - memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3, - dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, - runtimeImages: [config.defaultRuntimeImage] }; + const inventory = { + protocolVersion: 1, + cpuCount: 64, + cpuUsagePercent: 0, + memoryTotal: 128 * 1024 ** 3, + memoryAvailable: 120 * 1024 ** 3, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100_000_000_000, + runtimeImages: [config.defaultRuntimeImage] + }; hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); const versions = new Map(); for (const game of ['t5', 't6'] as const) { const version = libraries.createLibraryImport({ hostId: host.id, game }); - const manifest = JSON.stringify([{ path: 'zone/base.ff', size: 4, sha256: createHash('sha256').update('game').digest('hex') }]); + const manifest = JSON.stringify([ + { + path: 'zone/base.ff', + size: 4, + sha256: createHash('sha256').update('game').digest('hex') + } + ]); const hash = createHash('sha256').update(manifest).digest('hex'); - db().prepare("UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=4,published_at=? WHERE id=?") + db() + .prepare( + "UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=4,published_at=? WHERE id=?" + ) .run(manifest, hash, new Date().toISOString(), version.id); - libraries.updateHostLibraryCaches(host.id, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: hash }]); + libraries.updateHostLibraryCaches(host.id, [ + { libraryVersionId: version.id, status: 'ready', manifestSha256: hash } + ]); versions.set(game, version.id); } - const heartbeat = () => hosts.heartbeat(host.id, { ...inventory, libraries: [...versions.values()].map(id => ({ id, state: 'ready' as const })) }); + const heartbeat = () => + hosts.heartbeat(host.id, { + ...inventory, + libraries: [...versions.values()].map((id) => ({ id, state: 'ready' as const })) + }); heartbeat(); for (const [index, server] of servers.entries()) { - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,?,?,?,?)`).run(randomUUID(), server, host.id, versions.get(server.startsWith('t6_') ? 't6' : 't5'), - config.defaultRuntimeImage, 28960 + index, new Date().toISOString()); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,?,?,?,?)` + ) + .run( + randomUUID(), + server, + host.id, + versions.get(server.startsWith('t6_') ? 't6' : 't5'), + config.defaultRuntimeImage, + 28960 + index, + new Date().toISOString() + ); } return { host, versions, heartbeat }; } diff --git a/tests/helpers/provisioning.ts b/tests/helpers/provisioning.ts index 3b29298..5733770 100644 --- a/tests/helpers/provisioning.ts +++ b/tests/helpers/provisioning.ts @@ -1,13 +1,19 @@ // Integration ticks reflect the production worker: callers submit/observe; // Provisioning alone executes jobs between those reconciliation passes. -let campaignFlight: Promise>> | null = null; +let campaignFlight: Promise< + Awaited> +> | null = null; export function sweep() { if (campaignFlight) return campaignFlight; campaignFlight = (async () => { const caller = await import('../../src/lib/server/pipeline/sweep.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await caller.sweep(); await processProvisioning(); return caller.sweep(); - })().finally(() => { campaignFlight = null; }); + await caller.sweep(); + await processProvisioning(); + return caller.sweep(); + })().finally(() => { + campaignFlight = null; + }); return campaignFlight; } let betaFlight: Promise<{ processed: number }> | null = null; @@ -16,8 +22,12 @@ export function betaTick() { betaFlight = (async () => { const caller = await import('../../src/lib/server/pipeline/beta-runner.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await caller.betaTick(); await processProvisioning(); return caller.betaTick(); - })().finally(() => { betaFlight = null; }); + await caller.betaTick(); + await processProvisioning(); + return caller.betaTick(); + })().finally(() => { + betaFlight = null; + }); return betaFlight; } let maintenanceFlight: Promise | null = null; @@ -26,7 +36,11 @@ export function maintenanceTick(): Promise { maintenanceFlight = (async () => { const caller = await import('../../src/lib/server/fleet/plutonium.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await caller.maintenanceTick(); await processProvisioning(); await caller.maintenanceTick(); - })().finally(() => { maintenanceFlight = null; }); + await caller.maintenanceTick(); + await processProvisioning(); + await caller.maintenanceTick(); + })().finally(() => { + maintenanceFlight = null; + }); return maintenanceFlight; } diff --git a/tests/helpers/settings.ts b/tests/helpers/settings.ts index d3968ec..753ddf4 100644 --- a/tests/helpers/settings.ts +++ b/tests/helpers/settings.ts @@ -4,5 +4,7 @@ export function initializeStandaloneSettings(handle: Database.Database) { handle.exec(`CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY,value TEXT NOT NULL); CREATE TABLE IF NOT EXISTS retention_controls (category TEXT NOT NULL,record_id TEXT NOT NULL,protected INTEGER NOT NULL DEFAULT 0,resolved_at TEXT,PRIMARY KEY(category,record_id)); CREATE TABLE IF NOT EXISTS retention_expirations (category TEXT NOT NULL,record_id TEXT NOT NULL,expired_at TEXT NOT NULL,PRIMARY KEY(category,record_id));`); - handle.prepare("INSERT OR IGNORE INTO settings(key,value) VALUES('fleet_settings',?)").run(JSON.stringify(defaultFleetSettings)); + handle + .prepare("INSERT OR IGNORE INTO settings(key,value) VALUES('fleet_settings',?)") + .run(JSON.stringify(defaultFleetSettings)); } diff --git a/tests/helpers/staged-files.ts b/tests/helpers/staged-files.ts index b83b2bd..f4691f2 100644 --- a/tests/helpers/staged-files.ts +++ b/tests/helpers/staged-files.ts @@ -1,9 +1,14 @@ /** File/history unit tests simulate the activation leaf. Provisioning tests cover * the occupancy, capacity, source, and restart gates before reaching this leaf. */ export async function activateFiles(kind: 'beta' | 'campaign', id: number | string): Promise { - const { stagedFileChanges, applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js'); + const { stagedFileChanges, applyStagedFilesForOperation } = + await import('../../src/lib/server/fleet/staged-files.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/index.js'); - for (const server of new Set(stagedFileChanges(kind, String(id)).filter(row => row.state === 'pending').map(row => row.server))) { + for (const server of new Set( + stagedFileChanges(kind, String(id)) + .filter((row) => row.state === 'pending') + .map((row) => row.server) + )) { const operation = requestRestart(server, { kind, id: String(id) }); applyStagedFilesForOperation(operation.id, server); } diff --git a/tests/helpers/telemetry.ts b/tests/helpers/telemetry.ts index 6d24a64..76033a1 100644 --- a/tests/helpers/telemetry.ts +++ b/tests/helpers/telemetry.ts @@ -6,18 +6,34 @@ export function telemetryAuthHeaders(): Record { return { Authorization: `Bearer ${TEST_TELEMETRY_TOKEN}` }; } -export const roster = (count = 4): TelemetryPlayerStats[] => Array.from({ length: count }, (_, i) => ({ - name: `Player ${i + 1}`, guid: `1234567890123456${i}`, kills: 100 + i, - headshots: 50, downs: 1, revives: 2, score: 5000 -})); -export function gameEvent(event: GameEvent['event'], round = 25, players = 4, server = 't5_zombie_theater_1', map = 'zombie_theater'): GameEvent { +export const roster = (count = 4): TelemetryPlayerStats[] => + Array.from({ length: count }, (_, i) => ({ + name: `Player ${i + 1}`, + guid: `1234567890123456${i}`, + kills: 100 + i, + headshots: 50, + downs: 1, + revives: 2, + score: 5000 + })); +export function gameEvent( + event: GameEvent['event'], + round = 25, + players = 4, + server = 't5_zombie_theater_1', + map = 'zombie_theater' +): GameEvent { return event === 'game_start' ? { event, server, map, players: roster(players).map(({ name, guid }) => ({ name, guid })) } : { event, server, map, round, players: roster(players) }; } export async function postTelemetry(body: unknown, contentType = 'application/json') { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); - return POST({ request: new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': contentType, ...telemetryAuthHeaders() }, body: JSON.stringify(body) - }) }); + return POST({ + request: new Request('http://localhost/api/telemetry', { + method: 'POST', + headers: { 'Content-Type': contentType, ...telemetryAuthHeaders() }, + body: JSON.stringify(body) + }) + }); } diff --git a/tests/helpers/vacuum-memory.ts b/tests/helpers/vacuum-memory.ts index fdba825..23bd358 100644 --- a/tests/helpers/vacuum-memory.ts +++ b/tests/helpers/vacuum-memory.ts @@ -8,7 +8,9 @@ try { handle.pragma('cache_size = -16384'); handle.exec('CREATE TABLE records(id INTEGER PRIMARY KEY, content BLOB)'); const insert = handle.prepare('INSERT INTO records(content) VALUES(zeroblob(1048576))'); - handle.transaction(() => { for (let i = 0; i < 256; i++) insert.run(); })(); + handle.transaction(() => { + for (let i = 0; i < 256; i++) insert.run(); + })(); handle.pragma('wal_checkpoint(TRUNCATE)'); handle.pragma('shrink_memory'); // Reproduce the dangerous caller settings. The rebuild must override these. @@ -16,8 +18,15 @@ try { handle.pragma('cache_size = -131072'); const beforeKiB = process.resourceUsage().maxRSS; enableIncrementalVacuum(handle); - console.log(JSON.stringify({ beforeKiB, peakKiB: process.resourceUsage().maxRSS, - vacuum: handle.pragma('auto_vacuum', { simple: true }), - integrity: handle.pragma('quick_check', { simple: true }), - rows: handle.prepare('SELECT COUNT(*) n FROM records').get() })); -} finally { handle.close(); } + console.log( + JSON.stringify({ + beforeKiB, + peakKiB: process.resourceUsage().maxRSS, + vacuum: handle.pragma('auto_vacuum', { simple: true }), + integrity: handle.pragma('quick_check', { simple: true }), + rows: handle.prepare('SELECT COUNT(*) n FROM records').get() + }) + ); +} finally { + handle.close(); +} diff --git a/tests/unit/activity.test.ts b/tests/unit/activity.test.ts index c2e8359..6a33fca 100644 --- a/tests/unit/activity.test.ts +++ b/tests/unit/activity.test.ts @@ -4,8 +4,14 @@ import { parseActivityQuery } from '../../src/lib/activity.js'; import type { ActivityQuery } from '../../src/lib/activity.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); const server = 't5_zombie_theater_1'; const at = '2026-10-07T12:00:00.000Z'; @@ -19,41 +25,136 @@ async function setup() { const insert = handle.prepare(`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,server_deleted_at,observation_json,error) VALUES(?,?,?,?,?,?,?,?,?,?)`); - function operation(id: string, options: { state?: string; action?: string; server?: string; source?: string; at?: string; deleted?: string } = {}) { - insert.run(id, options.server ?? server, options.action ?? 'restart', options.state ?? 'done', 'Private stage token=secret-stage', - options.at ?? at, options.source ?? 'ui', options.deleted ?? null, '{"token":"secret-observation"}', 'secret-error'); + function operation( + id: string, + options: { + state?: string; + action?: string; + server?: string; + source?: string; + at?: string; + deleted?: string; + } = {} + ) { + insert.run( + id, + options.server ?? server, + options.action ?? 'restart', + options.state ?? 'done', + 'Private stage token=secret-stage', + options.at ?? at, + options.source ?? 'ui', + options.deleted ?? null, + '{"token":"secret-observation"}', + 'secret-error' + ); } - function remote(id: string, options: { kind?: string; state?: string; key?: string; routine?: number; success?: number; expired?: string; host?: string; instance?: string } = {}) { - handle.prepare(`INSERT INTO ec_host_operations (id,host_id,kind,instance_id,idempotency_key,payload,result,state,created_at,updated_at, - routine,observation_success,server_name,history_expired_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) - .run(id, options.host ?? host.id, options.kind ?? 'instance.restart', options.instance ?? null, options.key ?? null, - '{"token":"secret-payload"}', '{"token":"secret-result"}', options.state ?? 'completed', at, at, - options.routine ?? 0, options.success ?? 0, server, options.expired ?? null); + function remote( + id: string, + options: { + kind?: string; + state?: string; + key?: string; + routine?: number; + success?: number; + expired?: string; + host?: string; + instance?: string; + } = {} + ) { + handle + .prepare( + `INSERT INTO ec_host_operations (id,host_id,kind,instance_id,idempotency_key,payload,result,state,created_at,updated_at, + routine,observation_success,server_name,history_expired_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)` + ) + .run( + id, + options.host ?? host.id, + options.kind ?? 'instance.restart', + options.instance ?? null, + options.key ?? null, + '{"token":"secret-payload"}', + '{"token":"secret-result"}', + options.state ?? 'completed', + at, + at, + options.routine ?? 0, + options.success ?? 0, + server, + options.expired ?? null + ); } function link(operationId: string, kind: string, id: string) { - handle.prepare('INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,?,?,?)').run(operationId, kind, id, server); + handle + .prepare( + 'INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,?,?,?)' + ) + .run(operationId, kind, id, server); } function maintenance(id: number, state = 'done', operationId: string | null = null) { - handle.prepare(`INSERT INTO maintenance_jobs(id,server,state,target_revision,requested_at,kind,origin,provisioning_id,detail) - VALUES(?,?,?,123,?,'update','manual',?,'secret-maintenance-detail')`).run(id, server, state, at, operationId); + handle + .prepare( + `INSERT INTO maintenance_jobs(id,server,state,target_revision,requested_at,kind,origin,provisioning_id,detail) + VALUES(?,?,?,123,?,'update','manual',?,'secret-maintenance-detail')` + ) + .run(id, server, state, at, operationId); } function incident(id: string, kind = 'health', state = 'resolved') { - handle.prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) - VALUES(?,?,?,?,'secret-summary','{"credential":"secret-details"}',?,?)`).run(id, server, kind, state, at, at); + handle + .prepare( + `INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) + VALUES(?,?,?,?,'secret-summary','{"credential":"secret-details"}',?,?)` + ) + .run(id, server, kind, state, at, at); } function instance(id: string, hostId = host.id) { - handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,'library','image',28960,?)`).run(id, server, hostId, at); + handle + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,'library','image',28960,?)` + ) + .run(id, server, hostId, at); } - return { handle, activityHistory, operation, remote, link, maintenance, incident, instance, host, other }; + return { + handle, + activityHistory, + operation, + remote, + link, + maintenance, + incident, + instance, + host, + other + }; } describe('Activity query parameters', () => { it('keeps exact identities and valid filters while bounding pagination and ignoring invalid enum values', () => { - expect(parseActivityQuery(new URLSearchParams('server=t5_zombie_theater_1&host=a&type=restart&source=maintenance&state=failed&days=7&q=Kino&limit=999&offset=20'))) - .toMatchObject({ server, host: 'a', type: 'restart', source: 'maintenance', state: 'failed', days: 7, q: 'Kino', limit: 100, offset: 20 }); - expect(parseActivityQuery(new URLSearchParams('type=sql&state=bad&source=bad&days=NaN&limit=-5&offset=Infinity'))) - .toEqual({ type: undefined, state: undefined, source: undefined, limit: 25, offset: 0 }); + expect( + parseActivityQuery( + new URLSearchParams( + 'server=t5_zombie_theater_1&host=a&type=restart&source=maintenance&state=failed&days=7&q=Kino&limit=999&offset=20' + ) + ) + ).toMatchObject({ + server, + host: 'a', + type: 'restart', + source: 'maintenance', + state: 'failed', + days: 7, + q: 'Kino', + limit: 100, + offset: 20 + }); + expect( + parseActivityQuery( + new URLSearchParams( + 'type=sql&state=bad&source=bad&days=NaN&limit=-5&offset=Infinity' + ) + ) + ).toEqual({ type: undefined, state: undefined, source: undefined, limit: 25, offset: 0 }); }); }); @@ -69,9 +170,20 @@ describe('unified retained Activity', () => { api.incident('health'); api.incident('drift', 'drift'); api.remote('check', { kind: 'instance.inspect', routine: 1 }); - for (const [id, trigger] of [['recovery', 'watchdog'], ['release', 'ui']]) { - api.handle.prepare("INSERT INTO deployments(id,action,started_at,triggered_by) VALUES(?,'drift',?,?)").run(id, at, trigger); - api.handle.prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,?,1,'done')").run(id, server); + for (const [id, trigger] of [ + ['recovery', 'watchdog'], + ['release', 'ui'] + ]) { + api.handle + .prepare( + "INSERT INTO deployments(id,action,started_at,triggered_by) VALUES(?,'drift',?,?)" + ) + .run(id, at, trigger); + api.handle + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,?,1,'done')" + ) + .run(id, server); } const maintenance = api.activityHistory({ source: 'maintenance' }); expect(maintenance.total).toBe(6); @@ -80,7 +192,10 @@ describe('unified retained Activity', () => { const second = api.activityHistory({ ...query, offset: 1 }); expect(first.total).toBe(2); expect(second.total).toBe(2); - expect([...first.rows, ...second.rows].map(row => row.id).sort()).toEqual([`campaign:release:${server}`, 'host:check']); + expect([...first.rows, ...second.rows].map((row) => row.id).sort()).toEqual([ + `campaign:release:${server}`, + 'host:check' + ]); for (const source of ['maintenance', 'recovery'] as const) { expect(api.activityHistory({ ...query, source }).total).toBe(0); } @@ -92,7 +207,14 @@ describe('unified retained Activity', () => { it('excludes provisioning jobs from log totals and pages regardless of action or trigger, while retaining contextual history', async () => { const api = await setup(); for (const action of ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy']) { - for (const source of ['ui', 'autoscaling:policy', 'maintenance:1', 'campaign:release', 'beta:1', 'watchdog:finding']) { + for (const source of [ + 'ui', + 'autoscaling:policy', + 'maintenance:1', + 'campaign:release', + 'beta:1', + 'watchdog:finding' + ]) { api.operation(`${action}-${source}`, { action, source }); } } @@ -109,7 +231,11 @@ describe('unified retained Activity', () => { expect(last.total).toBe(3); expect(first.rows).toHaveLength(2); expect(last.rows).toHaveLength(1); - expect([...first.rows, ...last.rows].map(row => row.id).sort()).toEqual(['host:check', 'maintenance:2', 'recovery:finding']); + expect([...first.rows, ...last.rows].map((row) => row.id).sort()).toEqual([ + 'host:check', + 'maintenance:2', + 'recovery:finding' + ]); expect(api.activityHistory({ ...query, type: 'create' }).total).toBe(0); expect(api.activityHistory({ ...query, type: 'restart' }).total).toBe(0); expect(api.activityHistory({ ...query, source: 'autoscaling' }).total).toBe(0); @@ -122,7 +248,8 @@ describe('unified retained Activity', () => { for (let i = 0; i < 130; i++) api.operation(`life-${String(i).padStart(3, '0')}`); for (let i = 0; i < 33; i++) api.remote(`remote-${i}`); api.operation('removed', { deleted: at, at: '2025-01-01T00:00:00.000Z' }); - api.incident('finding'); api.maintenance(1); + api.incident('finding'); + api.maintenance(1); })(); const all = []; for (let offset = 0; offset < 166; offset += 25) { @@ -131,7 +258,7 @@ describe('unified retained Activity', () => { expect(page.rows.length).toBeLessThanOrEqual(25); all.push(...page.rows); } - expect(new Set(all.map(row => row.id)).size).toBe(166); + expect(new Set(all.map((row) => row.id)).size).toBe(166); expect(all.at(-1)).toMatchObject({ id: 'provisioning:removed', serverDeletedAt: at }); expect(api.activityHistory({ offset: 99_999 }).offset).toBe(150); }); @@ -142,13 +269,31 @@ describe('unified retained Activity', () => { api.operation('match', { state: 'failed' }); api.operation('old', { state: 'failed', at: '2025-01-01T00:00:00.000Z' }); api.instance('instance'); - api.handle.prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run('match', 'instance'); - const filters: ActivityQuery = { server, host: api.host.id, state: 'failed', type: 'restart', q: 'Kino', days: 7, limit: 1 }; + api.handle + .prepare( + 'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)' + ) + .run('match', 'instance'); + const filters: ActivityQuery = { + server, + host: api.host.id, + state: 'failed', + type: 'restart', + q: 'Kino', + days: 7, + limit: 1 + }; const page = api.activityHistory(filters, Date.parse(at)); expect(page.total).toBe(1); - expect(page.rows[0]).toMatchObject({ id: 'provisioning:match', hostId: api.host.id, hostName: 'Dallas host' }); + expect(page.rows[0]).toMatchObject({ + id: 'provisioning:match', + hostId: api.host.id, + hostName: 'Dallas host' + }); expect(api.activityHistory({ ...filters, server: 'Kino' }, Date.parse(at)).total).toBe(0); - expect(api.activityHistory({ ...filters, host: api.other.id }, Date.parse(at)).total).toBe(0); + expect(api.activityHistory({ ...filters, host: api.other.id }, Date.parse(at)).total).toBe( + 0 + ); expect(api.activityHistory({ q: "' OR 1=1 --" }).total).toBe(0); expect(api.activityHistory({ q: 'Dallas' }).total).toBe(1); }); @@ -160,7 +305,11 @@ describe('unified retained Activity', () => { api.operation('sql-yesterday', { at: '2026-10-06 14:00:00' }); api.operation('too-old', { at: '2026-10-06T11:00:00.000Z' }); const page = api.activityHistory({ days: 1 }, Date.parse(at)); - expect(page.rows.map(row => row.id)).toEqual(['provisioning:sql-afternoon', 'provisioning:iso-noon', 'provisioning:sql-yesterday']); + expect(page.rows.map((row) => row.id)).toEqual([ + 'provisioning:sql-afternoon', + 'provisioning:iso-noon', + 'provisioning:sql-yesterday' + ]); expect(page.total).toBe(3); }); @@ -173,11 +322,22 @@ describe('unified retained Activity', () => { api.link('shared', 'beta', '99'); const row = api.activityHistory().rows[0]; expect(api.activityHistory().total).toBe(1); - expect(row).toMatchObject({ id: 'provisioning:shared', type: 'update', state: 'failed', source: 'maintenance', - sources: ['maintenance', 'campaign', 'beta'], href: '/provisioning/shared', detail: 'The restart completed, but maintenance verification failed.' }); - for (const source of ['maintenance', 'campaign', 'beta'] as const) expect(api.activityHistory({ source }).total).toBe(1); + expect(row).toMatchObject({ + id: 'provisioning:shared', + type: 'update', + state: 'failed', + source: 'maintenance', + sources: ['maintenance', 'campaign', 'beta'], + href: '/provisioning/shared', + detail: 'The restart completed, but maintenance verification failed.' + }); + for (const source of ['maintenance', 'campaign', 'beta'] as const) + expect(api.activityHistory({ source }).total).toBe(1); api.handle.prepare("UPDATE maintenance_jobs SET state='restarting' WHERE id=1").run(); - expect(api.activityHistory().rows[0]).toMatchObject({ state: 'running', rawState: 'verifying' }); + expect(api.activityHistory().rows[0]).toMatchObject({ + state: 'running', + rawState: 'verifying' + }); api.handle.prepare("UPDATE maintenance_jobs SET state='done' WHERE id=1").run(); expect(api.activityHistory().rows[0].state).toBe('succeeded'); }); @@ -190,7 +350,7 @@ describe('unified retained Activity', () => { expect(api.activityHistory({ source: 'maintenance' }).total).toBe(3); const recovery = api.activityHistory({ source: 'recovery' }); expect(recovery.total).toBe(2); - expect(recovery.rows.map(row => row.type).sort()).toEqual(['drift', 'rebuild']); + expect(recovery.rows.map((row) => row.type).sort()).toEqual(['drift', 'rebuild']); expect(api.activityHistory({ source: 'provisioning' }).total).toBe(0); }); @@ -201,79 +361,145 @@ describe('unified retained Activity', () => { api.maintenance(1, 'failed'); const rows = api.activityHistory().rows; expect(rows).toHaveLength(2); - expect(rows.find(row => row.id === 'provisioning:old')?.state).toBe('succeeded'); - expect(rows.find(row => row.id === 'provisioning:new')?.state).toBe('failed'); + expect(rows.find((row) => row.id === 'provisioning:old')?.state).toBe('succeeded'); + expect(rows.find((row) => row.id === 'provisioning:new')?.state).toBe('failed'); }); it('deduplicates current command pointers and historical idempotency attempts without dropping independent work', async () => { const api = await setup(); - api.operation('operation'); api.instance('instance'); - api.remote('pointed'); api.remote('staged'); api.remote('migration'); - api.handle.prepare(`INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,stage_operation_id) - VALUES('operation','instance',1,'pointed','staged')`).run(); - api.handle.prepare(`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id) - VALUES('operation',?,'instance','migration')`).run(api.host.id); - for (const key of ['operation', 'operation:stage', 'operation:migration-create:1', 'operation:migrate-remove:2']) api.remote(`linked-${key}`, { key }); + api.operation('operation'); + api.instance('instance'); + api.remote('pointed'); + api.remote('staged'); + api.remote('migration'); + api.handle + .prepare( + `INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,stage_operation_id) + VALUES('operation','instance',1,'pointed','staged')` + ) + .run(); + api.handle + .prepare( + `INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id) + VALUES('operation',?,'instance','migration')` + ) + .run(api.host.id); + for (const key of [ + 'operation', + 'operation:stage', + 'operation:migration-create:1', + 'operation:migrate-remove:2' + ]) + api.remote(`linked-${key}`, { key }); api.remote('independent'); api.remote('not-a-retry', { key: 'operation:migration-create:not-a-number' }); api.remote('source-retained-away', { key: 'expired-operation' }); - expect(api.activityHistory().rows.map(row => row.id).sort()).toEqual([ - 'host:independent', 'host:not-a-retry', 'host:source-retained-away', 'provisioning:operation' + expect( + api + .activityHistory() + .rows.map((row) => row.id) + .sort() + ).toEqual([ + 'host:independent', + 'host:not-a-retry', + 'host:source-retained-away', + 'provisioning:operation' ]); }); it('keeps failed host observations, hides successful routine polls and expired records, and never projects private data', async () => { const api = await setup(); - api.operation('operation'); api.incident('finding'); api.maintenance(1); + api.operation('operation'); + api.incident('finding'); + api.maintenance(1); api.remote('routine', { kind: 'instance.inspect', routine: 1, success: 1 }); api.remote('bad-check', { kind: 'instance.inspect', routine: 1 }); api.remote('failed-check', { kind: 'instance.inspect', routine: 1, state: 'failed' }); api.remote('expired', { expired: at }); const result = api.activityHistory(); - expect(result.rows.filter(row => row.source === 'host').map(row => row.id).sort()).toEqual(['host:bad-check', 'host:failed-check']); - expect(result.rows.find(row => row.id === 'host:bad-check')).toMatchObject({ state: 'failed', rawState: 'observation-failed', href: '/hosts/operations?operation=bad-check&filter=all' }); + expect( + result.rows + .filter((row) => row.source === 'host') + .map((row) => row.id) + .sort() + ).toEqual(['host:bad-check', 'host:failed-check']); + expect(result.rows.find((row) => row.id === 'host:bad-check')).toMatchObject({ + state: 'failed', + rawState: 'observation-failed', + href: '/hosts/operations?operation=bad-check&filter=all' + }); expect(JSON.stringify(result)).not.toContain('secret-'); expect(JSON.stringify(result)).not.toMatch(/payload|observation_json|result|credential/); }); it('retains file campaigns and beta tests as distinct work with friendly titles', async () => { const api = await setup(); - api.handle.exec(`INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'Script','script.gsc'); + api.handle + .exec(`INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'Script','script.gsc'); INSERT INTO script_versions(id,script_id,version,sha256,size) VALUES(1,1,1,'hash',0); INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at) VALUES(1,1,1,'ui','promoted','${at}'); INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(1,'${server}',0,'ready'); INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote','${at}'); INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,server_deleted_at) VALUES('campaign','${server}',1,'done','${at}');`); const rows = api.activityHistory().rows; - expect(rows).toEqual(expect.arrayContaining([ - expect.objectContaining({ source: 'campaign', title: 'Promote files', detail: '1 file changed.', state: 'succeeded', serverDeletedAt: at, href: '/campaigns/campaign' }), - expect.objectContaining({ source: 'beta', title: 'Beta test #1', state: 'succeeded', href: '/betas/1' }) - ])); + expect(rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: 'campaign', + title: 'Promote files', + detail: '1 file changed.', + state: 'succeeded', + serverDeletedAt: at, + href: '/campaigns/campaign' + }), + expect.objectContaining({ + source: 'beta', + title: 'Beta test #1', + state: 'succeeded', + href: '/betas/1' + }) + ]) + ); api.handle.prepare("UPDATE betas SET state='running' WHERE id=1").run(); for (const stage of ['preparing', 'staging', 'testing', 'observing']) { api.handle.prepare('UPDATE beta_servers SET stage=? WHERE beta_id=1').run(stage); expect(api.activityHistory({ source: 'beta' }).rows[0].state).toBe('running'); } - api.handle.prepare("UPDATE deployments SET triggered_by='watchdog' WHERE id='campaign'").run(); - expect(api.activityHistory({ source: 'maintenance' }).rows[0]).toMatchObject({ sources: ['recovery', 'campaign'] }); + api.handle + .prepare("UPDATE deployments SET triggered_by='watchdog' WHERE id='campaign'") + .run(); + expect(api.activityHistory({ source: 'maintenance' }).rows[0]).toMatchObject({ + sources: ['recovery', 'campaign'] + }); expect(api.activityHistory({ source: 'recovery' }).total).toBe(1); expect(api.activityHistory({ source: 'campaign' }).total).toBe(1); }); it('uses the same query contract for API and page loaders with no-store API responses', async () => { const api = await setup(); - api.operation('a'); api.operation('b', { state: 'failed' }); + api.operation('a'); + api.operation('b', { state: 'failed' }); api.remote('failed-check', { kind: 'instance.inspect', routine: 1, state: 'failed' }); api.maintenance(1, 'failed'); api.incident('finding', 'health', 'attention'); - const url = new URL('http://localhost/activity?state=failed&limit=1&excludeProvisioning=true&excludeMaintenance=true'); + const url = new URL( + 'http://localhost/activity?state=failed&limit=1&excludeProvisioning=true&excludeMaintenance=true' + ); const { GET } = await import('../../src/routes/api/activity/+server.js'); const { load } = await import('../../src/routes/activity/+page.server.js'); const response = GET({ url }); expect(response.headers.get('cache-control')).toBe('no-store'); expect(await response.json()).toEqual(JSON.parse(JSON.stringify(load({ url }).activity))); expect(load({ url }).activity.rows[0].id).toBe('host:failed-check'); - expect(load({ url: new URL('http://localhost/activity?excludeProvisioning=false') }).activity.rows.some(row => row.id.startsWith('provisioning:'))).toBe(false); - expect(load({ url: new URL('http://localhost/activity?excludeMaintenance=false') }).activity.rows.map(row => row.id)).toEqual(['host:failed-check']); + expect( + load({ + url: new URL('http://localhost/activity?excludeProvisioning=false') + }).activity.rows.some((row) => row.id.startsWith('provisioning:')) + ).toBe(false); + expect( + load({ + url: new URL('http://localhost/activity?excludeMaintenance=false') + }).activity.rows.map((row) => row.id) + ).toEqual(['host:failed-check']); }); }); diff --git a/tests/unit/agent-operation-delivery.test.ts b/tests/unit/agent-operation-delivery.test.ts index 82d7816..27b3029 100644 --- a/tests/unit/agent-operation-delivery.test.ts +++ b/tests/unit/agent-operation-delivery.test.ts @@ -6,22 +6,34 @@ vi.mock('../../src/lib/server/hosts/preparation.js', () => ({ reconcileHostPrepa let fixture: Fixture; const inventory = { - protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }], + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: ['plutainer:test'], + libraries: [{ id: 'test-library', state: 'ready' }], capabilities: ['instance.cancel-create', 'operations.health-lane'] }; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + await fixture.cleanup(); +}); async function enrolled() { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Delivery host' }); const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); const { GET } = await import('../../src/routes/api/agent/operations/+server.js'); - const request = (query = '') => new Request(`https://ec.test/api/agent/operations${query}`, { - headers: { authorization: `Bearer ${credentials.credential}` } - }); + const request = (query = '') => + new Request(`https://ec.test/api/agent/operations${query}`, { + headers: { authorization: `Bearer ${credentials.credential}` } + }); return { ...hosts, host, GET, request }; } @@ -33,15 +45,18 @@ describe('agent operation delivery', () => { value: 'x'.repeat(64 * 1024 - '{"value":"'.length - 1) + '🧟 café "quoted"\n', files: [{ path: 'plugins/runtime.dll', contentBase64: 'A'.repeat(8 * 1024 ** 2) }] }; - const operation = api.queueHostOperation(api.host.id, 'instance.create', payload, - { instanceId: 'instance-one', generation: 2 }); + const operation = api.queueHostOperation(api.host.id, 'instance.create', payload, { + instanceId: 'instance-one', + generation: 2 + }); const result = { phase: 'prepared', description: '世界', details: [null, true, 42] }; api.reportHostOperation(api.host.id, operation.id, { state: 'running', result }); const expected = api.getHostOperation(operation.id)!; const serialized = JSON.stringify(payload); const originalParse = JSON.parse; const parse = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => { - if (value === serialized) throw new Error('Delivery must not deserialize launch files.'); + if (value === serialized) + throw new Error('Delivery must not deserialize launch files.'); return originalParse(value, reviver); }); @@ -50,7 +65,9 @@ describe('agent operation delivery', () => { expect(response.headers.get('content-type')).toContain('application/json'); let complete = false; let interleaved = false; - const controllerWork = setImmediate().then(() => { interleaved = !complete; }); + const controllerWork = setImmediate().then(() => { + interleaved = !complete; + }); const chunks: Uint8Array[] = []; const reader = response.body!.getReader(); while (true) { @@ -74,20 +91,31 @@ describe('agent operation delivery', () => { const payload = Buffer.from(JSON.stringify({ file: 'A'.repeat(512 * 1024) })); api.queueHostOperationSerialized(api.host.id, 'instance.create', payload); const second = api.queueHostOperationSerialized(api.host.id, 'instance.create', payload); - api.reportHostOperationStatus(api.host.id, second.id, { state: 'accepted', result: { phase: 'selected' } }); + api.reportHostOperationStatus(api.host.id, second.id, { + state: 'accepted', + result: { phase: 'selected' } + }); const { db } = await import('../../src/lib/server/db/index.js'); const prepare = vi.spyOn(db(), 'prepare'); - const payloadReads = () => prepare.mock.calls.filter(([sql]) => sql.startsWith('SELECT CAST(') && sql.includes(' AS BLOB) AS payload')).length; + const payloadReads = () => + prepare.mock.calls.filter( + ([sql]) => sql.startsWith('SELECT CAST(') && sql.includes(' AS BLOB) AS payload') + ).length; const response = await api.GET({ request: api.request('?lane=write') }); - const selection = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operations o'))![0]; + const selection = prepare.mock.calls.find(([sql]) => + sql.includes('FROM ec_host_operations o') + )![0]; const projection = selection.split(' FROM ')[0]; expect(projection).not.toContain('payload'); expect(projection).not.toContain('*'); expect(payloadReads()).toBe(0); // A receipt arriving during a streamed response must not mix a newer // result with the state and timestamps chosen for this delivery. - api.reportHostOperationStatus(api.host.id, second.id, { state: 'running', result: { phase: 'newer' } }); + api.reportHostOperationStatus(api.host.id, second.id, { + state: 'running', + result: { phase: 'newer' } + }); const reader = response.body!.getReader(); const first = await reader.read(); expect(first.done).toBe(false); @@ -107,29 +135,45 @@ describe('agent operation delivery', () => { expect(checkedNextPayload).toBe(true); const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); expect(body.operations).toHaveLength(2); - expect(body.operations[1]).toMatchObject({ id: second.id, state: 'accepted', result: { phase: 'selected' } }); + expect(body.operations[1]).toMatchObject({ + id: second.id, + state: 'accepted', + result: { phase: 'selected' } + }); expect(body.operations[0].payload).toEqual(JSON.parse(payload.toString('utf8'))); expect(body.operations[1].payload).toEqual(body.operations[0].payload); const other = api.createHost({ name: 'Other host' }); - expect(() => api.getHostOperationPayload(other.id, second.id)).toThrow('not found for this host'); + expect(() => api.getHostOperationPayload(other.id, second.id)).toThrow( + 'not found for this host' + ); }); it('retains health lanes, delivery rotation, and cancellation suppression through the streamed route', async () => { const api = await enrolled(); - const operations = Array.from({ length: 27 }, (_, index) => api.queueHostOperation(api.host.id, 'instance.restart', { index })); + const operations = Array.from({ length: 27 }, (_, index) => + api.queueHostOperation(api.host.id, 'instance.restart', { index }) + ); const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); const first = await (await api.GET({ request: api.request('?lane=write') })).json(); - expect(first.operations.map((operation: { id: string }) => operation.id)).toEqual(operations.slice(0, 25).map(operation => operation.id)); + expect(first.operations.map((operation: { id: string }) => operation.id)).toEqual( + operations.slice(0, 25).map((operation) => operation.id) + ); const second = await (await api.GET({ request: api.request('?lane=write') })).json(); - expect(second.operations.slice(0, 2).map((operation: { id: string }) => operation.id)).toEqual(operations.slice(25).map(operation => operation.id)); + expect( + second.operations.slice(0, 2).map((operation: { id: string }) => operation.id) + ).toEqual(operations.slice(25).map((operation) => operation.id)); const health = await (await api.GET({ request: api.request('?lane=health') })).json(); expect(health.operations).toEqual([inspect]); const create = api.queueHostOperation(api.host.id, 'instance.create', {}); - const cancellation = api.queueHostOperation(api.host.id, 'instance.cancel-create', { createOperationId: create.id }); + const cancellation = api.queueHostOperation(api.host.id, 'instance.cancel-create', { + createOperationId: create.id + }); const cancelled = await (await api.GET({ request: api.request('?lane=write') })).json(); expect(cancelled.operations[0].id).toBe(cancellation.id); - expect(cancelled.operations.some((operation: { id: string }) => operation.id === create.id)).toBe(false); + expect( + cancelled.operations.some((operation: { id: string }) => operation.id === create.id) + ).toBe(false); }); it('rechecks credentials after waking a long poll before sending work', async () => { diff --git a/tests/unit/agent-update-inventory.test.ts b/tests/unit/agent-update-inventory.test.ts index 3cf5140..8d0ce40 100644 --- a/tests/unit/agent-update-inventory.test.ts +++ b/tests/unit/agent-update-inventory.test.ts @@ -1,15 +1,38 @@ import { describe, expect, it } from 'vitest'; import { validateInventory } from '../../src/lib/server/hosts/index.js'; -const update = { enabled: true, state: 'updated', currentVersion: '1.2.0', targetVersion: '1.2.0', checkedAt: '2026-10-04T12:00:00Z' }; +const update = { + enabled: true, + state: 'updated', + currentVersion: '1.2.0', + targetVersion: '1.2.0', + checkedAt: '2026-10-04T12:00:00Z' +}; describe('agent update inventory', () => { it('accepts legacy inventory and only publishes recognized update display fields', () => { expect(validateInventory({ protocolVersion: 1 })).toEqual({ protocolVersion: 1 }); - expect(validateInventory({ agentUpdate: { ...update, credential: 'private', journal: { backups: 'local-path' } } }).agentUpdate).toEqual(update); + expect( + validateInventory({ + agentUpdate: { + ...update, + credential: 'private', + journal: { backups: 'local-path' } + } + }).agentUpdate + ).toEqual(update); }); - it.each([null, [], 'updated', { ...update, enabled: 'yes' }, { ...update, state: '', - published_at: '2026-07-27T12:00:00Z', draft: false, prerelease: false, - assets: [{ id: 200, name: 't6-gsc-utils.dll', size: content.length, digest: `sha256:${sha256}`, state: 'uploaded' }], + id: 100, + tag_name: 'v3.1.0', + name: 'Utilities 3.1', + body: 'Release notes ', + published_at: '2026-07-27T12:00:00Z', + draft: false, + prerelease: false, + assets: [ + { + id: 200, + name: 't6-gsc-utils.dll', + size: content.length, + digest: `sha256:${sha256}`, + state: 'uploaded' + } + ], ...overrides }); let fetchMock: ReturnType; @@ -18,38 +31,80 @@ beforeEach(async () => { fixture = await newFixture(); delete process.env.RCON_PASSWORD; - fetchMock = vi.fn(async (input: string | URL) => String(input).startsWith('https://api.github.com/') - ? Response.json(metadata()) : new Response(content)); + fetchMock = vi.fn(async (input: string | URL) => + String(input).startsWith('https://api.github.com/') + ? Response.json(metadata()) + : new Response(content) + ); vi.stubGlobal('fetch', fetchMock); }); -afterEach(async () => { vi.unstubAllGlobals(); await fixture.cleanup(); }); +afterEach(async () => { + vi.unstubAllGlobals(); + await fixture.cleanup(); +}); describe('GitHub release lookup', () => { it('supports repository, latest, tag and the supplied release-anchor URLs', async () => { - const { parseGithubReleaseUrl, getGithubRelease } = await import('../../src/lib/server/integrations/github-releases.js'); - expect(parseGithubReleaseUrl('https://github.com/alicealys/t6-gsc-utils/releases#release-v3.1.0')).toEqual({ repository: 'alicealys/t6-gsc-utils', tag: 'v3.1.0' }); - expect(parseGithubReleaseUrl('https://github.com/alicealys/t6-gsc-utils/releases/latest').tag).toBeNull(); + const { parseGithubReleaseUrl, getGithubRelease } = + await import('../../src/lib/server/integrations/github-releases.js'); + expect( + parseGithubReleaseUrl( + 'https://github.com/alicealys/t6-gsc-utils/releases#release-v3.1.0' + ) + ).toEqual({ repository: 'alicealys/t6-gsc-utils', tag: 'v3.1.0' }); + expect( + parseGithubReleaseUrl('https://github.com/alicealys/t6-gsc-utils/releases/latest').tag + ).toBeNull(); expect(parseGithubReleaseUrl('https://github.com/alicealys/t6-gsc-utils').tag).toBeNull(); - expect(parseGithubReleaseUrl('https://github.com/alicealys/t6-gsc-utils/releases/tag/release%2F3.1').tag).toBe('release/3.1'); + expect( + parseGithubReleaseUrl( + 'https://github.com/alicealys/t6-gsc-utils/releases/tag/release%2F3.1' + ).tag + ).toBe('release/3.1'); const release = await getGithubRelease(url); - expect(release).toMatchObject({ repository: 'alicealys/t6-gsc-utils', tag: 'v3.1.0', assets: [{ name: 't6-gsc-utils.dll' }] }); - expect(fetchMock.mock.calls[0][0]).toBe('https://api.github.com/repos/alicealys/t6-gsc-utils/releases/tags/v3.1.0'); + expect(release).toMatchObject({ + repository: 'alicealys/t6-gsc-utils', + tag: 'v3.1.0', + assets: [{ name: 't6-gsc-utils.dll' }] + }); + expect(fetchMock.mock.calls[0][0]).toBe( + 'https://api.github.com/repos/alicealys/t6-gsc-utils/releases/tags/v3.1.0' + ); }); - it.each(['http://github.com/a/b', 'https://github.com.evil.test/a/b', 'https://github.com@127.0.0.1/a/b', - 'https://github.com:444/a/b', 'https://github.com/a/b/issues/1', 'https://github.com/a/b/releases#unknown', - 'https://github.com/a/b/releases/tag/', 'https://github.com/a/b/releases/tag/v1%0A'])('rejects unsupported URL %s without fetching it', async (input) => { - const { getGithubRelease } = await import('../../src/lib/server/integrations/github-releases.js'); + it.each([ + 'http://github.com/a/b', + 'https://github.com.evil.test/a/b', + 'https://github.com@127.0.0.1/a/b', + 'https://github.com:444/a/b', + 'https://github.com/a/b/issues/1', + 'https://github.com/a/b/releases#unknown', + 'https://github.com/a/b/releases/tag/', + 'https://github.com/a/b/releases/tag/v1%0A' + ])('rejects unsupported URL %s without fetching it', async (input) => { + const { getGithubRelease } = + await import('../../src/lib/server/integrations/github-releases.js'); await expect(getGithubRelease(input)).rejects.toThrow(); expect(fetchMock).not.toHaveBeenCalled(); }); it('filters out archives, unsafe filenames and oversized assets; shares concurrent lookups', async () => { const normal = metadata().assets[0]; - fetchMock.mockImplementation(async () => Response.json(metadata({ assets: [normal, - { ...normal, id: 201, name: 'source.zip' }, { ...normal, id: 202, name: '../evil.dll' }, - { ...normal, id: 203, size: 65 * 1024 * 1024 }, { ...normal, id: 204, state: 'new' }] }))); - const { getGithubRelease } = await import('../../src/lib/server/integrations/github-releases.js'); + fetchMock.mockImplementation(async () => + Response.json( + metadata({ + assets: [ + normal, + { ...normal, id: 201, name: 'source.zip' }, + { ...normal, id: 202, name: '../evil.dll' }, + { ...normal, id: 203, size: 65 * 1024 * 1024 }, + { ...normal, id: 204, state: 'new' } + ] + }) + ) + ); + const { getGithubRelease } = + await import('../../src/lib/server/integrations/github-releases.js'); const [first, second] = await Promise.all([getGithubRelease(url), getGithubRelease(url)]); expect(first.assets).toHaveLength(1); expect(second).toEqual(first); @@ -58,7 +113,8 @@ describe('GitHub release lookup', () => { it('reports rate limits and retries failed lookups', async () => { fetchMock.mockResolvedValueOnce(new Response(null, { status: 403 })); - const { getGithubRelease } = await import('../../src/lib/server/integrations/github-releases.js'); + const { getGithubRelease } = + await import('../../src/lib/server/integrations/github-releases.js'); await expect(getGithubRelease(url)).rejects.toMatchObject({ status: 429 }); await expect(getGithubRelease(url)).resolves.toMatchObject({ tag: 'v3.1.0' }); }); @@ -66,17 +122,51 @@ describe('GitHub release lookup', () => { describe('GitHub DLL imports', () => { it('captures provenance and notes, leaves promotion untouched, and deduplicates concurrent imports', async () => { - const { importPluginRelease, pluginSource } = await import('../../src/lib/server/fleet/plugin-releases.js'); - const { getScript, getVersion, listVersions } = await import('../../src/lib/server/fleet/scripts.js'); - const imported = await importPluginRelease({ url, assetId: 200, name: 'Utilities', scopeType: 'game', scopeValue: 't6' }); + const { importPluginRelease, pluginSource } = + await import('../../src/lib/server/fleet/plugin-releases.js'); + const { getScript, getVersion, listVersions } = + await import('../../src/lib/server/fleet/scripts.js'); + const imported = await importPluginRelease({ + url, + assetId: 200, + name: 'Utilities', + scopeType: 'game', + scopeValue: 't6' + }); expect(imported.sha256).toBe(sha256); - expect(imported.betaUrl).toBe(`/betas?pluginId=${imported.pluginId}&versionId=${imported.versionId}`); - expect(getScript(imported.pluginId)).toMatchObject({ kind: 'plugin', target_rel_path: 't6-gsc-utils.dll', scope_value: 't6', current_version_id: null }); - expect(getVersion(imported.versionId)).toMatchObject({ release_version: 'v3.1.0', notes: metadata().body, state: 'untested' }); - expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(imported.versionId)).toEqual(content); - expect(pluginSource(imported.pluginId)).toMatchObject({ repository: 'alicealys/t6-gsc-utils', release_id: 100, asset_id: 200, release_url: url }); - const duplicates = await Promise.all([1, 2].map(() => importPluginRelease({ url, assetId: 200, pluginId: imported.pluginId }))); - expect(duplicates.every(r => r.alreadyImported && r.versionId === imported.versionId)).toBe(true); + expect(imported.betaUrl).toBe( + `/betas?pluginId=${imported.pluginId}&versionId=${imported.versionId}` + ); + expect(getScript(imported.pluginId)).toMatchObject({ + kind: 'plugin', + target_rel_path: 't6-gsc-utils.dll', + scope_value: 't6', + current_version_id: null + }); + expect(getVersion(imported.versionId)).toMatchObject({ + release_version: 'v3.1.0', + notes: metadata().body, + state: 'untested' + }); + expect( + (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent( + imported.versionId + ) + ).toEqual(content); + expect(pluginSource(imported.pluginId)).toMatchObject({ + repository: 'alicealys/t6-gsc-utils', + release_id: 100, + asset_id: 200, + release_url: url + }); + const duplicates = await Promise.all( + [1, 2].map(() => + importPluginRelease({ url, assetId: 200, pluginId: imported.pluginId }) + ) + ); + expect( + duplicates.every((r) => r.alreadyImported && r.versionId === imported.versionId) + ).toBe(true); expect(listVersions(imported.pluginId)).toHaveLength(1); const { GET } = await import('../../src/routes/api/scripts/[id]/+server.js'); const response = await (await GET({ params: { id: String(imported.pluginId) } })).json(); @@ -86,100 +176,205 @@ describe('GitHub DLL imports', () => { it('follows GitHub asset redirects and verifies the bytes', async () => { fetchMock.mockImplementation(async (input) => { - if (String(input).startsWith('https://api.github.com/')) return Response.json(metadata()); - if (String(input).startsWith('https://github.com/')) return new Response(null, { status: 302, headers: { location: 'https://release-assets.githubusercontent.com/blob/test' } }); + if (String(input).startsWith('https://api.github.com/')) + return Response.json(metadata()); + if (String(input).startsWith('https://github.com/')) + return new Response(null, { + status: 302, + headers: { location: 'https://release-assets.githubusercontent.com/blob/test' } + }); return new Response(content); }); - const { importPluginRelease } = await import('../../src/lib/server/fleet/plugin-releases.js'); - await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).resolves.toMatchObject({ sha256 }); + const { importPluginRelease } = + await import('../../src/lib/server/fleet/plugin-releases.js'); + await expect( + importPluginRelease({ url, assetId: 200, name: 'Utilities' }) + ).resolves.toMatchObject({ sha256 }); expect(fetchMock).toHaveBeenCalledTimes(3); }); - it.each(['http://127.0.0.1/internal', 'https://evil.test/file.dll', 'https://user:pass@github.com/file.dll'])('blocks unsafe download redirects to %s', async (location) => { - fetchMock.mockImplementation(async (input) => String(input).startsWith('https://api.github.com/') - ? Response.json(metadata()) : new Response(null, { status: 302, headers: { location } })); - const { importPluginRelease } = await import('../../src/lib/server/fleet/plugin-releases.js'); + it.each([ + 'http://127.0.0.1/internal', + 'https://evil.test/file.dll', + 'https://user:pass@github.com/file.dll' + ])('blocks unsafe download redirects to %s', async (location) => { + fetchMock.mockImplementation(async (input) => + String(input).startsWith('https://api.github.com/') + ? Response.json(metadata()) + : new Response(null, { status: 302, headers: { location } }) + ); + const { importPluginRelease } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const { listScripts } = await import('../../src/lib/server/fleet/scripts.js'); - await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow(/unsupported host/); + await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow( + /unsupported host/ + ); expect(fetchMock).toHaveBeenCalledTimes(2); expect(listScripts()).toHaveLength(0); }); it('rejects checksum mismatches and truncated downloads without creating a plugin', async () => { - const { importPluginRelease } = await import('../../src/lib/server/fleet/plugin-releases.js'); + const { importPluginRelease } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const { listScripts } = await import('../../src/lib/server/fleet/scripts.js'); - fetchMock.mockImplementation(async (input) => String(input).startsWith('https://api.github.com/') - ? Response.json(metadata()) : new Response(Buffer.alloc(content.length))); - await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow(/checksum/); - fetchMock.mockImplementation(async (input) => String(input).startsWith('https://api.github.com/') - ? Response.json(metadata()) : new Response(Buffer.alloc(1))); - await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow(/size differs/); + fetchMock.mockImplementation(async (input) => + String(input).startsWith('https://api.github.com/') + ? Response.json(metadata()) + : new Response(Buffer.alloc(content.length)) + ); + await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow( + /checksum/ + ); + fetchMock.mockImplementation(async (input) => + String(input).startsWith('https://api.github.com/') + ? Response.json(metadata()) + : new Response(Buffer.alloc(1)) + ); + await expect(importPluginRelease({ url, assetId: 200, name: 'Utilities' })).rejects.toThrow( + /size differs/ + ); expect(listScripts()).toHaveLength(0); }); it('rejects a missing asset and refuses to import into non-plugins or active betas', async () => { - const { importPluginRelease } = await import('../../src/lib/server/fleet/plugin-releases.js'); + const { importPluginRelease } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); - await expect(importPluginRelease({ url, assetId: 999, name: 'Utilities' })).rejects.toThrow(/no longer available/); + await expect(importPluginRelease({ url, assetId: 999, name: 'Utilities' })).rejects.toThrow( + /no longer available/ + ); const script = createScript({ name: 'Script', targetRelPath: 's.gsc', content }); - await expect(importPluginRelease({ url, assetId: 200, pluginId: script.script.id })).rejects.toMatchObject({ status: 404 }); - const plugin = createScript({ name: 'Plugin', kind: 'plugin', targetRelPath: 'p.dll', content }); - const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'version', 'running')") + await expect( + importPluginRelease({ url, assetId: 200, pluginId: script.script.id }) + ).rejects.toMatchObject({ status: 404 }); + const plugin = createScript({ + name: 'Plugin', + kind: 'plugin', + targetRelPath: 'p.dll', + content + }); + const beta = db() + .prepare( + "INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'version', 'running')" + ) .run(plugin.script.id, plugin.version.id); - db().prepare('INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, ?)') + db() + .prepare( + 'INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, ?)' + ) .run(beta.lastInsertRowid, plugin.script.id, plugin.version.id, 'plugin'); - await expect(importPluginRelease({ url, assetId: 200, pluginId: plugin.script.id })).rejects.toMatchObject({ status: 409 }); + await expect( + importPluginRelease({ url, assetId: 200, pluginId: plugin.script.id }) + ).rejects.toMatchObject({ status: 409 }); }); }); describe('plugin update discovery', () => { it('finds newer stable releases, imports into the same plugin, and distinguishes imported from promoted', async () => { - const { importPluginRelease, checkPluginUpdate } = await import('../../src/lib/server/fleet/plugin-releases.js'); + const { importPluginRelease, checkPluginUpdate } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const { getScript, listVersions } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const first = await importPluginRelease({ url, assetId: 200, name: 'Utilities' }); - const next = metadata({ id: 101, tag_name: 'v3.1.1', published_at: '2026-08-11T12:00:00Z', - assets: [{ ...metadata().assets[0], id: 201 }] }); - fetchMock.mockImplementation(async (input) => String(input).startsWith('https://api.github.com/') ? Response.json(next) : new Response(content)); + const next = metadata({ + id: 101, + tag_name: 'v3.1.1', + published_at: '2026-08-11T12:00:00Z', + assets: [{ ...metadata().assets[0], id: 201 }] + }); + fetchMock.mockImplementation(async (input) => + String(input).startsWith('https://api.github.com/') + ? Response.json(next) + : new Response(content) + ); const update = await checkPluginUpdate(first.pluginId); expect(update).toMatchObject({ status: 'available', release: { tag: 'v3.1.1' } }); - const second = await importPluginRelease({ url: update.release!.url, assetId: 201, pluginId: first.pluginId }); + const second = await importPluginRelease({ + url: update.release!.url, + assetId: 201, + pluginId: first.pluginId + }); expect(listVersions(first.pluginId)).toHaveLength(2); expect(getScript(first.pluginId)!.current_version_id).toBeNull(); - expect(await checkPluginUpdate(first.pluginId)).toMatchObject({ status: 'imported', versionId: second.versionId }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(second.versionId, first.pluginId); - expect(await checkPluginUpdate(first.pluginId)).toMatchObject({ status: 'current', versionId: second.versionId }); + expect(await checkPluginUpdate(first.pluginId)).toMatchObject({ + status: 'imported', + versionId: second.versionId + }); + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(second.versionId, first.pluginId); + expect(await checkPluginUpdate(first.pluginId)).toMatchObject({ + status: 'current', + versionId: second.versionId + }); }); it('does not recommend an older stable release after importing a newer prerelease', async () => { - const { importPluginRelease, checkPluginUpdate } = await import('../../src/lib/server/fleet/plugin-releases.js'); + const { importPluginRelease, checkPluginUpdate } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const stable = await importPluginRelease({ url, assetId: 200, name: 'Utilities' }); - fetchMock.mockImplementation(async (input) => String(input).startsWith('https://api.github.com/') ? Response.json(metadata({ id: 105, tag_name: 'v4.0-rc1', prerelease: true, published_at: '2026-09-01T12:00:00Z' })) : new Response(content)); - const imported = await importPluginRelease({ url: url.replace('v3.1.0', 'v4.0-rc1'), assetId: 200, pluginId: stable.pluginId }); + fetchMock.mockImplementation(async (input) => + String(input).startsWith('https://api.github.com/') + ? Response.json( + metadata({ + id: 105, + tag_name: 'v4.0-rc1', + prerelease: true, + published_at: '2026-09-01T12:00:00Z' + }) + ) + : new Response(content) + ); + const imported = await importPluginRelease({ + url: url.replace('v3.1.0', 'v4.0-rc1'), + assetId: 200, + pluginId: stable.pluginId + }); fetchMock.mockImplementation(async () => Response.json(metadata())); expect(await checkPluginUpdate(imported.pluginId)).toMatchObject({ status: 'ahead' }); }); it('reports untracked and GitHub failures per plugin without breaking the list', async () => { - const { importPluginRelease } = await import('../../src/lib/server/fleet/plugin-releases.js'); + const { importPluginRelease } = + await import('../../src/lib/server/fleet/plugin-releases.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); await importPluginRelease({ url, assetId: 200, name: 'Imported' }); createScript({ name: 'Manual', kind: 'plugin', targetRelPath: 'manual.dll', content }); fetchMock.mockResolvedValue(new Response(null, { status: 404 })); const { GET } = await import('../../src/routes/api/plugins/updates/+server.js'); - const result = await (await GET({ url: new URL('http://localhost/api/plugins/updates') })).json(); - expect(result.updates.map((u: { status: string }) => u.status)).toEqual(['error', 'untracked']); + const result = await ( + await GET({ url: new URL('http://localhost/api/plugins/updates') }) + ).json(); + expect(result.updates.map((u: { status: string }) => u.status)).toEqual([ + 'error', + 'untracked' + ]); }); it('serves the preview and import endpoints and validates bad requests', async () => { const { GET } = await import('../../src/routes/api/plugins/github/release/+server.js'); const { POST } = await import('../../src/routes/api/plugins/github/import/+server.js'); - const preview = await GET({ url: new URL(`http://localhost/api/plugins/github/release?url=${encodeURIComponent(url)}`) }); + const preview = await GET({ + url: new URL( + `http://localhost/api/plugins/github/release?url=${encodeURIComponent(url)}` + ) + }); expect((await preview.json()).release.assets[0].id).toBe(200); - const request = (body: unknown) => new Request('http://localhost/api/plugins/github/import', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const request = (body: unknown) => + new Request('http://localhost/api/plugins/github/import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); expect((await POST({ request: request(null) })).status).toBe(400); - expect((await POST({ request: request({ url, assetId: 200, name: 'Test', scopeType: 'other' }) })).status).toBe(400); + expect( + ( + await POST({ + request: request({ url, assetId: 200, name: 'Test', scopeType: 'other' }) + }) + ).status + ).toBe(400); const response = await POST({ request: request({ url, assetId: 200, name: 'Test' }) }); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ ok: true, sha256, releaseVersion: 'v3.1.0' }); diff --git a/tests/unit/health-observations.test.ts b/tests/unit/health-observations.test.ts index f74b2ed..f29869b 100644 --- a/tests/unit/health-observations.test.ts +++ b/tests/unit/health-observations.test.ts @@ -7,28 +7,43 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: const server = 'managed'; const now = Date.parse('2026-10-05T16:00:00Z'); const healthy = { ok: true as const, state: 'running' }; -const playing = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }; +const playing = { + ok: true as const, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } +}; let fixture: Fixture; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); + vi.resetModules(); + vi.clearAllMocks(); fixture = await newFixture([]); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(now); +}); +afterEach(async () => { + vi.useRealTimers(); + await fixture.cleanup(); + vi.restoreAllMocks(); }); -afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); - database.prepare('INSERT INTO servers(name,created_at,game_port) VALUES(?,?,28960)') + database + .prepare('INSERT INTO servers(name,created_at,game_port) VALUES(?,?,28960)') .run(server, new Date(now).toISOString()); vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const inspect = vi.mocked(nativeHealth).mockResolvedValue(healthy); const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing); - return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') }; + return { + database, + inspect, + rcon, + ...(await import('../../src/lib/server/fleet/health-observations.js')) + }; } async function native(api: Awaited>) { @@ -36,26 +51,40 @@ async function native(api: Awaited>) { const host = hosts.createHost({ name: 'Monitoring host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); const instanceId = randomUUID(); - api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,'library','image',28960,?)`).run(instanceId, server, host.id, new Date(now).toISOString()); + api.database + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,'library','image',28960,?)` + ) + .run(instanceId, server, host.id, new Date(now).toISOString()); return { host, instanceId, hosts }; } function deferred() { let resolve!: (value: T) => void; - const promise = new Promise(done => { resolve = done; }); + const promise = new Promise((done) => { + resolve = done; + }); return { promise, resolve }; } describe('shared monitoring observations', () => { it('retains failures for watchdog consumption without caching them as reusable probes', async () => { const api = await setup(); - api.rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' }); + api.rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'No reply' + }); const publish = api.monitoringStatusWriter(server); publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server)); expect(api.monitoringStatus(server)).toMatchObject({ container: { observedAt: new Date(now).toISOString(), result: healthy }, - game: { observedAt: new Date(now).toISOString(), result: { ok: false, reason: 'timeout' } } + game: { + observedAt: new Date(now).toISOString(), + result: { ok: false, reason: 'timeout' } + } }); vi.setSystemTime(now + 40_000); expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString()); @@ -67,20 +96,37 @@ describe('shared monitoring observations', () => { expect(api.monitoringStatus(server)).toBeNull(); }); - it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)('invalidates a completed status snapshot after %s changes', async change => { - const api = await setup(); - const host = await native(api); - const publish = api.monitoringStatusWriter(server); - publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server)); - expect(api.monitoringStatus(server)).not.toBeNull(); - if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); - if (change === 'generation') api.database.prepare('UPDATE native_instances SET generation=2').run(); - if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); - if (change === 'endpoint') api.database.prepare('UPDATE native_instances SET port=28961').run(); - if (change === 'manual') api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); - if (change === 'offline') api.database.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run(new Date(now - 300_000).toISOString(), host.host.id); - expect(api.monitoringStatus(server)).toBeNull(); - }); + it.each(['registration', 'generation', 'lifecycle', 'endpoint', 'manual', 'offline'] as const)( + 'invalidates a completed status snapshot after %s changes', + async (change) => { + const api = await setup(); + const host = await native(api); + const publish = api.monitoringStatusWriter(server); + publish(await api.inspectForMonitoring(server), await api.rconForMonitoring(server)); + expect(api.monitoringStatus(server)).not.toBeNull(); + if (change === 'registration') + api.database + .prepare('UPDATE servers SET created_at=? WHERE name=?') + .run(new Date(now + 1).toISOString(), server); + if (change === 'generation') + api.database.prepare('UPDATE native_instances SET generation=2').run(); + if (change === 'lifecycle') + api.database + .prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)') + .run(server, new Date(now + 1).toISOString()); + if (change === 'endpoint') + api.database.prepare('UPDATE native_instances SET port=28961').run(); + if (change === 'manual') + api.database + .prepare('UPDATE servers SET container_checked_at=? WHERE name=?') + .run(new Date(now + 1).toISOString(), server); + if (change === 'offline') + api.database + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run(new Date(now - 300_000).toISOString(), host.host.id); + expect(api.monitoringStatus(server)).toBeNull(); + } + ); it('does not publish a status pass across an instance replacement', async () => { const api = await setup(); @@ -93,17 +139,23 @@ describe('shared monitoring observations', () => { expect(api.monitoringStatus(server)).toBeNull(); }); - it.each(['inspect', 'rcon'] as const)('invalidates the routine snapshot when a forced %s check starts', async kind => { - const api = await setup(); - const publish = api.monitoringStatusWriter(server); - const container = await api.inspectForMonitoring(server); - const game = await api.rconForMonitoring(server); - publish(container, game); - expect(api.monitoringStatus(server)).not.toBeNull(); - await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)(server, true); - publish(container, game); - expect(api.monitoringStatus(server)).toBeNull(); - }); + it.each(['inspect', 'rcon'] as const)( + 'invalidates the routine snapshot when a forced %s check starts', + async (kind) => { + const api = await setup(); + const publish = api.monitoringStatusWriter(server); + const container = await api.inspectForMonitoring(server); + const game = await api.rconForMonitoring(server); + publish(container, game); + expect(api.monitoringStatus(server)).not.toBeNull(); + await (kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring)( + server, + true + ); + publish(container, game); + expect(api.monitoringStatus(server)).toBeNull(); + } + ); it('excludes old game evidence even when the container sample is fresh', async () => { const api = await setup(); @@ -116,19 +168,22 @@ describe('shared monitoring observations', () => { expect(api.rcon).toHaveBeenCalledTimes(1); }); - it.each(['inspect', 'rcon'] as const)('reuses %s results until 30 seconds after request start without refreshing their timestamp', async kind => { - const api = await setup(); - const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; - const request = kind === 'inspect' ? api.inspect : api.rcon; - const first = await call(server); - expect(first.observedAt).toBe(new Date(now).toISOString()); - vi.setSystemTime(now + 29_999); - expect(await call(server)).toEqual(first); - expect(request).toHaveBeenCalledTimes(1); - vi.setSystemTime(now + 30_000); - expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString()); - expect(request).toHaveBeenCalledTimes(2); - }); + it.each(['inspect', 'rcon'] as const)( + 'reuses %s results until 30 seconds after request start without refreshing their timestamp', + async (kind) => { + const api = await setup(); + const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; + const request = kind === 'inspect' ? api.inspect : api.rcon; + const first = await call(server); + expect(first.observedAt).toBe(new Date(now).toISOString()); + vi.setSystemTime(now + 29_999); + expect(await call(server)).toEqual(first); + expect(request).toHaveBeenCalledTimes(1); + vi.setSystemTime(now + 30_000); + expect((await call(server)).observedAt).toBe(new Date(now + 30_000).toISOString()); + expect(request).toHaveBeenCalledTimes(2); + } + ); it('shares overlapping monitoring requests, but keeps inspection and RCON separate', async () => { const api = await setup(); @@ -140,7 +195,9 @@ describe('shared monitoring observations', () => { expect(api.rcon).toHaveBeenCalledTimes(1); vi.setSystemTime(now + 5000); pending.resolve(healthy); - expect(await Promise.all(observations)).toEqual(Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() })); + expect(await Promise.all(observations)).toEqual( + Array(2).fill({ result: healthy, observedAt: new Date(now + 5000).toISOString() }) + ); }); it('timestamps a slow result on completion without making it reusable after its request expires', async () => { @@ -151,7 +208,10 @@ describe('shared monitoring observations', () => { await Promise.resolve(); vi.setSystemTime(now + 31_000); pending.resolve(healthy); - expect(await observation).toEqual({ result: healthy, observedAt: new Date(now + 31_000).toISOString() }); + expect(await observation).toEqual({ + result: healthy, + observedAt: new Date(now + 31_000).toISOString() + }); await api.inspectForMonitoring(server); expect(api.inspect).toHaveBeenCalledTimes(2); }); @@ -165,7 +225,9 @@ describe('shared monitoring observations', () => { vi.setSystemTime(now + 20_000); pending.resolve(healthy); const completed = await observation; - api.database.prepare('UPDATE servers SET container_checked_at=? WHERE name=?').run(completed.observedAt, server); + api.database + .prepare('UPDATE servers SET container_checked_at=? WHERE name=?') + .run(completed.observedAt, server); vi.setSystemTime(now + 25_000); expect(await api.inspectForMonitoring(server)).toEqual(completed); expect(api.inspect).toHaveBeenCalledTimes(1); @@ -211,22 +273,40 @@ describe('shared monitoring observations', () => { await Promise.resolve(); vi.setSystemTime(now + 1000); const latest = await api.inspectForMonitoring(server, true); - old.resolve(healthy); await first; + old.resolve(healthy); + await first; expect(await api.inspectForMonitoring(server)).toEqual(latest); expect(api.inspect).toHaveBeenCalledTimes(2); }); - it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)('invalidates cached observations after a %s change', async change => { - const api = await setup(); - await api.inspectForMonitoring(server); - if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date(now + 1).toISOString(), server); - if (change === 'lifecycle') api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); - if (change === 'desired state') api.database.prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(server); - if (change === 'stop') api.database.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) - VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')`).run(server, new Date(now).toISOString(), new Date(now + 1).toISOString()); - await api.inspectForMonitoring(server); - expect(api.inspect).toHaveBeenCalledTimes(2); - }); + it.each(['registration', 'lifecycle', 'desired state', 'stop'] as const)( + 'invalidates cached observations after a %s change', + async (change) => { + const api = await setup(); + await api.inspectForMonitoring(server); + if (change === 'registration') + api.database + .prepare('UPDATE servers SET created_at=? WHERE name=?') + .run(new Date(now + 1).toISOString(), server); + if (change === 'lifecycle') + api.database + .prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)') + .run(server, new Date(now + 1).toISOString()); + if (change === 'desired state') + api.database + .prepare("UPDATE servers SET desired_state='stopped' WHERE name=?") + .run(server); + if (change === 'stop') + api.database + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) + VALUES('stop',?,'stop','awaiting-health','Stopping',?,?,'ui')` + ) + .run(server, new Date(now).toISOString(), new Date(now + 1).toISOString()); + await api.inspectForMonitoring(server); + expect(api.inspect).toHaveBeenCalledTimes(2); + } + ); it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => { const api = await setup(); @@ -238,7 +318,8 @@ describe('shared monitoring observations', () => { await Promise.resolve(); api.database.prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); const current = await api.inspectForMonitoring(server); - pending.resolve(healthy); await rejected; + pending.resolve(healthy); + await rejected; expect(await api.inspectForMonitoring(server)).toEqual(current); expect(api.inspect).toHaveBeenCalledTimes(2); }); @@ -250,8 +331,11 @@ describe('shared monitoring observations', () => { const old = api.rconForMonitoring(server); const rejected = expect(old).rejects.toThrow('Server changed'); await Promise.resolve(); - api.database.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, new Date(now + 1).toISOString()); - pending.resolve(playing); await rejected; + api.database + .prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)') + .run(server, new Date(now + 1).toISOString()); + pending.resolve(playing); + await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); @@ -264,8 +348,11 @@ describe('shared monitoring observations', () => { const rejected = expect(old).rejects.toBeInstanceOf(api.MonitoringChangedError); await Promise.resolve(); vi.setSystemTime(now + 1000); - api.database.prepare('UPDATE servers SET last_checked_at=? WHERE name=?').run(new Date().toISOString(), server); - pending.resolve(playing); await rejected; + api.database + .prepare('UPDATE servers SET last_checked_at=? WHERE name=?') + .run(new Date().toISOString(), server); + pending.resolve(playing); + await rejected; await api.rconForMonitoring(server); expect(api.rcon).toHaveBeenCalledTimes(2); }); @@ -291,6 +378,7 @@ describe('shared monitoring observations', () => { const rejected = expect(observation).rejects.toThrow('Host agent is offline'); await Promise.resolve(); api.database.prepare('UPDATE ec_hosts SET last_heartbeat=NULL WHERE id=?').run(host.id); - pending.resolve(healthy); await rejected; + pending.resolve(healthy); + await rejected; }); }); diff --git a/tests/unit/historical-server-identity.test.ts b/tests/unit/historical-server-identity.test.ts index b409f7e..5d1e1af 100644 --- a/tests/unit/historical-server-identity.test.ts +++ b/tests/unit/historical-server-identity.test.ts @@ -6,7 +6,9 @@ it('retains deleted identities without linking to a replacement using the same s const props = { name: 't5_zombie_theater_1', href: '/servers/t5_zombie_theater_1' }; const live = render(HistoricalServerIdentity, { props }).body; expect(live).toContain('href="/servers/t5_zombie_theater_1"'); - const deleted = render(HistoricalServerIdentity, { props: { ...props, deletedAt: '2026-09-27T12:00:00Z' } }).body; + const deleted = render(HistoricalServerIdentity, { + props: { ...props, deletedAt: '2026-09-27T12:00:00Z' } + }).body; expect(deleted).toContain('t5_zombie_theater_1'); expect(deleted).toContain('Deleted'); expect(deleted).not.toContain('href='); diff --git a/tests/unit/hooks-server.test.ts b/tests/unit/hooks-server.test.ts index 2685482..0dd607f 100644 --- a/tests/unit/hooks-server.test.ts +++ b/tests/unit/hooks-server.test.ts @@ -4,13 +4,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('../../src/lib/server/db/migrate.js', () => ({ migrate: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ startWorker: vi.fn() })); vi.mock('../../src/lib/server/provisioning/history.js', () => ({ interruptDeployments: vi.fn() })); -vi.mock('../../src/lib/server/auth/guard.js', () => ({ authenticationHandle: vi.fn(async () => new Response('OK')) })); +vi.mock('../../src/lib/server/auth/guard.js', () => ({ + authenticationHandle: vi.fn(async () => new Response('OK')) +})); vi.mock('../../src/lib/server/pipeline/bus.js', () => ({ invalidateFleet: vi.fn() })); vi.mock('../../src/lib/server/performance.js', () => ({ startPerformanceMonitor: vi.fn() })); let originalListeners = process.listeners('unhandledRejection'); beforeEach(() => { - vi.resetModules(); vi.resetAllMocks(); + vi.resetModules(); + vi.resetAllMocks(); originalListeners = process.listeners('unhandledRejection'); }); afterEach(() => { @@ -22,21 +25,36 @@ afterEach(() => { describe('controller error handling', () => { it.each(['migration', 'deployment recovery', 'worker startup', 'performance monitor'] as const)( - 'retries failed %s before serving requests and boots only once after recovery', async phase => { + 'retries failed %s before serving requests and boots only once after recovery', + async (phase) => { const { handle } = await import('../../src/hooks.server.js'); const { migrate } = await import('../../src/lib/server/db/migrate.js'); - const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); + const { interruptDeployments } = + await import('../../src/lib/server/provisioning/history.js'); const { startWorker } = await import('../../src/lib/server/pipeline/worker.js'); const { startPerformanceMonitor } = await import('../../src/lib/server/performance.js'); const { authenticationHandle } = await import('../../src/lib/server/auth/guard.js'); const steps = [migrate, interruptDeployments, startWorker, startPerformanceMonitor]; - const failedStep = ['migration', 'deployment recovery', 'worker startup', 'performance monitor'].indexOf(phase); + const failedStep = [ + 'migration', + 'deployment recovery', + 'worker startup', + 'performance monitor' + ].indexOf(phase); const failure = new Error('database or disk is full'); vi.mocked(steps[failedStep]) - .mockImplementationOnce(() => { throw failure; }) - .mockImplementationOnce(() => { throw failure; }); + .mockImplementationOnce(() => { + throw failure; + }) + .mockImplementationOnce(() => { + throw failure; + }); const input = { - event: { request: new Request('http://localhost/'), route: { id: '/' }, locals: {} }, + event: { + request: new Request('http://localhost/'), + route: { id: '/' }, + locals: {} + }, resolve: vi.fn() } as unknown as Parameters[0]; @@ -54,12 +72,18 @@ describe('controller error handling', () => { for (const [index, step] of steps.entries()) { expect(step).toHaveBeenCalledTimes(index <= failedStep ? 3 : 1); const lastCall = vi.mocked(step).mock.invocationCallOrder.at(-1)!; - expect(lastCall).toBeLessThan(vi.mocked(authenticationHandle).mock.invocationCallOrder[0]); + expect(lastCall).toBeLessThan( + vi.mocked(authenticationHandle).mock.invocationCallOrder[0] + ); if (index > 0) { - expect(lastCall).toBeGreaterThan(vi.mocked(steps[index - 1]).mock.invocationCallOrder.at(-1)!); + expect(lastCall).toBeGreaterThan( + vi.mocked(steps[index - 1]).mock.invocationCallOrder.at(-1)! + ); } } - const listeners = process.listeners('unhandledRejection').filter(listener => !originalListeners.includes(listener)); + const listeners = process + .listeners('unhandledRejection') + .filter((listener) => !originalListeners.includes(listener)); expect(listeners).toHaveLength(1); } ); @@ -77,11 +101,15 @@ describe('controller error handling', () => { await handle(input); await handle(input); - const listeners = process.listeners('unhandledRejection').filter(listener => !originalListeners.includes(listener)); + const listeners = process + .listeners('unhandledRejection') + .filter((listener) => !originalListeners.includes(listener)); expect(listeners).toHaveLength(1); expect(startWorker).toHaveBeenCalledTimes(1); const registration = on.mock.calls.findIndex(([event]) => event === 'unhandledRejection'); - expect(on.mock.invocationCallOrder[registration]).toBeLessThan(vi.mocked(startWorker).mock.invocationCallOrder[0]); + expect(on.mock.invocationCallOrder[registration]).toBeLessThan( + vi.mocked(startWorker).mock.invocationCallOrder[0] + ); const failure = new Error('unexpected background failure'); listeners[0](failure, Promise.resolve()); diff --git a/tests/unit/host-agent-lanes.test.ts b/tests/unit/host-agent-lanes.test.ts index 4076257..3ef0863 100644 --- a/tests/unit/host-agent-lanes.test.ts +++ b/tests/unit/host-agent-lanes.test.ts @@ -3,43 +3,91 @@ import { hostAgentLaneState, type HostAgentLane } from '../../src/lib/hosts.js'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -const inventory = { protocolVersion:1, dockerRunning:true, directoriesWritable:true, storageAvailable:100 * 1024 ** 3, - runtimeImages:['image'], libraries:[{id:'library',state:'ready'}] }; -const lane: HostAgentLane = {name:'health-1',alive:true,busy:false,lastPollAt:'2026-10-06T12:00:00Z',pollAgeSeconds:5}; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +const inventory = { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: ['image'], + libraries: [{ id: 'library', state: 'ready' }] +}; +const lane: HostAgentLane = { + name: 'health-1', + alive: true, + busy: false, + lastPollAt: '2026-10-06T12:00:00Z', + pollAgeSeconds: 5 +}; +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); it('flags a stopped or stalled health worker despite a live heartbeat and recovers on fresh polls', async () => { const api = await import('../../src/lib/server/hosts/index.js'); - const host = api.createHost({name:'Worker host'}); - api.enrollHost(api.createEnrollment(host.id).token,inventory); - for (const broken of [{...lane,alive:false},{...lane,busy:true,pollAgeSeconds:91}]) { - const updated = api.heartbeat(host.id,{...inventory,agentLanes:[broken,{...lane,name:'health-2'}]}); - expect(updated).toMatchObject({connected:true,connectionState:'online',readiness:'degraded'}); + const host = api.createHost({ name: 'Worker host' }); + api.enrollHost(api.createEnrollment(host.id).token, inventory); + for (const broken of [ + { ...lane, alive: false }, + { ...lane, busy: true, pollAgeSeconds: 91 } + ]) { + const updated = api.heartbeat(host.id, { + ...inventory, + agentLanes: [broken, { ...lane, name: 'health-2' }] + }); + expect(updated).toMatchObject({ + connected: true, + connectionState: 'online', + readiness: 'degraded' + }); expect(updated.readinessReasons[0]).toContain('health check 1 worker'); // Diagnostics must not prevent health polls from recovering the worker. - const check = api.queueHostOperation(host.id,'instance.inspect',{}); - expect(api.pendingHostOperations(host.id,'health').map(op=>op.id)).toContain(check.id); + const check = api.queueHostOperation(host.id, 'instance.inspect', {}); + expect(api.pendingHostOperations(host.id, 'health').map((op) => op.id)).toContain(check.id); } - expect(api.heartbeat(host.id,{...inventory,agentLanes:[lane]}).readiness).toBe('ready'); + expect(api.heartbeat(host.id, { ...inventory, agentLanes: [lane] }).readiness).toBe('ready'); // Older agents remain compatible without diagnostics. - expect(api.heartbeat(host.id,inventory).readiness).toBe('ready'); + expect(api.heartbeat(host.id, inventory).readiness).toBe('ready'); }); it('allows long reads and mutations while detecting stalled idle workers and missing initial polls', () => { - expect(hostAgentLaneState({...lane,lastPollAt:null})).toBe('starting'); - expect(hostAgentLaneState({...lane,lastPollAt:null,pollAgeSeconds:91})).toBe('stalled'); - expect(hostAgentLaneState({...lane,busy:true,pollAgeSeconds:91})).toBe('stalled'); - for (const name of ['read','write'] as const) { - expect(hostAgentLaneState({...lane,name,busy:true,pollAgeSeconds:3600})).toBe('busy'); - expect(hostAgentLaneState({...lane,name,busy:false,pollAgeSeconds:91})).toBe('stalled'); - expect(hostAgentLaneState({...lane,name,alive:false,busy:true})).toBe('stopped'); + expect(hostAgentLaneState({ ...lane, lastPollAt: null })).toBe('starting'); + expect(hostAgentLaneState({ ...lane, lastPollAt: null, pollAgeSeconds: 91 })).toBe('stalled'); + expect(hostAgentLaneState({ ...lane, busy: true, pollAgeSeconds: 91 })).toBe('stalled'); + for (const name of ['read', 'write'] as const) { + expect(hostAgentLaneState({ ...lane, name, busy: true, pollAgeSeconds: 3600 })).toBe( + 'busy' + ); + expect(hostAgentLaneState({ ...lane, name, busy: false, pollAgeSeconds: 91 })).toBe( + 'stalled' + ); + expect(hostAgentLaneState({ ...lane, name, alive: false, busy: true })).toBe('stopped'); } }); it('sanitizes lane diagnostics without rejecting a heartbeat for one malformed sample', async () => { const { validateInventory } = await import('../../src/lib/server/hosts/index.js'); - expect(validateInventory({...inventory,agentLanes:[null,{...lane,secret:'omit'},{...lane},{...lane,name:'write',pollAgeSeconds:-1}]}).agentLanes) - .toEqual([lane]); - expect(validateInventory({...inventory,agentLanes:[{...lane,pollAgeSeconds:'stale'},{...lane,lastPollAt:'invalid'}]}).agentLanes).toEqual([]); + expect( + validateInventory({ + ...inventory, + agentLanes: [ + null, + { ...lane, secret: 'omit' }, + { ...lane }, + { ...lane, name: 'write', pollAgeSeconds: -1 } + ] + }).agentLanes + ).toEqual([lane]); + expect( + validateInventory({ + ...inventory, + agentLanes: [ + { ...lane, pollAgeSeconds: 'stale' }, + { ...lane, lastPollAt: 'invalid' } + ] + }).agentLanes + ).toEqual([]); }); diff --git a/tests/unit/host-agent-release.test.ts b/tests/unit/host-agent-release.test.ts index d8414dc..a9fae5e 100644 --- a/tests/unit/host-agent-release.test.ts +++ b/tests/unit/host-agent-release.test.ts @@ -6,19 +6,33 @@ vi.mock('node:fs/promises', () => ({ readFile: files.readFile })); import { agentReleaseAsset } from '../../src/lib/server/hosts/releases.js'; import { GET } from '../../src/routes/api/agent/release/[file]/+server.js'; -beforeEach(() => { files.readFile.mockReset(); vi.stubEnv('NODE_ENV', 'production'); }); -afterEach(() => { vi.unstubAllEnvs(); }); +beforeEach(() => { + files.readFile.mockReset(); + vi.stubEnv('NODE_ENV', 'production'); +}); +afterEach(() => { + vi.unstubAllEnvs(); +}); describe('public agent release artifacts', () => { it('keeps every development artifact on the fresh static release when an older build also exists', async () => { vi.stubEnv('NODE_ENV', 'development'); - files.readFile.mockImplementation(async (path: string) => path.startsWith(resolve('static/agent')) ? 'fresh-release' : 'old-build'); - for (const file of ['install.sh', 'ec-agent.py', 'ec-agent-update.py', 'manifest.json'] as const) { + files.readFile.mockImplementation(async (path: string) => + path.startsWith(resolve('static/agent')) ? 'fresh-release' : 'old-build' + ); + for (const file of [ + 'install.sh', + 'ec-agent.py', + 'ec-agent-update.py', + 'manifest.json' + ] as const) { expect(await agentReleaseAsset(file)).toBe('fresh-release'); } - expect(files.readFile.mock.calls.map(call => call[0])).toEqual([ - resolve('static/agent/install.sh'), resolve('static/agent/ec-agent.py'), - resolve('static/agent/ec-agent-update.py'), resolve('static/agent/manifest.json') + expect(files.readFile.mock.calls.map((call) => call[0])).toEqual([ + resolve('static/agent/install.sh'), + resolve('static/agent/ec-agent.py'), + resolve('static/agent/ec-agent-update.py'), + resolve('static/agent/manifest.json') ]); }); @@ -27,33 +41,55 @@ describe('public agent release artifacts', () => { const missing = Object.assign(new Error('Not packaged yet'), { code: 'ENOENT' }); files.readFile.mockRejectedValueOnce(missing).mockResolvedValue('old-build'); await expect(agentReleaseAsset('ec-agent-update.py')).rejects.toBe(missing); - expect(files.readFile.mock.calls).toEqual([[resolve('static/agent/ec-agent-update.py'), 'utf8']]); - }); - - it('uses the packaged production directory for every artifact even when source assets exist', async () => { - files.readFile.mockImplementation(async (path: string) => path.startsWith(resolve('build/client/agent')) ? 'packaged-release' : 'source-release'); - for (const file of ['install.sh', 'ec-agent.py', 'ec-agent-update.py', 'manifest.json'] as const) { - expect(await agentReleaseAsset(file)).toBe('packaged-release'); - } - expect(files.readFile.mock.calls.map(call => call[0])).toEqual([ - resolve('build/client/agent/install.sh'), resolve('build/client/agent/ec-agent.py'), - resolve('build/client/agent/ec-agent-update.py'), resolve('build/client/agent/manifest.json') + expect(files.readFile.mock.calls).toEqual([ + [resolve('static/agent/ec-agent-update.py'), 'utf8'] ]); }); - it.each(['ec-agent.py', 'ec-agent-update.py', 'manifest.json'])('serves the packaged %s without a browser session or host credential', async file => { - const content = file.endsWith('.json') ? '{"version":"1.2.0","updaterProtocolVersion":1}' : '# verified release source\n'; - files.readFile.mockResolvedValue(content); - const response = await GET({ params: { file } }); - expect(response.status).toBe(200); - expect(await response.text()).toBe(content); - expect(response.headers.get('cache-control')).toBe('no-cache'); - expect(response.headers.get('content-type')).toContain(file.endsWith('.json') ? 'application/json' : 'text/plain'); - expect(files.readFile.mock.calls).toEqual([[resolve('build/client/agent', file), 'utf8']]); + it('uses the packaged production directory for every artifact even when source assets exist', async () => { + files.readFile.mockImplementation(async (path: string) => + path.startsWith(resolve('build/client/agent')) ? 'packaged-release' : 'source-release' + ); + for (const file of [ + 'install.sh', + 'ec-agent.py', + 'ec-agent-update.py', + 'manifest.json' + ] as const) { + expect(await agentReleaseAsset(file)).toBe('packaged-release'); + } + expect(files.readFile.mock.calls.map((call) => call[0])).toEqual([ + resolve('build/client/agent/install.sh'), + resolve('build/client/agent/ec-agent.py'), + resolve('build/client/agent/ec-agent-update.py'), + resolve('build/client/agent/manifest.json') + ]); }); + it.each(['ec-agent.py', 'ec-agent-update.py', 'manifest.json'])( + 'serves the packaged %s without a browser session or host credential', + async (file) => { + const content = file.endsWith('.json') + ? '{"version":"1.2.0","updaterProtocolVersion":1}' + : '# verified release source\n'; + files.readFile.mockResolvedValue(content); + const response = await GET({ params: { file } }); + expect(response.status).toBe(200); + expect(await response.text()).toBe(content); + expect(response.headers.get('cache-control')).toBe('no-cache'); + expect(response.headers.get('content-type')).toContain( + file.endsWith('.json') ? 'application/json' : 'text/plain' + ); + expect(files.readFile.mock.calls).toEqual([ + [resolve('build/client/agent', file), 'utf8'] + ]); + } + ); + it('serves the updater from development assets when no packaged asset exists', async () => { - files.readFile.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })).mockResolvedValueOnce('# updater\n'); + files.readFile + .mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })) + .mockResolvedValueOnce('# updater\n'); expect(await agentReleaseAsset('ec-agent-update.py')).toBe('# updater\n'); expect(files.readFile.mock.calls).toEqual([ [resolve('build/client/agent/ec-agent-update.py'), 'utf8'], @@ -68,7 +104,15 @@ describe('public agent release artifacts', () => { expect(files.readFile).toHaveBeenCalledTimes(1); }); - it.each(['../ec-agent.py', '%2e%2e%2fmanifest.json', 'agent-health.json', 'update-status.json', 'config.json', 'install.sh', 'ec-agent-update.py/anything'])('rejects non-release file %s before reading disk', async file => { + it.each([ + '../ec-agent.py', + '%2e%2e%2fmanifest.json', + 'agent-health.json', + 'update-status.json', + 'config.json', + 'install.sh', + 'ec-agent-update.py/anything' + ])('rejects non-release file %s before reading disk', async (file) => { await expect(GET({ params: { file } })).rejects.toMatchObject({ status: 404 }); expect(files.readFile).not.toHaveBeenCalled(); }); diff --git a/tests/unit/host-agent-update-status.test.ts b/tests/unit/host-agent-update-status.test.ts index 1a1c3a0..22fef47 100644 --- a/tests/unit/host-agent-update-status.test.ts +++ b/tests/unit/host-agent-update-status.test.ts @@ -1,15 +1,27 @@ import { describe, expect, it } from 'vitest'; -import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, type HostInventory } from '../../src/lib/hosts.js'; +import { + hostAgentUpdateLabel, + hostAgentUpdateMessage, + hostAgentUpdatesEnabled, + type HostInventory +} from '../../src/lib/hosts.js'; const installed: HostInventory = { - agentVersion: '1.2.0', capabilities: ['host.decommission', 'agent.update'], - agentUpdate: { enabled: true, state: 'current', currentVersion: '1.2.0', checkedAt: '2026-10-04T12:00:00Z' } + agentVersion: '1.2.0', + capabilities: ['host.decommission', 'agent.update'], + agentUpdate: { + enabled: true, + state: 'current', + currentVersion: '1.2.0', + checkedAt: '2026-10-04T12:00:00Z' + } }; describe('host automatic update status', () => { it('requires the installed updater capability instead of inferring setup from agent version', () => { for (const inventory of [ - {}, { agentVersion: '1.2.0' }, + {}, + { agentVersion: '1.2.0' }, { ...installed, capabilities: ['host.decommission'] }, { ...installed, agentUpdate: { ...installed.agentUpdate!, enabled: false } } ]) { @@ -22,27 +34,60 @@ describe('host automatic update status', () => { }); it('does not claim an unchecked fresh installation has verified the latest release', () => { - const fresh = { ...installed, agentUpdate: { ...installed.agentUpdate!, checkedAt: undefined } }; + const fresh = { + ...installed, + agentUpdate: { ...installed.agentUpdate!, checkedAt: undefined } + }; expect(hostAgentUpdateLabel(fresh)).toBe('Awaiting first check'); expect(hostAgentUpdateMessage(fresh)).toContain('will check'); expect(hostAgentUpdateMessage(installed)).toContain('last check'); }); it('distinguishes a deferred check from a failed update without assuming every hold is an active operation', () => { - const deferred = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'deferred' as const, targetVersion: '1.3.0' } }; + const deferred = { + ...installed, + agentUpdate: { + ...installed.agentUpdate!, + state: 'deferred' as const, + targetVersion: '1.3.0' + } + }; expect(hostAgentUpdateLabel(deferred)).toBe('Deferred'); expect(hostAgentUpdateMessage(deferred)).toContain('reported condition to clear'); expect(hostAgentUpdateMessage(deferred)).toContain('checks again automatically'); - const failed = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'failed' as const, error: 'Checksum mismatch' } }; + const failed = { + ...installed, + agentUpdate: { + ...installed.agentUpdate!, + state: 'failed' as const, + error: 'Checksum mismatch' + } + }; expect(hostAgentUpdateLabel(failed)).toBe('Needs attention'); expect(hostAgentUpdateMessage(failed)).toContain('did not complete'); }); it('describes connection verification until the updater confirms success', () => { - const updating = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'updating' as const, targetVersion: '1.3.0' } }; + const updating = { + ...installed, + agentUpdate: { + ...installed.agentUpdate!, + state: 'updating' as const, + targetVersion: '1.3.0' + } + }; expect(hostAgentUpdateLabel(updating)).toBe('Updating'); - expect(hostAgentUpdateMessage(updating)).toContain('checking that the new process connects'); - const updated = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'updated' as const, currentVersion: '1.3.0' } }; + expect(hostAgentUpdateMessage(updating)).toContain( + 'checking that the new process connects' + ); + const updated = { + ...installed, + agentUpdate: { + ...installed.agentUpdate!, + state: 'updated' as const, + currentVersion: '1.3.0' + } + }; expect(hostAgentUpdateLabel(updated)).toBe('Updated'); expect(hostAgentUpdateMessage(updated)).toContain('connected successfully'); }); diff --git a/tests/unit/host-availability.test.ts b/tests/unit/host-availability.test.ts index 07baaa8..e5c0314 100644 --- a/tests/unit/host-availability.test.ts +++ b/tests/unit/host-availability.test.ts @@ -6,24 +6,44 @@ let fixture: Fixture; let hostId: string; const GiB = 1024 ** 3; const epoch = Date.parse('2026-10-05T12:00:00Z'); -const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB, - cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, - runtimeImages: ['image'], libraries: [{ id: 't5-library', state: 'ready' }, { id: 't6-library', state: 'ready' }] }; +const inventory = { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * GiB, + cpuCount: 16, + cpuUsagePercent: 0, + memoryTotal: 64 * GiB, + memoryAvailable: 60 * GiB, + runtimeImages: ['image'], + libraries: [ + { id: 't5-library', state: 'ready' }, + { id: 't6-library', state: 'ready' } + ] +}; const t5 = { libraryId: 't5-library', image: 'image', automatic: true }; const t6 = { libraryId: 't6-library', image: 'image', automatic: true }; -async function settle() { await setImmediate(); await setImmediate(); } +async function settle() { + await setImmediate(); + await setImmediate(); +} beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(epoch); const hosts = await import('../../src/lib/server/hosts/index.js'); hostId = hosts.createHost({ name: 'Availability host', portMin: 28960, portMax: 28962 }).id; hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory); hosts.heartbeat(hostId, inventory); await settle(); }); -afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); }); +afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + await fixture.cleanup(); +}); describe('cached launch availability', () => { it('reads cached estimates without placement, historical scans, or repeated database reads', async () => { @@ -34,7 +54,10 @@ describe('cached launch availability', () => { const prepare = vi.spyOn(db(), 'prepare'); for (let count = 0; count < 100; count++) { expect(launchAvailability([t5, t6]).availability).toMatchObject({ - updatedAt: new Date(epoch).toISOString(), requested: 2, estimatedAvailable: 2, portsAvailable: 3 + updatedAt: new Date(epoch).toISOString(), + requested: 2, + estimatedAvailable: 2, + portsAvailable: 3 }); } expect(prepare).not.toHaveBeenCalled(); @@ -46,44 +69,69 @@ describe('cached launch availability', () => { const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js'); heartbeat(hostId, { ...inventory, memoryAvailable: GiB + 500_000_000 }); const limited = launchAvailability([t5, t6]); - expect(limited.availability).toMatchObject({ requested: 2, estimatedAvailable: 1, portsAvailable: 3, - cpuCoresAvailable: 15.2, memoryBytesAvailable: GiB }); + expect(limited.availability).toMatchObject({ + requested: 2, + estimatedAvailable: 1, + portsAvailable: 3, + cpuCoresAvailable: 15.2, + memoryBytesAvailable: GiB + }); expect(limited.warnings.join(' ')).toContain('1 of 2'); vi.setSystemTime(epoch + 15_000); heartbeat(hostId, { ...inventory, memoryAvailable: 2 * GiB + 500_000_000 }); - expect(launchAvailability([t5, t6]).availability).toMatchObject({ estimatedAvailable: 2, - memoryBytesAvailable: 2 * GiB, updatedAt: new Date(epoch + 15_000).toISOString() }); + expect(launchAvailability([t5, t6]).availability).toMatchObject({ + estimatedAvailable: 2, + memoryBytesAvailable: 2 * GiB, + updatedAt: new Date(epoch + 15_000).toISOString() + }); heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 }); expect(launchAvailability([t5]).availability.estimatedAvailable).toBe(0); }); it('refreshes port reservations and startup queue snapshots after lifecycle events', async () => { - const { reserveHostPort, releaseHostPorts } = await import('../../src/lib/server/hosts/index.js'); + const { reserveHostPort, releaseHostPorts } = + await import('../../src/lib/server/hosts/index.js'); const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js'); - const { beginDeployment, deploymentStage } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, deploymentStage } = + await import('../../src/lib/server/provisioning/history.js'); const { db } = await import('../../src/lib/server/db/index.js'); const limit = launchAvailability([t5]).availability.startupLimit!; reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'udp' }); reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'tcp' }); await settle(); - expect(launchAvailability([t5, t6, t5]).availability).toMatchObject({ portsAvailable: 2, estimatedAvailable: 2 }); + expect(launchAvailability([t5, t6, t5]).availability).toMatchObject({ + portsAvailable: 2, + estimatedAvailable: 2 + }); const active = beginDeployment('t5_zombie_theater_1', 'create'); const queued = beginDeployment('t6_zombie_tranzit_1', 'create'); db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(queued); - deploymentStage(queued, 'Awaiting placement', { checkpoint: { key: 'queued', label: 'Queued for placement' } }); + deploymentStage(queued, 'Awaiting placement', { + checkpoint: { key: 'queued', label: 'Queued for placement' } + }); await settle(); - expect(launchAvailability([t5]).availability).toMatchObject({ queued: 1, startupSlots: Math.max(0, limit - 1) }); + expect(launchAvailability([t5]).availability).toMatchObject({ + queued: 1, + startupSlots: Math.max(0, limit - 1) + }); db().prepare("UPDATE server_deployments SET state='done' WHERE id=?").run(active); releaseHostPorts('reserved'); await settle(); - expect(launchAvailability([t5]).availability).toMatchObject({ portsAvailable: 3, startupSlots: limit }); + expect(launchAvailability([t5]).availability).toMatchObject({ + portsAvailable: 3, + startupSlots: limit + }); }); it('stops counting a stale host without presenting old measurements as current capacity', async () => { const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js'); vi.setSystemTime(epoch + 91_000); const result = launchAvailability([t5]); - expect(result.availability).toMatchObject({ updatedAt: new Date(epoch).toISOString(), estimatedAvailable: 0, portsAvailable: 0 }); + expect(result.availability).toMatchObject({ + updatedAt: new Date(epoch).toISOString(), + estimatedAvailable: 0, + portsAvailable: 0 + }); expect(result.warnings.join(' ')).toContain('Waiting for an eligible host'); }); @@ -99,6 +147,9 @@ describe('cached launch availability', () => { expect(errors).not.toHaveBeenCalled(); db().exec('ALTER TABLE unavailable_native_provisioning RENAME TO native_provisioning'); heartbeat(hostId, inventory); - expect(launchAvailability([t5]).availability).toMatchObject({ estimatedAvailable: 1, portsAvailable: 3 }); + expect(launchAvailability([t5]).availability).toMatchObject({ + estimatedAvailable: 1, + portsAvailable: 3 + }); }); }); diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index 07e7d75..095f9f8 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -5,25 +5,48 @@ let fixture: Fixture; let hostId: string; const epoch = Date.parse('2026-10-04T12:00:00Z'); const GiB = 1024 ** 3; -const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, - dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB, - runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] }; -const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 }; +const inventory = { + cpuCount: 16, + cpuUsagePercent: 10, + memoryTotal: 64 * GiB, + memoryAvailable: 60 * GiB, + dockerRunning: true, + protocolVersion: 1, + directoriesWritable: true, + storageAvailable: 100 * GiB, + runtimeImages: ['image'], + libraries: [{ id: 'library', state: 'ready' }] +}; +const sample = { + instanceId: 'instance-1', + generation: 1, + uptimeSeconds: 600, + cpuCores: 0.1, + memoryBytes: GiB / 4 +}; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(epoch); const hosts = await import('../../src/lib/server/hosts/index.js'); hostId = hosts.createHost({ name: 'First' }).id; hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory); await instance(hostId, sample.instanceId); }); -afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); +afterEach(async () => { + vi.useRealTimers(); + await fixture.cleanup(); +}); async function instance(host: string, id: string) { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString()); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,'library','image',28960,?)` + ) + .run(id, 't5_zombie_theater_' + id, host, new Date().toISOString()); } async function report(minute: number, overrides = {}, host = hostId) { vi.setSystemTime(epoch + minute * 60_000); @@ -32,30 +55,54 @@ async function report(minute: number, overrides = {}, host = hostId) { } async function learned() { // One real startup followed by twenty running minute peaks. - for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 }); + for (let minute = 0; minute < 4; minute++) + await report(minute, { + uptimeSeconds: minute * 60, + cpuCores: 0.2, + memoryBytes: (3 * GiB) / 8 + }); for (let minute = 5; minute < 25; minute++) await report(minute); return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId); } describe('measured instance capacity', () => { it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => { - for (let minute = 0; minute < 101; minute++) await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 }); + for (let minute = 0; minute < 101; minute++) + await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 }); const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }); const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 }); expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 }); expect(first.samples).toHaveLength(50); - expect(first.samples[0]).toMatchObject({ hostId, hostName: 'First', instanceId: sample.instanceId, - server: 't5_zombie_theater_instance-1', generation: 1, phase: 'running', - cpuCores: 0.1, memoryBytes: GiB / 4, timestamp: new Date(epoch + 100 * 60_000).toISOString() }); - expect(new Set([...first.samples, ...second.samples].map(row => row.timestamp)).size).toBe(100); - expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ page: 3, samples: [expect.any(Object)] }); + expect(first.samples[0]).toMatchObject({ + hostId, + hostName: 'First', + instanceId: sample.instanceId, + server: 't5_zombie_theater_instance-1', + generation: 1, + phase: 'running', + cpuCores: 0.1, + memoryBytes: GiB / 4, + timestamp: new Date(epoch + 100 * 60_000).toISOString() + }); + expect( + new Set([...first.samples, ...second.samples].map((row) => row.timestamp)).size + ).toBe(100); + expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ + page: 3, + samples: [expect.any(Object)] + }); const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 }); expect(startup.total).toBe(5); - expect(startup.samples.every(row => row.phase === 'startup')).toBe(true); - expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(96); + expect(startup.samples.every((row) => row.phase === 'startup')).toBe(true); + expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe( + 96 + ); vi.setSystemTime(epoch + 26 * 60 * 60_000); - expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ total: 0, samples: [] }); + expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ + total: 0, + samples: [] + }); }); it('supports fleet samples and retains instance identity after retirement', async () => { @@ -66,27 +113,53 @@ describe('measured instance capacity', () => { await instance(other.id, 'instance-2'); await report(1, { instanceId: 'instance-2' }, other.id); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), sample.instanceId); const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(row => row.instanceId)).toEqual([sample.instanceId]); + expect( + capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map( + (row) => row.instanceId + ) + ).toEqual([sample.instanceId]); const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 }); expect(fleet.total).toBe(2); - expect(fleet.samples.map(row => row.hostName)).toEqual(['Second', 'First']); + expect(fleet.samples.map((row) => row.hostName)).toEqual(['Second', 'First']); }); it('validates the samples API and rejects agent credentials and cross-origin reads', async () => { await report(0); const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js'); - const request = (query = '', headers = {}, id = hostId) => GET({ request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { headers }), params: { id } }); + const request = (query = '', headers = {}, id = hostId) => + GET({ + request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { + headers + }), + params: { id } + }); const response = request(); expect(response.headers.get('cache-control')).toBe('no-store'); - expect(await response.json()).toMatchObject({ total: 1, samples: [expect.objectContaining({ cpuCores: 0.1 })] }); - for (const query of ['?scope=wrong', '?phase=wrong', '?page=-1', '?page=1.5', '?page=NaN', '?page=Infinity']) { + expect(await response.json()).toMatchObject({ + total: 1, + samples: [expect.objectContaining({ cpuCores: 0.1 })] + }); + for (const query of [ + '?scope=wrong', + '?phase=wrong', + '?page=-1', + '?page=1.5', + '?page=NaN', + '?page=Infinity' + ]) { expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 })); } expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 })); - expect(() => request('', { origin: 'https://other.test' })).toThrow(expect.objectContaining({ status: 403 })); - expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(expect.objectContaining({ status: 403 })); + expect(() => request('', { origin: 'https://other.test' })).toThrow( + expect.objectContaining({ status: 403 }) + ); + expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow( + expect.objectContaining({ status: 403 }) + ); }); it('reports sampling failure without inventing measurements and clears it after recovery', async () => { @@ -97,7 +170,9 @@ describe('measured instance capacity', () => { expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error); expect(capacityCalibration(hostId).running.samples).toBe(0); for (const invalid of [42, {}, 'x'.repeat(501)]) { - expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error'); + expect(() => + heartbeat(hostId, { ...inventory, instanceResourcesError: invalid }) + ).toThrow('Invalid instance resource sampling error'); } await report(0); expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined(); @@ -106,27 +181,53 @@ describe('measured instance capacity', () => { it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => { const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB }); + expect(capacityCalibration(hostId).startup).toMatchObject({ + source: 'bootstrap', + cpuCores: 0.5, + memoryBytes: GiB + }); const model = await learned(); - expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.1, memoryBytes: GiB / 4, samples: 20 }); - expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.2, memoryBytes: 3 * GiB / 8, samples: 4 }); + expect(model.running).toMatchObject({ + source: 'host', + cpuCores: 0.1, + memoryBytes: GiB / 4, + samples: 20 + }); + expect(model.startup).toMatchObject({ + source: 'host', + cpuCores: 0.2, + memoryBytes: (3 * GiB) / 8, + samples: 4 + }); }); it('raises allowances immediately on a measured spike, including during cold start', async () => { await report(0, { cpuCores: 2, memoryBytes: 2 * GiB }); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2, memoryBytes: 2 * GiB }); + expect(capacityCalibration(hostId).running).toMatchObject({ + source: 'bootstrap', + cpuCores: 2, + memoryBytes: 2 * GiB + }); await learned(); await report(25, { cpuCores: 3, memoryBytes: 3 * GiB }); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3, memoryBytes: 3 * GiB }); + expect(capacityCalibration(hostId).running).toMatchObject({ + source: 'host', + cpuCores: 3, + memoryBytes: 3 * GiB + }); expect(capacityCalibration(hostId).startup.cpuCores).toBe(3); }); it('uses recent fleet evidence on new hosts and survives a database restart', async () => { const model = await learned(); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Second' }); - const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' }); + expect(capacityCalibration(other.id).running).toMatchObject({ + ...model.running, + source: 'fleet' + }); expect(capacityCalibration(hostId)).toEqual(model); }); it('removes legacy cached padding once while preserving host and fleet evidence', async () => { @@ -134,11 +235,16 @@ describe('measured instance capacity', () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); // Recreate the previous schema and its padded allowance values. db().exec('ALTER TABLE ec_resource_allowances DROP COLUMN usage_margin'); - db().exec('UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25'); + db().exec( + 'UPDATE ec_resource_allowances SET cpu_cores=cpu_cores*1.25, memory_bytes=memory_bytes*1.25' + ); closeDb(); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId)).toEqual(model); - expect(capacityCalibration('new-host').startup).toEqual({ ...model.startup, source: 'fleet' }); + expect(capacityCalibration('new-host').startup).toEqual({ + ...model.startup, + source: 'fleet' + }); closeDb(); expect(capacityCalibration(hostId)).toEqual(model); }); @@ -148,22 +254,40 @@ describe('measured instance capacity', () => { const other = hosts.createHost({ name: 'Slower CPU' }); hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); await instance(other.id, 'instance-2'); - for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id); + for (let minute = 25; minute < 45; minute++) + await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.1 }); - expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2 }); - expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2 }); + expect(capacityCalibration(hostId).running).toMatchObject({ + source: 'host', + cpuCores: 0.1 + }); + expect(capacityCalibration(other.id).running).toMatchObject({ + source: 'host', + cpuCores: 2 + }); + expect(capacityCalibration('new-host').running).toMatchObject({ + source: 'fleet', + cpuCores: 2 + }); }); it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => { const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); for (let n = 0; n < 40; n++) { vi.setSystemTime(epoch + n * 1000); - heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] }); + heartbeat(hostId, { + ...inventory, + instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] + }); } const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]); + expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([ + { cpu_cores: 0.4 } + ]); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 }); + expect(capacityCalibration(hostId).running).toMatchObject({ + source: 'bootstrap', + samples: 1 + }); }); it('does not train from missing, foreign, retired or old-generation samples', async () => { const { db } = await import('../../src/lib/server/db/index.js'); @@ -173,17 +297,33 @@ describe('measured instance capacity', () => { await instance(other.id, 'foreign'); await report(0, { instanceId: 'foreign' }); await report(1, { generation: 2 }); - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), sample.instanceId); await report(2); expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]); }); it('rejects impossible and malformed measurements before changing heartbeat state', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); - for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 }, - { memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) { - expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource'); + for (const overrides of [ + { cpuCores: -1 }, + { cpuCores: NaN }, + { cpuCores: 17 }, + { memoryBytes: 0 }, + { memoryBytes: 65 * GiB }, + { generation: 0 }, + { uptimeSeconds: -1 } + ]) { + expect(() => + heartbeat(hostId, { + ...inventory, + instanceResources: [{ ...sample, ...overrides }] + }) + ).toThrow('Invalid instance resource'); } - expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource'); + expect(() => + heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] }) + ).toThrow('Invalid instance resource'); expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined(); }); it('expires stale models and prunes old measurements as new samples arrive', async () => { @@ -196,29 +336,55 @@ describe('measured instance capacity', () => { expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1); expect(capacityCalibration(hostId).running.source).toBe('bootstrap'); }); - it.each(['startup', 'running'] as const)('uses the %s 95th percentile for CPU and memory, keeping admission reads cached', async phase => { - await learned(); - // Repeated starts or running observations include isolated CPU and - // memory outliers in different minutes; neither should set the allowance. - const uptimeSeconds = phase === 'startup' ? 60 : 600; - for (let minute = 25; minute < 125; minute++) await report(minute, { - uptimeSeconds, cpuCores: minute === 25 ? 2 : 0.4, memoryBytes: minute === 26 ? 3 * GiB : GiB / 2 - }); - const { db } = await import('../../src/lib/server/db/index.js'); - const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); - const prepare = vi.spyOn(db(), 'prepare'); - expect(capacityCalibration(hostId)[phase]).toMatchObject({ source: 'host', cpuCores: 0.4, memoryBytes: GiB / 2 }); - expect(capacityCalibration('new-host')[phase]).toMatchObject({ source: 'fleet', cpuCores: 0.4, memoryBytes: GiB / 2 }); - expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false); - prepare.mockRestore(); + it.each(['startup', 'running'] as const)( + 'uses the %s 95th percentile for CPU and memory, keeping admission reads cached', + async (phase) => { + await learned(); + // Repeated starts or running observations include isolated CPU and + // memory outliers in different minutes; neither should set the allowance. + const uptimeSeconds = phase === 'startup' ? 60 : 600; + for (let minute = 25; minute < 125; minute++) + await report(minute, { + uptimeSeconds, + cpuCores: minute === 25 ? 2 : 0.4, + memoryBytes: minute === 26 ? 3 * GiB : GiB / 2 + }); + const { db } = await import('../../src/lib/server/db/index.js'); + const { capacityCalibration } = + await import('../../src/lib/server/hosts/calibration.js'); + const prepare = vi.spyOn(db(), 'prepare'); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ + source: 'host', + cpuCores: 0.4, + memoryBytes: GiB / 2 + }); + expect(capacityCalibration('new-host')[phase]).toMatchObject({ + source: 'fleet', + cpuCores: 0.4, + memoryBytes: GiB / 2 + }); + expect( + prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples')) + ).toBe(false); + prepare.mockRestore(); - // Both phases still raise immediately for fresh spikes, hold that value - // within the minute, then return to P95 on the next recalculation. - await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB }); - expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); - await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); - expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 3, memoryBytes: 2 * GiB }); - await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); - expect(capacityCalibration(hostId)[phase]).toMatchObject({ cpuCores: 0.4, memoryBytes: GiB / 2 }); - }); + // Both phases still raise immediately for fresh spikes, hold that value + // within the minute, then return to P95 on the next recalculation. + await report(124.25, { uptimeSeconds, cpuCores: 3, memoryBytes: 2 * GiB }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ + cpuCores: 3, + memoryBytes: 2 * GiB + }); + await report(124.5, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ + cpuCores: 3, + memoryBytes: 2 * GiB + }); + await report(125, { uptimeSeconds, cpuCores: 0.4, memoryBytes: GiB / 2 }); + expect(capacityCalibration(hostId)[phase]).toMatchObject({ + cpuCores: 0.4, + memoryBytes: GiB / 2 + }); + } + ); }); diff --git a/tests/unit/host-cpu-inventory.test.ts b/tests/unit/host-cpu-inventory.test.ts index 911bc01..3e6204e 100644 --- a/tests/unit/host-cpu-inventory.test.ts +++ b/tests/unit/host-cpu-inventory.test.ts @@ -5,10 +5,15 @@ describe('host CPU inventory', () => { it('accepts older agents without CPU usage', () => { expect(validateInventory({ cpuCount: 4 }).cpuUsagePercent).toBeUndefined(); }); - it.each([0, 0.1, 42.3, 100])('accepts CPU usage %s', cpuUsagePercent => { + it.each([0, 0.1, 42.3, 100])('accepts CPU usage %s', (cpuUsagePercent) => { expect(validateInventory({ cpuUsagePercent }).cpuUsagePercent).toBe(cpuUsagePercent); }); - it.each([-1, 100.1, NaN, Infinity, -Infinity, null, '25', true, {}])('rejects invalid CPU usage %j', cpuUsagePercent => { - expect(() => validateInventory({ cpuUsagePercent })).toThrow('Invalid inventory cpuUsagePercent'); - }); + it.each([-1, 100.1, NaN, Infinity, -Infinity, null, '25', true, {}])( + 'rejects invalid CPU usage %j', + (cpuUsagePercent) => { + expect(() => validateInventory({ cpuUsagePercent })).toThrow( + 'Invalid inventory cpuUsagePercent' + ); + } + ); }); diff --git a/tests/unit/host-deletion.test.ts b/tests/unit/host-deletion.test.ts index 47608e0..8ba84a7 100644 --- a/tests/unit/host-deletion.test.ts +++ b/tests/unit/host-deletion.test.ts @@ -5,9 +5,22 @@ import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 }; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); }); +const inventory = { + hostname: 'games-1', + protocolVersion: 1, + capabilities: ['host.decommission'], + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3 +}; +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.unstubAllEnvs(); + await fixture.cleanup(); +}); async function enrolled() { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -28,8 +41,11 @@ async function finishCleanup(hosts: Awaited>) { async function instance(hostId: string, retired = false) { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at) - VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)`) + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at) + VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)` + ) .run(hostId, new Date().toISOString(), retired ? new Date().toISOString() : null); } @@ -38,10 +54,19 @@ describe('host deletion', () => { const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js'); const legacy = new Database(':memory:'); try { - legacy.exec("CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')"); - ensureHostSchema(legacy); ensureHostSchema(legacy); - expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({ id: 'old', name: 'Existing host', deleted_at: null }); - } finally { legacy.close(); } + legacy.exec( + "CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')" + ); + ensureHostSchema(legacy); + ensureHostSchema(legacy); + expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({ + id: 'old', + name: 'Existing host', + deleted_at: null + }); + } finally { + legacy.close(); + } }); it('deletes an unenrolled host and invalidates its unused enrollment', async () => { @@ -50,7 +75,8 @@ describe('host deletion', () => { const enrollment = hosts.createEnrollment(host.id); expect(hosts.hostDeletionStatus(host.id)).toEqual({ allowed: true, blockers: [] }); hosts.deleteHost(host.id); - expect(hosts.getHost(host.id)).toBeNull(); expect(hosts.listHosts()).toEqual([]); + expect(hosts.getHost(host.id)).toBeNull(); + expect(hosts.listHosts()).toEqual([]); expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired'); expect(() => hosts.createEnrollment(host.id)).toThrow('not found'); expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 })); @@ -59,58 +85,113 @@ describe('host deletion', () => { it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => { const hosts = await enrolled(); const other = hosts.createHost({ name: 'Keep' }); - const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); + const otherCredentials = hosts.enrollHost( + hosts.createEnrollment(other.id).token, + inventory + ); const enrollment = hosts.createEnrollment(hosts.host.id); const completed = hosts.queueHostOperation(hosts.host.id, 'host.inspect', {}); - hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'completed', result: { hostname: 'games-1' } }); + hosts.reportHostOperation(hosts.host.id, completed.id, { + state: 'completed', + result: { hostname: 'games-1' } + }); await instance(hosts.host.id, true); const file = join(fixture.root, 'game-files-sentinel'); await writeFile(file, 'Game files remain'); const { db } = await import('../../src/lib/server/db/index.js'); const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js'); ensureGameFileSchema(db()); - db().prepare("INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)").run(new Date().toISOString()); - db().prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) - VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id); - db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString()); + db() + .prepare( + "INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)" + ) + .run(new Date().toISOString()); + db() + .prepare( + `INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) + VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')` + ) + .run(new Date().toISOString(), file, hosts.host.id); + db() + .prepare( + "INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)" + ) + .run(hosts.host.id, new Date().toISOString()); const queued = hosts.deleteHost(hosts.host.id); expect(queued.deleted).toBe(false); expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued'); - const { nativeLaunchSelection } = await import('../../src/lib/server/provisioning/native.js'); - expect(() => nativeLaunchSelection({ hostId: hosts.host.id, libraryId: 'version', image: 'plutainer:test' }, 't5')) - .toThrow('The selected host is being removed'); + const { nativeLaunchSelection } = + await import('../../src/lib/server/provisioning/native.js'); + expect(() => + nativeLaunchSelection( + { hostId: hosts.host.id, libraryId: 'version', image: 'plutainer:test' }, + 't5' + ) + ).toThrow('The selected host is being removed'); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); await finishCleanup(hosts); expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull(); expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id); expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired'); - expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]); - expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } }); + expect(hosts.listHosts().map((host) => host.id)).toEqual([other.id]); + expect(hosts.getHostOperation(completed.id)).toMatchObject({ + state: 'completed', + result: { hostname: 'games-1' } + }); expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2); - expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) }); - expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) }); - expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 }); - expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({ count: 1 }); - expect(db().prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'").get()).toEqual({ state: 'ready', source_host_id: hosts.host.id }); + expect( + db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id) + ).toEqual({ mode: 'disabled', deleted_at: expect.any(String) }); + expect( + db() + .prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?') + .get(hosts.host.id) + ).toEqual({ revoked_at: expect.any(String) }); + expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ + count: 1 + }); + expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({ + count: 1 + }); + expect( + db() + .prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'") + .get() + ).toEqual({ state: 'ready', source_host_id: hosts.host.id }); expect(await readFile(file, 'utf8')).toBe('Game files remain'); expect(() => hosts.heartbeat(hosts.host.id, inventory)).toThrow('not found'); expect(() => hosts.rotateCredential(hosts.host.id)).toThrow('not found'); - expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('not found'); - expect(() => hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' })).toThrow('not found'); + expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow( + 'not found' + ); + expect(() => + hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' }) + ).toThrow('not found'); expect(() => hosts.updateHost(hosts.host.id, { mode: 'active' })).toThrow('not found'); }); it('cancels only undelivered queued operations and retains their journal', async () => { const hosts = await enrolled(); - const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map(kind => hosts.queueHostOperation(hosts.host.id, kind, {})); + const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map((kind) => + hosts.queueHostOperation(hosts.host.id, kind, {}) + ); expect(hosts.hostDeletionStatus(hosts.host.id).allowed).toBe(true); hosts.deleteHost(hosts.host.id); const { db } = await import('../../src/lib/server/db/index.js'); for (const operation of operations) { - expect(hosts.getHostOperation(operation.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('before this operation was delivered') }); - expect(db().prepare('SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id').all(operation.id)).toEqual([{ state: 'pending' }, { state: 'failed' }]); + expect(hosts.getHostOperation(operation.id)).toMatchObject({ + state: 'failed', + error: expect.stringContaining('before this operation was delivered') + }); + expect( + db() + .prepare( + 'SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id' + ) + .all(operation.id) + ).toEqual([{ state: 'pending' }, { state: 'failed' }]); } }); @@ -118,25 +199,41 @@ describe('host deletion', () => { const hosts = await enrolled(); await instance(hosts.host.id); const status = hosts.hostDeletionStatus(hosts.host.id); - expect(status.allowed).toBe(false); expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1'); - expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); + expect(status.allowed).toBe(false); + expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1'); + expect(() => hosts.deleteHost(hosts.host.id)).toThrow( + expect.objectContaining({ status: 409 }) + ); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); }); it('blocks an accepted unplaced pinned launch until it is cancelled', async () => { const hosts = await enrolled(); - const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js'); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { queueNativeCreationIntent } = + await import('../../src/lib/server/provisioning/native.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const queued = queueNativeCreationIntent({ server: 't5_zombie_theater_1', game: 't5', mapSlug: 'theater', - plutoKey: 'saved-key', selection: { hostId: hosts.host.id, libraryId: 'library', image: 'plutainer:test' } }); + const queued = queueNativeCreationIntent({ + server: 't5_zombie_theater_1', + game: 't5', + mapSlug: 'theater', + plutoKey: 'saved-key', + selection: { hostId: hosts.host.id, libraryId: 'library', image: 'plutainer:test' } + }); expect(queued.ok).toBe(true); if (!queued.ok) throw new Error(queued.message); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); - expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: false, - blockers: [expect.stringContaining('Cancel the queued server launches pinned to this host')] }); - expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); + expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ + allowed: false, + blockers: [ + expect.stringContaining('Cancel the queued server launches pinned to this host') + ] + }); + expect(() => hosts.deleteHost(hosts.host.id)).toThrow( + expect.objectContaining({ status: 409 }) + ); expect(hosts.getHost(hosts.host.id)?.decommission).toBeNull(); expect(cancelCreation(queued.deploymentId!)).toBeTruthy(); expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] }); @@ -147,13 +244,21 @@ describe('host deletion', () => { it('does not pin a host for automatic launches or launches assigned elsewhere', async () => { const hosts = await enrolled(); const other = hosts.createHost({ name: 'Other host' }); - const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js'); + const { queueNativeCreationIntent } = + await import('../../src/lib/server/provisioning/native.js'); for (const [index, selection] of [ { hostId: hosts.host.id, automatic: true }, { hostId: other.id, automatic: false } ].entries()) { - expect(queueNativeCreationIntent({ server: `t5_zombie_theater_${index + 1}`, game: 't5', mapSlug: 'theater', - plutoKey: `saved-key-${index}`, selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } }).ok).toBe(true); + expect( + queueNativeCreationIntent({ + server: `t5_zombie_theater_${index + 1}`, + game: 't5', + mapSlug: 'theater', + plutoKey: `saved-key-${index}`, + selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } + }).ok + ).toBe(true); } expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] }); expect(hosts.hostDeletionStatus(other.id).allowed).toBe(false); @@ -164,45 +269,72 @@ describe('host deletion', () => { it('blocks residual port reservations independently of live instances', async () => { const hosts = await enrolled(); hosts.reserveHostPort(hosts.host.id, 'unresolved-instance'); - expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] }); - expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); + expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ + allowed: false, + blockers: [expect.stringContaining('reserved game port')] + }); + expect(() => hosts.deleteHost(hosts.host.id)).toThrow( + expect.objectContaining({ status: 409 }) + ); hosts.releaseHostPorts('unresolved-instance'); await finishCleanup(hosts); expect(hosts.getHost(hosts.host.id)).toBeNull(); }); - it.each(['pending', 'accepted', 'running', 'unknown'] as const)('blocks unresolved %s work after delivery without revoking access or cancelling other work', async state => { - const hosts = await enrolled(); - const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {}); - hosts.pendingHostOperations(hosts.host.id); - if (state !== 'pending') hosts.reportHostOperation(hosts.host.id, operation.id, { state }); - const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {}); - hosts.updateHost(hosts.host.id, { mode: 'disabled' }); - expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('delivered operation')] }); - expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); - expect(hosts.getHostOperation(queued.id)?.state).toBe('pending'); - expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); - hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }); - hosts.deleteHost(hosts.host.id); - expect(hosts.getHostOperation(queued.id)?.state).toBe('failed'); - }); + it.each(['pending', 'accepted', 'running', 'unknown'] as const)( + 'blocks unresolved %s work after delivery without revoking access or cancelling other work', + async (state) => { + const hosts = await enrolled(); + const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {}); + hosts.pendingHostOperations(hosts.host.id); + if (state !== 'pending') + hosts.reportHostOperation(hosts.host.id, operation.id, { state }); + const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {}); + hosts.updateHost(hosts.host.id, { mode: 'disabled' }); + expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ + allowed: false, + blockers: [expect.stringContaining('delivered operation')] + }); + expect(() => hosts.deleteHost(hosts.host.id)).toThrow( + expect.objectContaining({ status: 409 }) + ); + expect(hosts.getHostOperation(queued.id)?.state).toBe('pending'); + expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe( + hosts.host.id + ); + hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }); + hosts.deleteHost(hosts.host.id); + expect(hosts.getHostOperation(queued.id)?.state).toBe('failed'); + } + ); it('rechecks blockers on DELETE even after a successful eligibility check', async () => { const hosts = await enrolled(); const route = await import('../../src/routes/api/hosts/[id]/+server.js'); const url = `https://ec.test/api/hosts/${hosts.host.id}`; const params = { id: hosts.host.id }; - expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({ deletion: { allowed: true, blockers: [] } }); + expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({ + deletion: { allowed: true, blockers: [] } + }); const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {}); hosts.pendingHostOperations(hosts.host.id); const blocked = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }); expect(blocked.status).toBe(409); - expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] }); + expect(await blocked.json()).toMatchObject({ + error: expect.any(String), + blockers: [expect.stringContaining('delivered operation')] + }); hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }); const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }); - expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) }); + expect(removed.status).toBe(202); + expect(await removed.json()).toMatchObject({ + deleted: false, + operationId: expect.any(String) + }); await finishCleanup(hosts); - expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404); + expect( + route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status + ).toBe(404); }); it('keeps DELETE restricted to admin sessions and same-origin management requests', async () => { @@ -210,10 +342,22 @@ describe('host deletion', () => { const { routePermission, roleAllows } = await import('../../src/lib/server/auth/access.js'); const permission = routePermission('/api/hosts/[id]', 'DELETE'); expect(roleAllows('admin', permission)).toBe(true); - expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false); + expect(roleAllows('operator', permission)).toBe(false); + expect(roleAllows('viewer', permission)).toBe(false); const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js'); - for (const headers of [new Headers({ origin: 'https://attacker.test' }), new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })]) { - expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 })); + for (const headers of [ + new Headers({ origin: 'https://attacker.test' }), + new Headers({ authorization: `Bearer ${hosts.credentials.credential}` }) + ]) { + expect(() => + DELETE({ + request: new Request('https://ec.test/api/hosts/id', { + method: 'DELETE', + headers + }), + params: { id: hosts.host.id } + }) + ).toThrow(expect.objectContaining({ status: 403 })); } expect(hosts.getHost(hosts.host.id)).not.toBeNull(); }); @@ -221,8 +365,13 @@ describe('host deletion', () => { it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => { const hosts = await enrolled(); hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] }); - expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] }); - expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); + expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ + allowed: false, + blockers: [expect.stringContaining('Update this host agent')] + }); + expect(() => hosts.deleteHost(hosts.host.id)).toThrow( + expect.objectContaining({ status: 409 }) + ); expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy(); expect(hosts.getHost(hosts.host.id)?.mode).toBe('active'); }); @@ -230,19 +379,29 @@ describe('host deletion', () => { it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => { const hosts = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id); + db() + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run('2020-01-01T00:00:00Z', hosts.host.id); const queued = hosts.deleteHost(hosts.host.id); expect(hosts.deleteHost(hosts.host.id)).toEqual(queued); - expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } }); + expect(hosts.getHost(hosts.host.id)).toMatchObject({ + connected: false, + readiness: 'deleting', + decommission: { stage: 'queued' } + }); expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]); expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup'); - expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup'); + expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow( + 'cleanup' + ); expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup'); expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false); expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup'); expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup'); hosts.heartbeat(hosts.host.id, inventory); - expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]); + expect(hosts.pendingHostOperations(hosts.host.id).map((operation) => operation.id)).toEqual( + [queued.operationId] + ); }); it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => { @@ -250,17 +409,27 @@ describe('host deletion', () => { const queued = hosts.deleteHost(hosts.host.id); const operation = hosts.getHostOperation(queued.operationId!)!; const token = operation.payload.receiptToken as string; - const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage }); + const receipt = (stage: string, credential = token) => + hosts.receiveDecommissionReceipt(operation.id, credential, { stage }); expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 })); hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); - expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 })); + expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow( + expect.objectContaining({ status: 401 }) + ); expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 })); - expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker'); + expect(() => + hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }) + ).toThrow('cleanup worker'); receipt('cleaned'); expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned'); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); const route = await import('../../src/routes/api/hosts/[id]/+server.js'); - const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text(); + const detail = await route + .GET({ + request: new Request('https://ec.test/api/hosts/' + hosts.host.id), + params: { id: hosts.host.id } + }) + .text(); expect(detail).not.toContain(token); receipt('completed'); expect(hosts.getHost(hosts.host.id)).toBeNull(); @@ -273,15 +442,28 @@ describe('host deletion', () => { it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => { const hosts = await enrolled(); const first = hosts.deleteHost(hosts.host.id); - hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' }); - expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true }); + hosts.reportHostOperation(hosts.host.id, first.operationId!, { + state: 'failed', + error: 'Ownership marker requires repair' + }); + expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ + stage: 'failed', + retryable: true + }); expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy(); const second = hosts.deleteHost(hosts.host.id); expect(second.operationId).not.toBe(first.operationId); const operation = hosts.getHostOperation(second.operationId!)!; hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); - hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' }); - expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' }); + hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { + stage: 'failed', + error: 'Filesystem cleanup failed; retrying' + }); + expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ + stage: 'failed', + retryable: false, + error: 'Filesystem cleanup failed; retrying' + }); expect(hosts.deleteHost(hosts.host.id)).toEqual(second); await finishCleanup(hosts); expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed'); @@ -293,12 +475,25 @@ describe('host deletion', () => { const operation = hosts.getHostOperation(queued.operationId!)!; hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js'); - const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, { - method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage }) - }), params: { id: operation.id } }); + const send = (stage: string, token: string) => + POST({ + request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ stage }) + }), + params: { id: operation.id } + }); expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401); expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200); - expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200); - expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200); + expect((await send('completed', operation.payload.receiptToken as string)).status).toBe( + 200 + ); + expect((await send('completed', operation.payload.receiptToken as string)).status).toBe( + 200 + ); }); }); diff --git a/tests/unit/host-instances.test.ts b/tests/unit/host-instances.test.ts index 69e41a0..6f5d6f5 100644 --- a/tests/unit/host-instances.test.ts +++ b/tests/unit/host-instances.test.ts @@ -3,8 +3,13 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent } from '../helpers/telemetry.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); describe('host detail instances', () => { it('returns only current instances belonging to the requested host and refreshes their health', async () => { @@ -14,12 +19,18 @@ describe('host detail instances', () => { const { db } = await import('../../src/lib/server/db/index.js'); const now = new Date().toISOString(); const add = (name: string, hostId: string, retired = false, destroyed = false) => { - db().prepare(`INSERT INTO native_instances (id, server, host_id, generation, library_id, image, port, created_at, retired_at) - VALUES (?, ?, ?, 1, 'library', 'runtime:test', 28960, ?, ?)`) + db() + .prepare( + `INSERT INTO native_instances (id, server, host_id, generation, library_id, image, port, created_at, retired_at) + VALUES (?, ?, ?, 1, 'library', 'runtime:test', 28960, ?, ?)` + ) .run(name, name, hostId, now, retired ? now : null); - db().prepare(`INSERT INTO servers (name, + db() + .prepare( + `INSERT INTO servers (name, container_state, container_checked_at, last_seen_at, last_players, destroyed_at) - VALUES (?, 'running', ?, ?, 2, ?)`) + VALUES (?, 'running', ?, ?, 2, ?)` + ) .run(name, now, now, destroyed ? now : null); }; add('running', host.id); @@ -27,39 +38,80 @@ describe('host detail instances', () => { add('other-host', other.id); add('retired', host.id, true); add('destroyed', host.id, false, true); - db().prepare(`INSERT INTO native_instances (id, server, host_id, library_id, image, port, created_at, retired_at) - VALUES ('old-placement', 'other-host', ?, 'library', 'runtime:test', 28960, ?, ?)`) + db() + .prepare( + `INSERT INTO native_instances (id, server, host_id, library_id, image, port, created_at, retired_at) + VALUES ('old-placement', 'other-host', ?, 'library', 'runtime:test', 28960, ?, ?)` + ) .run(host.id, now, now); db().prepare("UPDATE servers SET container_state='exited' WHERE name='unavailable'").run(); const { hostOverview } = await import('../../src/lib/server/hosts/overview.js'); const overview = hostOverview(); - expect(overview.find(row => row.id === host.id)).toMatchObject({ serverCount: 2, playerCount: 2, unreportedInstances: 1 }); - expect(overview.find(row => row.id === other.id)).toMatchObject({ serverCount: 1, playerCount: 2, unreportedInstances: 0 }); + expect(overview.find((row) => row.id === host.id)).toMatchObject({ + serverCount: 2, + playerCount: 2, + unreportedInstances: 1 + }); + expect(overview.find((row) => row.id === other.id)).toMatchObject({ + serverCount: 1, + playerCount: 2, + unreportedInstances: 0 + }); const { GET: list } = await import('../../src/routes/api/hosts/+server.js'); const listed = await list({ request: new Request('https://ec.test/api/hosts') }).json(); expect(listed.serverNow).toEqual(expect.any(Number)); - expect(listed.hosts.find((row: { id: string }) => row.id === host.id)).toMatchObject({ serverCount: 2, playerCount: 2, unreportedInstances: 1 }); + expect(listed.hosts.find((row: { id: string }) => row.id === host.id)).toMatchObject({ + serverCount: 2, + playerCount: 2, + unreportedInstances: 1 + }); const { GET } = await import('../../src/routes/api/hosts/[id]/+server.js'); - const read = async (id: string) => (await GET({ request: new Request(`https://ec.test/api/hosts/${id}`), params: { id } }).json()); + const read = async (id: string) => + await GET({ + request: new Request(`https://ec.test/api/hosts/${id}`), + params: { id } + }).json(); const body = await read(host.id); - expect(body.instances.map((instance: { name: string }) => instance.name)).toEqual(['running', 'unavailable']); + expect(body.instances.map((instance: { name: string }) => instance.name)).toEqual([ + 'running', + 'unavailable' + ]); expect(body.instances[0]).toMatchObject({ health: 'online', last_players: 2 }); expect(body.instances[1]).toMatchObject({ health: 'offline', last_players: null }); expect(body.serverNow).toEqual(expect.any(Number)); db().prepare("UPDATE servers SET last_players=0 WHERE name='running'").run(); expect((await read(host.id)).instances[0].last_players).toBe(0); - expect((await read(other.id)).instances.map((instance: { name: string }) => instance.name)).toEqual(['other-host']); + expect( + (await read(other.id)).instances.map((instance: { name: string }) => instance.name) + ).toEqual(['other-host']); const empty = createHost({ name: 'Empty host' }); expect((await read(empty.id)).instances).toEqual([]); - expect(hostOverview().find(row => row.id === empty.id)).toMatchObject({ serverCount: 0, playerCount: 0, unreportedInstances: 0 }); + expect(hostOverview().find((row) => row.id === empty.id)).toMatchObject({ + serverCount: 0, + playerCount: 0, + unreportedInstances: 0 + }); const report = { ...gameEvent('heartbeat', 15, 4, 'running'), receivedAt: now }; - db().prepare('INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)') + db() + .prepare( + 'INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)' + ) .run('running', 15, 4, now, JSON.stringify(report)); - expect(hostOverview().find(row => row.id === host.id)).toMatchObject({ playerCount: 4, unreportedInstances: 1 }); + expect(hostOverview().find((row) => row.id === host.id)).toMatchObject({ + playerCount: 4, + unreportedInstances: 1 + }); db().prepare("DELETE FROM server_telemetry WHERE server='running'").run(); - db().prepare("UPDATE servers SET last_seen_at='2000-01-01T00:00:00Z',last_players=99 WHERE name='running'").run(); - expect(hostOverview().find(row => row.id === host.id)).toMatchObject({ playerCount: 0, unreportedInstances: 2 }); + db() + .prepare( + "UPDATE servers SET last_seen_at='2000-01-01T00:00:00Z',last_players=99 WHERE name='running'" + ) + .run(); + expect(hostOverview().find((row) => row.id === host.id)).toMatchObject({ + playerCount: 0, + unreportedInstances: 2 + }); }); }); diff --git a/tests/unit/host-log-results.test.ts b/tests/unit/host-log-results.test.ts index b4f23db..b81e2ea 100644 --- a/tests/unit/host-log-results.test.ts +++ b/tests/unit/host-log-results.test.ts @@ -2,8 +2,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); async function setup() { const api = await import('../../src/lib/server/hosts/index.js'); @@ -16,41 +21,80 @@ async function setup() { describe('transient log results', () => { it.each([ { lines: ['private log body'], size: 17, mtime: '2026-10-08T12:00:00Z', truncated: false }, - { contentBase64: Buffer.from('private log body').toString('base64'), size: 17, reset: false, cursor: { offset: 17 } } - ])('delivers full replies to current readers while storing only metadata', async result => { + { + contentBase64: Buffer.from('private log body').toString('base64'), + size: 17, + reset: false, + cursor: { offset: 17 } + } + ])('delivers full replies to current readers while storing only metadata', async (result) => { const api = await setup(); const first = api.waitHostOperationStatus(api.operation.id); const second = api.waitHostOperation(api.operation.id); - const receipt = api.reportHostOperationStatus(api.host.id, api.operation.id, { state: 'completed', result }); + const receipt = api.reportHostOperationStatus(api.host.id, api.operation.id, { + state: 'completed', + result + }); expect((await first).result).toEqual(result); expect((await second).result).toEqual(result); expect(receipt.result).toMatchObject({ retained: false, size: 17 }); expect(api.getHostOperation(api.operation.id)?.result).toEqual(receipt.result); - const stored = api.db().prepare('SELECT result FROM ec_host_operations WHERE id=?').get(api.operation.id); + const stored = api + .db() + .prepare('SELECT result FROM ec_host_operations WHERE id=?') + .get(api.operation.id); expect(JSON.stringify(stored)).not.toMatch(/private log body|contentBase64|cursor|"lines"/); - expect(api.db().prepare("SELECT * FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?").all(api.operation.id)).toEqual([]); + expect( + api + .db() + .prepare( + "SELECT * FROM file_blob_references WHERE owner_type='operation-result' AND owner_id=?" + ) + .all(api.operation.id) + ).toEqual([]); // Completed reads cannot serve a stale body, including duplicate receipts. - await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow('no longer available'); - api.reportHostOperationStatus(api.host.id, api.operation.id, { state: 'completed', result }); - await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow('no longer available'); + await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow( + 'no longer available' + ); + api.reportHostOperationStatus(api.host.id, api.operation.id, { + state: 'completed', + result + }); + await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow( + 'no longer available' + ); }); it('preserves missing-log results and operation failures', async () => { const api = await setup(); const missing = api.waitHostOperationStatus(api.operation.id); - api.reportHostOperationStatus(api.host.id, api.operation.id, { state: 'completed', result: null }); + api.reportHostOperationStatus(api.host.id, api.operation.id, { + state: 'completed', + result: null + }); expect((await missing).result).toBeNull(); const failed = api.queueHostOperation(api.host.id, 'logs.read', {}); const failure = api.waitHostOperationStatus(failed.id); - api.reportHostOperationStatus(api.host.id, failed.id, { state: 'failed', error: 'Read failed' }); + api.reportHostOperationStatus(api.host.id, failed.id, { + state: 'failed', + error: 'Read failed' + }); expect(await failure).toMatchObject({ state: 'failed', error: 'Read failed' }); }); it('discards bodies arriving after the reader has timed out', async () => { const api = await setup(); await expect(api.waitHostOperationStatus(api.operation.id, 0)).rejects.toThrow(); - api.reportHostOperationStatus(api.host.id, api.operation.id, { state: 'completed', result: { lines: ['late body'] } }); - expect(api.getHostOperation(api.operation.id)?.result).toEqual({ retained: false, lineCount: 1 }); - await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow('no longer available'); + api.reportHostOperationStatus(api.host.id, api.operation.id, { + state: 'completed', + result: { lines: ['late body'] } + }); + expect(api.getHostOperation(api.operation.id)?.result).toEqual({ + retained: false, + lineCount: 1 + }); + await expect(api.waitHostOperationStatus(api.operation.id)).rejects.toThrow( + 'no longer available' + ); }); }); diff --git a/tests/unit/host-observation-retention.test.ts b/tests/unit/host-observation-retention.test.ts index 877b39f..d740ade 100644 --- a/tests/unit/host-observation-retention.test.ts +++ b/tests/unit/host-observation-retention.test.ts @@ -4,15 +4,27 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; const now = Date.parse('2026-10-05T16:00:00Z'); const day = 86_400_000; const at = (age: number) => new Date(now - age).toISOString(); -const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, - storageAvailable: 100 * 1024 ** 3, runtimeImages: ['plutainer:test'], libraries: [{ id: 'library', state: 'ready' }] }; +const inventory = { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: ['plutainer:test'], + libraries: [{ id: 'library', state: 'ready' }] +}; let fixture: Fixture; beforeEach(async () => { - vi.resetModules(); fixture = await newFixture([]); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now); + vi.resetModules(); + fixture = await newFixture([]); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(now); +}); +afterEach(async () => { + vi.useRealTimers(); + vi.restoreAllMocks(); + await fixture.cleanup(); }); -afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); }); async function setup() { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -21,67 +33,144 @@ async function setup() { const host = hosts.createHost({ name: 'Retention host' }); const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); const handle = db(); - function add(kind: string, options: { state?: string; payload?: Record; result?: unknown; - error?: string; age?: number; serialized?: boolean; routing?: Record } = {}) { + function add( + kind: string, + options: { + state?: string; + payload?: Record; + result?: unknown; + error?: string; + age?: number; + serialized?: boolean; + routing?: Record; + } = {} + ) { const payload = options.payload ?? {}; const operation = options.serialized - ? hosts.queueHostOperationSerialized(host.id, kind, Buffer.from(JSON.stringify(payload)), {}, options.routing ?? {}) + ? hosts.queueHostOperationSerialized( + host.id, + kind, + Buffer.from(JSON.stringify(payload)), + {}, + options.routing ?? {} + ) : hosts.queueHostOperation(host.id, kind, payload); const state = options.state ?? 'completed'; - if (state !== 'pending') hosts.reportHostOperationStatus(host.id, operation.id, { - state, result: options.result === undefined ? { ok: true } : options.result, error: options.error - }); - handle.prepare('UPDATE ec_host_operations SET created_at=?,updated_at=? WHERE id=?') + if (state !== 'pending') + hosts.reportHostOperationStatus(host.id, operation.id, { + state, + result: options.result === undefined ? { ok: true } : options.result, + error: options.error + }); + handle + .prepare('UPDATE ec_host_operations SET created_at=?,updated_at=? WHERE id=?') .run(at(10 * day), at(options.age ?? 2 * day), operation.id); return operation.id; } - return { ...hosts, handle, db, closeDb, host, credentials, fleetSettings, saveFleetSettings, add }; + return { + ...hosts, + handle, + db, + closeDb, + host, + credentials, + fleetSettings, + saveFleetSettings, + add + }; } describe('routine host observation retention', () => { it('drains more than one batch while yielding and preserving recent or failed operations', async () => { const api = await setup(); const { drainRetention } = await import('../../src/lib/server/retention.js'); - const retained = [api.add('logs.read', { age: 0 }), api.add('logs.read', { state: 'failed' }), api.add('instance.restart')]; + const retained = [ + api.add('logs.read', { age: 0 }), + api.add('logs.read', { state: 'failed' }), + api.add('instance.restart') + ]; api.handle.transaction(() => { const insert = api.handle.prepare(`INSERT INTO ec_host_operations (id,host_id,kind,payload,state,created_at,updated_at,observation_success) VALUES(?,?,'logs.read','{}','completed',?,?,1)`); - for (let i = 0; i < 1501; i++) insert.run(`read-${i}`, api.host.id, at(2 * day), at(2 * day)); + for (let i = 0; i < 1501; i++) + insert.run(`read-${i}`, api.host.id, at(2 * day), at(2 * day)); })(); let yielded = false; - setImmediate(() => { yielded = true; }); + setImmediate(() => { + yielded = true; + }); expect(await drainRetention()).toBe(1501); expect(yielded).toBe(true); - expect((api.handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]).map(row => row.id).sort()).toEqual(retained.sort()); + expect( + (api.handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]) + .map((row) => row.id) + .sort() + ).toEqual(retained.sort()); expect(await drainRetention()).toBe(0); }); it('strips existing log bodies in batches without deleting recent or protected history', async () => { const api = await setup(); - const { drainRetention, setRetentionControl } = await import('../../src/lib/server/retention.js'); + const { drainRetention, setRetentionControl } = + await import('../../src/lib/server/retention.js'); const ids = Array.from({ length: 53 }, () => api.add('logs.read', { age: 0 })); setRetentionControl({ category: 'observation', id: ids[0], protected: true }); - api.handle.prepare("UPDATE ec_host_operations SET result=? WHERE kind='logs.read'") + api.handle + .prepare("UPDATE ec_host_operations SET result=? WHERE kind='logs.read'") .run(JSON.stringify({ lines: ['legacy log contents'], size: 20, truncated: false })); expect(await drainRetention()).toBe(0); - for (const id of ids) expect(api.getHostOperation(id)?.result).toEqual({ retained: false, lineCount: 1, size: 20, truncated: false }); - expect(api.handle.prepare("SELECT value FROM settings WHERE key='log_history_metadata_v1'").get()).toEqual({ value: '1' }); + for (const id of ids) + expect(api.getHostOperation(id)?.result).toEqual({ + retained: false, + lineCount: 1, + size: 20, + truncated: false + }); + expect( + api.handle + .prepare("SELECT value FROM settings WHERE key='log_history_metadata_v1'") + .get() + ).toEqual({ value: '1' }); expect(api.handle.pragma('foreign_key_check')).toEqual([]); }); it('removes aged successful reads, manifests, and read-only RCON while retaining failures, actions, and recent evidence', async () => { const api = await setup(); - const removed = ['host.inspect', 'instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read'] - .map(kind => api.add(kind)); - removed.push(...['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } }))); + const removed = [ + 'host.inspect', + 'instance.inspect', + 'files.read', + 'files.stat', + 'files.list', + 'files.manifest', + 'logs.read' + ].map((kind) => api.add(kind)); + removed.push( + ...['status', 'getstatus', 'version'].map((command) => + api.add('rcon.execute', { payload: { command } }) + ) + ); const retained = [ - ...['pending', 'accepted', 'running', 'failed', 'unknown'].map(state => api.add('files.read', { state })), - ...['instance.restart', 'instance.stop', 'instance.remove', 'files.stage', 'files.activate', 'files.rollback', 'runtime.pull', 'unknown.read'] - .map(kind => api.add(kind)), + ...['pending', 'accepted', 'running', 'failed', 'unknown'].map((state) => + api.add('files.read', { state }) + ), + ...[ + 'instance.restart', + 'instance.stop', + 'instance.remove', + 'files.stage', + 'files.activate', + 'files.rollback', + 'runtime.pull', + 'unknown.read' + ].map((kind) => api.add(kind)), api.add('rcon.execute', { payload: { command: 'map_restart' } }), api.add('rcon.execute', { payload: { command: 'status; map_restart' } }), - api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }), + api.add('rcon.execute', { + payload: { command: 'status' }, + result: { ok: false, reason: 'timeout' } + }), api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }), api.add('files.read', { error: 'Read failed' }), api.add('files.read', { result: { error: 'Read failed' } }), @@ -89,51 +178,93 @@ describe('routine host observation retention', () => { api.add('files.read', { age: day - 1 }) ]; expect(api.pruneHostObservations()).toBe(removed.length); - const ids = (api.handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]).map(row => row.id); + const ids = ( + api.handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[] + ).map((row) => row.id); expect(ids.sort()).toEqual(retained.sort()); expect(api.pruneHostObservations()).toBe(0); }); it('keeps per-server failures in routine completed batches and prunes them on schedule', async () => { const api = await setup(); - const results = [{ instanceId: 'one', generation: 1, reply: { ok: false, reason: 'timeout', text: '' } }]; - const batches = [false, true].map(ok => api.add('rcon.status-batch', { result: { ok, results } })); + const results = [ + { instanceId: 'one', generation: 1, reply: { ok: false, reason: 'timeout', text: '' } } + ]; + const batches = [false, true].map((ok) => + api.add('rcon.status-batch', { result: { ok, results } }) + ); const failed = api.add('rcon.status-batch', { state: 'failed', error: 'Batch failed' }); const malformed = api.add('rcon.status-batch', { result: { ok: false } }); - expect(api.listHostOperationHistory(1, 'important').operations.map(op => op.id).sort()).toEqual([failed, malformed].sort()); - for (const id of batches) expect(api.getHostOperation(id)?.result).toMatchObject({ results }); + expect( + api + .listHostOperationHistory(1, 'important') + .operations.map((op) => op.id) + .sort() + ).toEqual([failed, malformed].sort()); + for (const id of batches) + expect(api.getHostOperation(id)?.result).toMatchObject({ results }); expect(api.pruneHostObservations()).toBe(2); for (const id of batches) expect(api.getHostOperationStatus(id)).toBeNull(); }); it('reclassifies existing completed batches once without hiding operation errors', async () => { const api = await setup(); - const result = { ok: false, results: [{ instanceId: 'one', generation: 1, reply: { ok: false, reason: 'timeout' } }] }; + const result = { + ok: false, + results: [{ instanceId: 'one', generation: 1, reply: { ok: false, reason: 'timeout' } }] + }; const batch = api.add('rcon.status-batch', { result }); - const retained = [api.add('rcon.status-batch', { result, error: 'Incomplete' }), + const retained = [ + api.add('rcon.status-batch', { result, error: 'Incomplete' }), api.add('rcon.status-batch', { result, state: 'failed' }), - api.add('rcon.status-batch', { result: { ...result, error: 'Incomplete' } })]; - api.handle.exec("UPDATE ec_host_operations SET observation_success=0; DELETE FROM ec_host_schema_migrations WHERE name='completed-status-batches'"); + api.add('rcon.status-batch', { result: { ...result, error: 'Incomplete' } }) + ]; + api.handle.exec( + "UPDATE ec_host_operations SET observation_success=0; DELETE FROM ec_host_schema_migrations WHERE name='completed-status-batches'" + ); api.closeDb(); - expect(api.listHostOperationHistory(1, 'important').operations.map(op => op.id).sort()).toEqual(retained.sort()); + expect( + api + .listHostOperationHistory(1, 'important') + .operations.map((op) => op.id) + .sort() + ).toEqual(retained.sort()); expect(api.getHostOperation(batch)?.result).toEqual(result); expect(api.pruneHostObservations()).toBe(1); // The migration is not a recurring scan or reclassification of retained evidence. - api.db().prepare("UPDATE ec_host_operations SET state='completed',error=NULL,result=? WHERE id=?").run(JSON.stringify(result), retained[0]); + api.db() + .prepare( + "UPDATE ec_host_operations SET state='completed',error=NULL,result=? WHERE id=?" + ) + .run(JSON.stringify(result), retained[0]); api.closeDb(); expect(api.pruneHostObservations()).toBe(0); }); it('classifies full serialized RCON payloads and deletes their events and immutable bytes together', async () => { const api = await setup(); - const read = api.add('rcon.execute', { serialized: true, payload: { command: 'status' }, routing: { command: 'map_restart' } }); - const action = api.add('rcon.execute', { serialized: true, payload: { command: 'map_restart' }, routing: { command: 'status' } }); - expect(api.handle.prepare('SELECT COUNT(*) AS count FROM ec_host_operation_payloads').get()).toEqual({ count: 2 }); + const read = api.add('rcon.execute', { + serialized: true, + payload: { command: 'status' }, + routing: { command: 'map_restart' } + }); + const action = api.add('rcon.execute', { + serialized: true, + payload: { command: 'map_restart' }, + routing: { command: 'status' } + }); + expect( + api.handle.prepare('SELECT COUNT(*) AS count FROM ec_host_operation_payloads').get() + ).toEqual({ count: 2 }); expect(api.pruneHostObservations()).toBe(1); expect(api.getHostOperationStatus(read)).toBeNull(); expect(api.getHostOperation(action)?.payload).toEqual({ command: 'map_restart' }); - expect(api.handle.prepare('SELECT DISTINCT operation_id FROM ec_host_operation_events').all()).toEqual([{ operation_id: action }]); - expect(api.handle.prepare('SELECT operation_id FROM ec_host_operation_payloads').all()).toEqual([{ operation_id: action }]); + expect( + api.handle.prepare('SELECT DISTINCT operation_id FROM ec_host_operation_events').all() + ).toEqual([{ operation_id: action }]); + expect( + api.handle.prepare('SELECT operation_id FROM ec_host_operation_payloads').all() + ).toEqual([{ operation_id: action }]); expect(api.handle.pragma('foreign_key_check')).toEqual([]); }); @@ -157,19 +288,44 @@ describe('routine host observation retention', () => { const insert = api.handle.prepare(`INSERT INTO ec_host_operations (id,host_id,kind,payload,state,created_at,updated_at,observation_success) VALUES(?,?,?,'{}','completed',?,?,?)`); - for (let index = 0; index < 2000; index++) insert.run(`action-${index}`, api.host.id, 'instance.restart', at(10 * day), at(10 * day), 0); - for (let index = 0; index < 503; index++) insert.run(`read-${index}`, api.host.id, 'files.read', at(2 * day), at(2 * day), 1); + for (let index = 0; index < 2000; index++) + insert.run( + `action-${index}`, + api.host.id, + 'instance.restart', + at(10 * day), + at(10 * day), + 0 + ); + for (let index = 0; index < 503; index++) + insert.run(`read-${index}`, api.host.id, 'files.read', at(2 * day), at(2 * day), 1); })(); const prepare = vi.spyOn(api.handle, 'prepare'); expect(api.pruneHostObservations()).toBe(500); - const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('SELECT id FROM ec_host_operations WHERE observation_success'))!; + const query = prepare.mock.calls + .map(([sql]) => sql) + .find((sql) => + sql.includes('SELECT id FROM ec_host_operations WHERE observation_success') + )!; prepare.mockRestore(); - const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(at(day)) as { detail: string }[]; - expect(plan.some(row => row.detail.includes('USING INDEX ec_host_operations_observation_retention'))).toBe(true); - expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false); + const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(at(day)) as { + detail: string; + }[]; + expect( + plan.some((row) => + row.detail.includes('USING INDEX ec_host_operations_observation_retention') + ) + ).toBe(true); + expect(plan.some((row) => row.detail.includes('TEMP B-TREE'))).toBe(false); expect(api.pruneHostObservations()).toBe(3); expect(api.pruneHostObservations()).toBe(0); - expect(api.handle.prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 2000 }); + expect( + api.handle + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'" + ) + .get() + ).toEqual({ count: 2000 }); }); it('reuses the same three delete statements across different batch sizes', async () => { @@ -180,11 +336,14 @@ describe('routine host observation retention', () => { const insert = api.handle.prepare(`INSERT INTO ec_host_operations (id,host_id,kind,payload,state,created_at,updated_at,observation_success) VALUES(?,?,'files.stat','{}','completed',?,?,1)`); - for (let i = 0; i < count; i++) insert.run(`read-${count}-${i}`, api.host.id, at(2 * day), at(2 * day)); + for (let i = 0; i < count; i++) + insert.run(`read-${count}-${i}`, api.host.id, at(2 * day), at(2 * day)); })(); const prepare = vi.spyOn(api.handle, 'prepare'); expect(api.pruneHostObservations()).toBe(count); - const deletes = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.startsWith('DELETE FROM ec_host_')); + const deletes = prepare.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.startsWith('DELETE FROM ec_host_')); prepare.mockRestore(); expect(deletes).toHaveLength(3); for (const sql of deletes) { @@ -200,17 +359,28 @@ describe('routine host observation retention', () => { it('backfills legacy observation classification once while preserving actions and negative or invalid results', async () => { const api = await setup(); const read = api.add('files.manifest'); - const serializedRead = api.add('rcon.execute', { serialized: true, payload: { command: 'status' } }); - const retained = [api.add('instance.restart'), api.add('rcon.execute', { payload: { command: 'map_restart' } }), - api.add('instance.inspect', { result: { healthy: false } }), api.add('files.read', { result: { ok: false } }), - api.add('files.read', { error: 'Unavailable' }), api.add('files.read', { result: { error: 'Unavailable' } }), - api.add('files.stat', { state: 'unknown' })]; + const serializedRead = api.add('rcon.execute', { + serialized: true, + payload: { command: 'status' } + }); + const retained = [ + api.add('instance.restart'), + api.add('rcon.execute', { payload: { command: 'map_restart' } }), + api.add('instance.inspect', { result: { healthy: false } }), + api.add('files.read', { result: { ok: false } }), + api.add('files.read', { error: 'Unavailable' }), + api.add('files.read', { result: { error: 'Unavailable' } }), + api.add('files.stat', { state: 'unknown' }) + ]; const malformed = api.add('files.read'); - api.handle.prepare('UPDATE ec_host_operations SET result=? WHERE id=?').run('{not-json', malformed); + api.handle + .prepare('UPDATE ec_host_operations SET result=? WHERE id=?') + .run('{not-json', malformed); retained.push(malformed); api.handle.exec('DROP INDEX ec_host_operations_observation_retention'); api.handle.exec('DROP INDEX ec_host_operations_important_history'); - api.handle.exec(`DROP TRIGGER operation_history_count_insert; DROP TRIGGER operation_history_count_delete; + api.handle + .exec(`DROP TRIGGER operation_history_count_insert; DROP TRIGGER operation_history_count_delete; DROP TRIGGER operation_history_count_update; DROP TABLE ec_host_operation_history_counts; DELETE FROM ec_host_schema_migrations WHERE name='history-counts-v1'`); api.handle.exec('ALTER TABLE ec_host_operations DROP COLUMN observation_success'); @@ -220,7 +390,11 @@ describe('routine host observation retention', () => { expect(api.getHostOperationStatus(read)).toBeNull(); expect(api.getHostOperationStatus(serializedRead)).toBeNull(); const handle = api.db(); - expect((handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]).map(row => row.id).sort()).toEqual(retained.sort()); + expect( + (handle.prepare('SELECT id FROM ec_host_operations').all() as { id: string }[]) + .map((row) => row.id) + .sort() + ).toEqual(retained.sort()); // Subsequent boots honor the recorded classification instead of repeatedly // deriving it from historical rows that were deliberately preserved. handle.prepare("UPDATE ec_host_operations SET result='{}' WHERE id=?").run(malformed); @@ -234,12 +408,21 @@ describe('routine host observation retention', () => { const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js'); for (const observationKeepDays of [7, 0]) { api.saveFleetSettings({ ...api.fleetSettings(), observationKeepDays }); - const response = await POST({ request: new Request('https://ec.test/api/agent/heartbeat', { - method: 'POST', headers: { authorization: `Bearer ${api.credentials.credential}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ inventory }) - }) }); + const response = await POST({ + request: new Request('https://ec.test/api/agent/heartbeat', { + method: 'POST', + headers: { + authorization: `Bearer ${api.credentials.credential}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ inventory }) + }) + }); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ observationKeepDays, heartbeatSeconds: 15 }); + expect(await response.json()).toMatchObject({ + observationKeepDays, + heartbeatSeconds: 15 + }); } }); }); diff --git a/tests/unit/host-operation-history.test.ts b/tests/unit/host-operation-history.test.ts index 1712eb6..7b78cc4 100644 --- a/tests/unit/host-operation-history.test.ts +++ b/tests/unit/host-operation-history.test.ts @@ -2,8 +2,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); async function seedHistory(count = 53) { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -22,21 +28,40 @@ async function seedHistory(count = 53) { const id = `operation-${String(count - index).padStart(4, '0')}`; const timestamp = index < 3 ? '2026-10-01T12:00:00.000Z' : '2026-10-05T12:00:00.000Z'; ids.push(id); - insert.run(id, [first, second, deleted][index % 3].id, - '{"credential":"payload-secret"}', '{"credential":"result-secret"}', timestamp, timestamp); + insert.run( + id, + [first, second, deleted][index % 3].id, + '{"credential":"payload-secret"}', + '{"credential":"result-secret"}', + timestamp, + timestamp + ); } - handle.prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?').run('2026-10-05T12:30:00.000Z', deleted.id); + handle + .prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?') + .run('2026-10-05T12:30:00.000Z', deleted.id); })(); return { ...hosts, handle, ids, first, second, deleted }; } async function operationHistory() { const api = await seedHistory(0); - function add(kind: string, options: { payload?: Record; state?: string; result?: unknown; error?: string } = {}) { + function add( + kind: string, + options: { + payload?: Record; + state?: string; + result?: unknown; + error?: string; + } = {} + ) { const operation = api.queueHostOperation(api.first.id, kind, options.payload ?? {}); - if (options.state !== 'pending') api.reportHostOperationStatus(api.first.id, operation.id, { - state: options.state ?? 'completed', result: options.result, error: options.error - }); + if (options.state !== 'pending') + api.reportHostOperationStatus(api.first.id, operation.id, { + state: options.state ?? 'completed', + result: options.result, + error: options.error + }); return operation.id; } return { ...api, add }; @@ -45,51 +70,116 @@ async function operationHistory() { describe('host operation history', () => { it('maintains exact scoped totals through receipts, expiration, rollback, deletion and reopening without recounting history', async () => { const api = await operationHistory(); - const { importantOperationSql } = await import('../../src/lib/server/hosts/observations.js'); + const { importantOperationSql } = + await import('../../src/lib/server/hosts/observations.js'); const { closeDb, db } = await import('../../src/lib/server/db/index.js'); - const ids = Array.from({ length: 35 }, (_, i) => api.add(i % 2 ? 'instance.inspect' : 'instance.restart', { state: 'pending' })); + const ids = Array.from({ length: 35 }, (_, i) => + api.add(i % 2 ? 'instance.inspect' : 'instance.restart', { state: 'pending' }) + ); const verify = () => { - for (const host of [undefined, api.first.id, api.second.id]) for (const server of [undefined, 'server-one']) { - for (const filter of ['all', 'routine', 'important'] as const) { - const where = ['o.history_expired_at IS NULL']; const args: string[] = []; - if (host) { where.push('o.host_id=?'); args.push(host); } - if (server) { where.push('o.server_name=?'); args.push(server); } - if (filter !== 'all') where.push(filter === 'routine' ? 'o.routine=1' : importantOperationSql); - const expected = db().prepare(`SELECT COUNT(*) total FROM ec_host_operations o WHERE ${where.join(' AND ')}`).get(...args) as { total: number }; - expect(api.listHostOperationHistory(1, filter, { host, server }).total).toBe(expected.total); + for (const host of [undefined, api.first.id, api.second.id]) + for (const server of [undefined, 'server-one']) { + for (const filter of ['all', 'routine', 'important'] as const) { + const where = ['o.history_expired_at IS NULL']; + const args: string[] = []; + if (host) { + where.push('o.host_id=?'); + args.push(host); + } + if (server) { + where.push('o.server_name=?'); + args.push(server); + } + if (filter !== 'all') + where.push( + filter === 'routine' ? 'o.routine=1' : importantOperationSql + ); + const expected = db() + .prepare( + `SELECT COUNT(*) total FROM ec_host_operations o WHERE ${where.join(' AND ')}` + ) + .get(...args) as { total: number }; + expect( + api.listHostOperationHistory(1, filter, { host, server }).total + ).toBe(expected.total); + } } - } }; verify(); - for (const [index, id] of ids.entries()) api.reportHostOperationStatus(api.first.id, id, - index % 3 ? { state: 'completed', result: { healthy: index % 5 !== 0 } } : { state: 'failed', error: 'failure' }); - api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one',host_id=? WHERE rowid%2=0").run(api.second.id); - api.handle.prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE rowid%4=0').run(new Date().toISOString()); - api.handle.exec('DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT id FROM ec_host_operations WHERE rowid%7=0); DELETE FROM ec_host_operations WHERE rowid%7=0'); - expect(() => api.handle.transaction(() => { api.handle.exec('DELETE FROM ec_host_operation_events; DELETE FROM ec_host_operations'); throw new Error('rollback'); })()).toThrow('rollback'); - verify(); closeDb(); verify(); + for (const [index, id] of ids.entries()) + api.reportHostOperationStatus( + api.first.id, + id, + index % 3 + ? { state: 'completed', result: { healthy: index % 5 !== 0 } } + : { state: 'failed', error: 'failure' } + ); + api.handle + .prepare( + "UPDATE ec_host_operations SET server_name='server-one',host_id=? WHERE rowid%2=0" + ) + .run(api.second.id); + api.handle + .prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE rowid%4=0') + .run(new Date().toISOString()); + api.handle.exec( + 'DELETE FROM ec_host_operation_events WHERE operation_id IN (SELECT id FROM ec_host_operations WHERE rowid%7=0); DELETE FROM ec_host_operations WHERE rowid%7=0' + ); + expect(() => + api.handle.transaction(() => { + api.handle.exec( + 'DELETE FROM ec_host_operation_events; DELETE FROM ec_host_operations' + ); + throw new Error('rollback'); + })() + ).toThrow('rollback'); + verify(); + closeDb(); + verify(); const prepare = vi.spyOn(db(), 'prepare'); - api.listHostOperationHistory(1); api.listHostOperationHistory(2); + api.listHostOperationHistory(1); + api.listHostOperationHistory(2); expect(prepare.mock.calls.some(([sql]) => /COUNT\(/i.test(sql))).toBe(false); - const count = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operation_history_counts'))![0]; - expect(db().prepare('EXPLAIN QUERY PLAN ' + count).all('', '')).toEqual(expect.arrayContaining([ - expect.objectContaining({ detail: expect.stringContaining('USING PRIMARY KEY') }) - ])); + const count = prepare.mock.calls.find(([sql]) => + sql.includes('FROM ec_host_operation_history_counts') + )![0]; + expect( + db() + .prepare('EXPLAIN QUERY PLAN ' + count) + .all('', '') + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ detail: expect.stringContaining('USING PRIMARY KEY') }) + ]) + ); }); it('scopes counts and pages to one host or server and preserves deleted-host history', async () => { const api = await seedHistory(90); - api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE host_id=?").run(api.first.id); + api.handle + .prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE host_id=?") + .run(api.first.id); const first = api.listHostOperationHistory(1, 'all', { host: api.first.id }); const second = api.listHostOperationHistory(2, 'all', { host: api.first.id }); expect(first).toMatchObject({ total: 30, totalPages: 2 }); expect([...first.operations, ...second.operations]).toHaveLength(30); - expect(first.operations.every(operation => operation.hostId === api.first.id)).toBe(true); + expect(first.operations.every((operation) => operation.hostId === api.first.id)).toBe(true); expect(api.listHostOperationHistory(1, 'all', { server: 'server-one' })).toEqual(first); - expect(api.listHostOperationHistory(1, 'all', { host: api.second.id, server: 'server-one' }).total).toBe(0); - expect(api.listHostOperationHistory(1, 'all', { host: api.deleted.id }).operations.every(operation => operation.hostName === null)).toBe(true); + expect( + api.listHostOperationHistory(1, 'all', { host: api.second.id, server: 'server-one' }) + .total + ).toBe(0); + expect( + api + .listHostOperationHistory(1, 'all', { host: api.deleted.id }) + .operations.every((operation) => operation.hostName === null) + ).toBe(true); const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); - const response = GET({ request: new Request(`https://ec.test/api/hosts/operations?filter=all&host=${api.first.id}&page=2`) }); + const response = GET({ + request: new Request( + `https://ec.test/api/hosts/operations?filter=all&host=${api.first.id}&page=2` + ) + }); expect(await response.json()).toEqual(second); }); @@ -97,43 +187,98 @@ describe('host operation history', () => { const api = await operationHistory(); const first = api.add('instance.restart'); const other = api.queueHostOperation(api.second.id, 'instance.inspect', {}); - api.reportHostOperationStatus(api.second.id, other.id, { state: 'failed', error: 'Unreachable' }); - expect(api.listHostOperationHistory(1, 'important', { host: api.first.id }).operations.map(operation => operation.id)).toEqual([first]); - expect(api.listHostOperationHistory(1, 'important', { host: api.second.id }).operations.map(operation => operation.id)).toEqual([other.id]); + api.reportHostOperationStatus(api.second.id, other.id, { + state: 'failed', + error: 'Unreachable' + }); + expect( + api + .listHostOperationHistory(1, 'important', { host: api.first.id }) + .operations.map((operation) => operation.id) + ).toEqual([first]); + expect( + api + .listHostOperationHistory(1, 'important', { host: api.second.id }) + .operations.map((operation) => operation.id) + ).toEqual([other.id]); }); it('finds an exact old operation without walking pages and keeps all scope and retention filters', async () => { const api = await seedHistory(1003); const operation = api.ids[0]; - api.handle.prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE id=?").run(operation); + api.handle + .prepare("UPDATE ec_host_operations SET server_name='server-one' WHERE id=?") + .run(operation); const prepare = vi.spyOn(api.handle, 'prepare'); - const history = api.listHostOperationHistory(999, 'all', { operation, host: api.first.id, server: 'server-one' }); - const exactCount = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.startsWith('SELECT 1 AS total'))!; + const history = api.listHostOperationHistory(999, 'all', { + operation, + host: api.first.id, + server: 'server-one' + }); + const exactCount = prepare.mock.calls + .map(([sql]) => sql) + .find((sql) => sql.startsWith('SELECT 1 AS total'))!; prepare.mockRestore(); expect(history).toMatchObject({ page: 1, total: 1, totalPages: 1 }); - expect(history.operations.map(row => row.id)).toEqual([operation]); + expect(history.operations.map((row) => row.id)).toEqual([operation]); expect(JSON.stringify(history)).not.toContain('secret'); - const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${exactCount}`).all(api.first.id, 'server-one', operation) as { detail: string }[]; - expect(plan.some(row => row.detail.includes('USING INDEX sqlite_autoindex_ec_host_operations_1'))).toBe(true); - expect(api.listHostOperationHistory(1, 'all', { operation, host: api.second.id }).total).toBe(0); - expect(api.listHostOperationHistory(1, 'all', { operation, server: 'another-server' }).total).toBe(0); + const plan = api.handle + .prepare(`EXPLAIN QUERY PLAN ${exactCount}`) + .all(api.first.id, 'server-one', operation) as { detail: string }[]; + expect( + plan.some((row) => + row.detail.includes('USING INDEX sqlite_autoindex_ec_host_operations_1') + ) + ).toBe(true); + expect( + api.listHostOperationHistory(1, 'all', { operation, host: api.second.id }).total + ).toBe(0); + expect( + api.listHostOperationHistory(1, 'all', { operation, server: 'another-server' }).total + ).toBe(0); expect(api.listHostOperationHistory(1, 'routine', { operation }).total).toBe(0); - expect(api.listHostOperationHistory(1, 'all', { operation: 'missing-operation' }).total).toBe(0); - api.handle.prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE id=?').run(new Date().toISOString(), operation); - expect(api.listHostOperationHistory(1, 'all', { operation })).toMatchObject({ operations: [], page: 1, total: 0 }); + expect( + api.listHostOperationHistory(1, 'all', { operation: 'missing-operation' }).total + ).toBe(0); + api.handle + .prepare('UPDATE ec_host_operations SET history_expired_at=? WHERE id=?') + .run(new Date().toISOString(), operation); + expect(api.listHostOperationHistory(1, 'all', { operation })).toMatchObject({ + operations: [], + page: 1, + total: 0 + }); }); it('pages all hosts in stable newest-first order, retaining deleted-host records', async () => { const api = await seedHistory(); - const pages = [1, 2, 3].map(page => api.listHostOperationHistory(page, 'all')); - expect(pages.map(page => page.operations.length)).toEqual([25, 25, 3]); - pages.forEach((page, index) => expect(page).toMatchObject({ page: index + 1, pageSize: 25, total: 53, totalPages: 3, filter: 'all' })); - const operations = pages.flatMap(page => page.operations); - expect(operations.map(operation => operation.id)).toEqual([...api.ids].reverse()); - expect(operations.find(operation => operation.hostId === api.first.id)?.hostName).toBe('Alpha host'); - expect(operations.find(operation => operation.hostId === api.second.id)?.hostName).toBe('Beta host'); - expect(operations.filter(operation => operation.hostId === api.deleted.id).length).toBeGreaterThan(0); - expect(operations.filter(operation => operation.hostId === api.deleted.id).every(operation => operation.hostName === null)).toBe(true); + const pages = [1, 2, 3].map((page) => api.listHostOperationHistory(page, 'all')); + expect(pages.map((page) => page.operations.length)).toEqual([25, 25, 3]); + pages.forEach((page, index) => + expect(page).toMatchObject({ + page: index + 1, + pageSize: 25, + total: 53, + totalPages: 3, + filter: 'all' + }) + ); + const operations = pages.flatMap((page) => page.operations); + expect(operations.map((operation) => operation.id)).toEqual([...api.ids].reverse()); + expect(operations.find((operation) => operation.hostId === api.first.id)?.hostName).toBe( + 'Alpha host' + ); + expect(operations.find((operation) => operation.hostId === api.second.id)?.hostName).toBe( + 'Beta host' + ); + expect( + operations.filter((operation) => operation.hostId === api.deleted.id).length + ).toBeGreaterThan(0); + expect( + operations + .filter((operation) => operation.hostId === api.deleted.id) + .every((operation) => operation.hostName === null) + ).toBe(true); expect(api.listHostOperationHistory(999, 'all')).toEqual(pages[2]); }); @@ -144,20 +289,38 @@ describe('host operation history', () => { api.listHostOperationHistory(1, 'important'); api.listHostOperationHistory(1, 'routine'); const statements = prepare.mock.calls.map(([sql]) => sql); - const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; + const query = prepare.mock.calls + .map(([sql]) => sql) + .find((sql) => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; prepare.mockRestore(); - expect(statements.every(sql => !/\b(?:payload|result|json_extract|ec_host_operation_payloads)\b/.test(sql))).toBe(true); + expect( + statements.every( + (sql) => !/\b(?:payload|result|json_extract|ec_host_operation_payloads)\b/.test(sql) + ) + ).toBe(true); const projection = query.split('FROM ec_host_operations')[0]; expect(projection).not.toContain('payload'); expect(projection).not.toContain('result'); expect(projection).not.toContain('*'); expect(query).toContain('LIMIT ? OFFSET ?'); - const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { detail: string }[]; - expect(plan.some(row => row.detail.includes('USING INDEX ec_host_operations_history'))).toBe(true); - expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false); + const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { + detail: string; + }[]; + expect( + plan.some((row) => row.detail.includes('USING INDEX ec_host_operations_history')) + ).toBe(true); + expect(plan.some((row) => row.detail.includes('TEMP B-TREE'))).toBe(false); expect(history.operations).toHaveLength(25); - expect(history.operations[0]).toMatchObject({ instanceId: 'instance-one', generation: 2, state: 'completed' }); - expect(history.operations.every(operation => !('payload' in operation) && !('result' in operation))).toBe(true); + expect(history.operations[0]).toMatchObject({ + instanceId: 'instance-one', + generation: 2, + state: 'completed' + }); + expect( + history.operations.every( + (operation) => !('payload' in operation) && !('result' in operation) + ) + ).toBe(true); expect(JSON.stringify(history)).not.toContain('secret'); }); @@ -166,18 +329,33 @@ describe('host operation history', () => { for (const filter of ['important', 'routine'] as const) { const prepare = vi.spyOn(api.handle, 'prepare'); api.listHostOperationHistory(2, filter); - const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; + const query = prepare.mock.calls + .map(([sql]) => sql) + .find((sql) => sql.includes('FROM ec_host_operations o LEFT JOIN'))!; prepare.mockRestore(); - const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { detail: string }[]; - expect(plan.some(row => row.detail.includes(`USING INDEX ec_host_operations_${filter}_history`))).toBe(true); - expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false); + const plan = api.handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(25, 25) as { + detail: string; + }[]; + expect( + plan.some((row) => + row.detail.includes(`USING INDEX ec_host_operations_${filter}_history`) + ) + ).toBe(true); + expect(plan.some((row) => row.detail.includes('TEMP B-TREE'))).toBe(false); expect(query).not.toMatch(/\b(?:payload|result|json_extract)\b/); } }); it('returns an empty first page and normalizes invalid internal page numbers', async () => { const api = await seedHistory(0); - const empty = { operations: [], page: 1, pageSize: 25, total: 0, totalPages: 1, filter: 'important' }; + const empty = { + operations: [], + page: 1, + pageSize: 25, + total: 0, + totalPages: 1, + filter: 'important' + }; expect(api.listHostOperationHistory()).toEqual(empty); for (const page of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, 999]) { expect(api.listHostOperationHistory(page)).toEqual(empty); @@ -188,11 +366,20 @@ describe('host operation history', () => { const api = await operationHistory(); const actions = Array.from({ length: 26 }, () => api.add('instance.restart')); actions.push(api.add('rcon.execute', { payload: { command: 'map_restart' } })); - const successful = Array.from({ length: 26 }, () => api.add('files.manifest', { result: { files: [] } })); - const pending = ['pending', 'accepted', 'running'].map(state => api.add('files.read', { state })); - const failures = ['failed', 'unknown'].map(state => api.add('files.read', { state, error: 'Observation unavailable' })); + const successful = Array.from({ length: 26 }, () => + api.add('files.manifest', { result: { files: [] } }) + ); + const pending = ['pending', 'accepted', 'running'].map((state) => + api.add('files.read', { state }) + ); + const failures = ['failed', 'unknown'].map((state) => + api.add('files.read', { state, error: 'Observation unavailable' }) + ); const negative = [ - api.add('rcon.execute', { payload: { command: 'status' }, result: { ok: false, reason: 'timeout' } }), + api.add('rcon.execute', { + payload: { command: 'status' }, + result: { ok: false, reason: 'timeout' } + }), api.add('instance.inspect', { result: { healthy: false, state: 'stopped' } }), api.add('files.read', { result: { error: 'Read was incomplete' } }), api.add('files.stat', { error: 'Could not inspect the file' }) @@ -203,78 +390,176 @@ describe('host operation history', () => { ['routine', [...successful, ...pending, ...failures, ...negative]], ['all', ids] ] as const) { - const pages = Array.from({ length: Math.ceil(matching.length / 25) }, (_, index) => api.listHostOperationHistory(index + 1, filter)); - expect(pages[0]).toMatchObject({ filter, total: matching.length, totalPages: pages.length, pageSize: 25 }); - expect(pages.flatMap(page => page.operations.map(operation => operation.id))).toEqual([...matching].reverse()); + const pages = Array.from({ length: Math.ceil(matching.length / 25) }, (_, index) => + api.listHostOperationHistory(index + 1, filter) + ); + expect(pages[0]).toMatchObject({ + filter, + total: matching.length, + totalPages: pages.length, + pageSize: 25 + }); + expect( + pages.flatMap((page) => page.operations.map((operation) => operation.id)) + ).toEqual([...matching].reverse()); expect(api.listHostOperationHistory(999, filter)).toEqual(pages.at(-1)); } - expect(api.listHostOperationHistory()).toEqual(api.listHostOperationHistory(1, 'important')); + expect(api.listHostOperationHistory()).toEqual( + api.listHostOperationHistory(1, 'important') + ); const routine = api.listHostOperationHistory(1, 'routine').operations; - expect(routine.every(operation => operation.routine)).toBe(true); - expect(api.listHostOperationHistory(2, 'important').operations.every(operation => !operation.routine)).toBe(true); + expect(routine.every((operation) => operation.routine)).toBe(true); + expect( + api + .listHostOperationHistory(2, 'important') + .operations.every((operation) => !operation.routine) + ).toBe(true); }); it('keeps manual RCON commands visible and only exposes safe read command names', async () => { const api = await operationHistory(); - const reads = ['status', 'getstatus', 'version'].map(command => api.add('rcon.execute', { payload: { command } })); + const reads = ['status', 'getstatus', 'version'].map((command) => + api.add('rcon.execute', { payload: { command } }) + ); const manual = api.add('rcon.execute', { payload: { command: 'say credential-secret' } }); const combined = api.add('rcon.execute', { payload: { command: 'status; map_restart' } }); const important = api.listHostOperationHistory(); - expect(important.operations.map(operation => operation.id)).toEqual([combined, manual]); - expect(important.operations.every(operation => !operation.routine && operation.targetPath === null)).toBe(true); + expect(important.operations.map((operation) => operation.id)).toEqual([combined, manual]); + expect( + important.operations.every( + (operation) => !operation.routine && operation.targetPath === null + ) + ).toBe(true); const routine = api.listHostOperationHistory(1, 'routine'); - expect(routine.operations.map(operation => operation.id)).toEqual([...reads].reverse()); - expect(routine.operations.map(operation => operation.targetPath)).toEqual(['version', 'getstatus', 'status']); - expect(JSON.stringify(api.listHostOperationHistory(1, 'all'))).not.toContain('credential-secret'); + expect(routine.operations.map((operation) => operation.id)).toEqual([...reads].reverse()); + expect(routine.operations.map((operation) => operation.targetPath)).toEqual([ + 'version', + 'getstatus', + 'status' + ]); + expect(JSON.stringify(api.listHostOperationHistory(1, 'all'))).not.toContain( + 'credential-secret' + ); }); it('persists server, target, and source summaries without depending on current instance records', async () => { const api = await operationHistory(); - const { withOperationSource } = await import('../../src/lib/server/hosts/operation-context.js'); - api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('history-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`) + const { withOperationSource } = + await import('../../src/lib/server/hosts/operation-context.js'); + api.handle + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('history-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)` + ) .run(api.first.id, new Date().toISOString()); - const read = withOperationSource('Routine status', () => api.queueHostOperation(api.first.id, 'files.read', { - path: 'configs/server.cfg', credential: 'payload-secret' - }, { instanceId: 'history-instance', generation: 1 })); - const manifest = withOperationSource('Watchdog drift scan', () => api.queueHostOperation(api.first.id, 'files.manifest', { - roots: ['scripts', 'configs', 'plugins'], ignoreNames: [], paths: ['configs/server.cfg'] - }, { instanceId: 'history-instance', generation: 1 })); - api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { contentBase64: 'result-secret' } }); + const read = withOperationSource('Routine status', () => + api.queueHostOperation( + api.first.id, + 'files.read', + { + path: 'configs/server.cfg', + credential: 'payload-secret' + }, + { instanceId: 'history-instance', generation: 1 } + ) + ); + const manifest = withOperationSource('Watchdog drift scan', () => + api.queueHostOperation( + api.first.id, + 'files.manifest', + { + roots: ['scripts', 'configs', 'plugins'], + ignoreNames: [], + paths: ['configs/server.cfg'] + }, + { instanceId: 'history-instance', generation: 1 } + ) + ); + api.reportHostOperationStatus(api.first.id, read.id, { + state: 'completed', + result: { contentBase64: 'result-secret' } + }); api.handle.prepare('DELETE FROM native_instances WHERE id=?').run('history-instance'); const history = api.listHostOperationHistory(1, 'all'); - expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({ - serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: 'Routine status', routine: true + expect(history.operations.find((operation) => operation.id === read.id)).toMatchObject({ + serverName: 't5_zombie_theater_1', + targetPath: 'configs/server.cfg', + source: 'Routine status', + routine: true }); - expect(history.operations.find(operation => operation.id === manifest.id)).toMatchObject({ - serverName: 't5_zombie_theater_1', targetPath: 'scripts, configs, plugins', source: 'Watchdog drift scan', routine: true + expect(history.operations.find((operation) => operation.id === manifest.id)).toMatchObject({ + serverName: 't5_zombie_theater_1', + targetPath: 'scripts, configs, plugins', + source: 'Watchdog drift scan', + routine: true }); expect(JSON.stringify(history)).not.toContain('secret'); }); it('refuses unsafe target summaries and never derives labels from arbitrary payload fields', async () => { const api = await operationHistory(); - const unsafe = ['/etc/private-secret', '../private-secret', 'configs/../../private-secret', 'configs/line\nprivate-secret', 'configs\\private-secret']; - const ids = unsafe.map(path => api.add('files.read', { payload: { path, command: 'command-secret', name: 'name-secret' } })); - ids.push(api.add('files.manifest', { payload: { roots: ['scripts', '../private-secret'] } })); - ids.push(api.add('instance.restart', { payload: { path: 'configs/private-secret', command: 'command-secret' } })); + const unsafe = [ + '/etc/private-secret', + '../private-secret', + 'configs/../../private-secret', + 'configs/line\nprivate-secret', + 'configs\\private-secret' + ]; + const ids = unsafe.map((path) => + api.add('files.read', { + payload: { path, command: 'command-secret', name: 'name-secret' } + }) + ); + ids.push( + api.add('files.manifest', { payload: { roots: ['scripts', '../private-secret'] } }) + ); + ids.push( + api.add('instance.restart', { + payload: { path: 'configs/private-secret', command: 'command-secret' } + }) + ); const history = api.listHostOperationHistory(1, 'all'); expect(history.operations).toHaveLength(ids.length); - expect(history.operations.every(operation => operation.targetPath === null && operation.serverName === null)).toBe(true); + expect( + history.operations.every( + (operation) => operation.targetPath === null && operation.serverName === null + ) + ).toBe(true); expect(JSON.stringify(history)).not.toContain('secret'); }); it('backfills safe legacy summaries from immutable payloads while leaving source unknown', async () => { const api = await operationHistory(); const { closeDb } = await import('../../src/lib/server/db/index.js'); - api.handle.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('legacy-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)`) + api.handle + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('legacy-instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?)` + ) .run(api.first.id, new Date().toISOString()); - const read = api.queueHostOperationSerialized(api.first.id, 'files.read', Buffer.from(JSON.stringify({ - path: 'configs/server.cfg', credential: 'payload-secret' - })), { instanceId: 'legacy-instance', generation: 1 }, { path: 'configs/routing-secret' }); - api.reportHostOperationStatus(api.first.id, read.id, { state: 'completed', result: { credential: 'result-secret' } }); - const command = api.queueHostOperationSerialized(api.first.id, 'rcon.execute', Buffer.from('{"command":"status"}'), {}, { command: 'command-secret' }); + const read = api.queueHostOperationSerialized( + api.first.id, + 'files.read', + Buffer.from( + JSON.stringify({ + path: 'configs/server.cfg', + credential: 'payload-secret' + }) + ), + { instanceId: 'legacy-instance', generation: 1 }, + { path: 'configs/routing-secret' } + ); + api.reportHostOperationStatus(api.first.id, read.id, { + state: 'completed', + result: { credential: 'result-secret' } + }); + const command = api.queueHostOperationSerialized( + api.first.id, + 'rcon.execute', + Buffer.from('{"command":"status"}'), + {}, + { command: 'command-secret' } + ); const manual = api.add('rcon.execute', { payload: { command: 'say command-secret' } }); const unsafe = api.add('files.read', { payload: { path: '../private-secret' } }); api.handle.exec(`DROP INDEX ec_host_operations_routine_history; @@ -292,12 +577,27 @@ describe('host operation history', () => { closeDb(); const history = api.listHostOperationHistory(1, 'all'); expect(history.operations).toHaveLength(4); - expect(history.operations.find(operation => operation.id === read.id)).toMatchObject({ - routine: true, serverName: 't5_zombie_theater_1', targetPath: 'configs/server.cfg', source: null + expect(history.operations.find((operation) => operation.id === read.id)).toMatchObject({ + routine: true, + serverName: 't5_zombie_theater_1', + targetPath: 'configs/server.cfg', + source: null + }); + expect(history.operations.find((operation) => operation.id === command.id)).toMatchObject({ + routine: true, + targetPath: 'status', + source: null + }); + expect(history.operations.find((operation) => operation.id === manual)).toMatchObject({ + routine: false, + targetPath: null, + source: null + }); + expect(history.operations.find((operation) => operation.id === unsafe)).toMatchObject({ + routine: true, + targetPath: null, + source: null }); - expect(history.operations.find(operation => operation.id === command.id)).toMatchObject({ routine: true, targetPath: 'status', source: null }); - expect(history.operations.find(operation => operation.id === manual)).toMatchObject({ routine: false, targetPath: null, source: null }); - expect(history.operations.find(operation => operation.id === unsafe)).toMatchObject({ routine: true, targetPath: null, source: null }); expect(JSON.stringify(history)).not.toContain('secret'); closeDb(); expect(api.listHostOperationHistory(1, 'all')).toEqual(history); @@ -306,15 +606,23 @@ describe('host operation history', () => { it('serves fixed-size API pages without caching or exposing private operation data', async () => { const api = await seedHistory(); const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); - const first = GET({ request: new Request('https://ec.test/api/hosts/operations?filter=all') }); + const first = GET({ + request: new Request('https://ec.test/api/hosts/operations?filter=all') + }); expect(first.headers.get('cache-control')).toBe('no-store'); expect(await first.json()).toEqual(api.listHostOperationHistory(1, 'all')); - const second = GET({ request: new Request('https://ec.test/api/hosts/operations?page=2&pageSize=500&filter=all') }); + const second = GET({ + request: new Request( + 'https://ec.test/api/hosts/operations?page=2&pageSize=500&filter=all' + ) + }); const body = await second.json(); expect(body).toEqual(api.listHostOperationHistory(2, 'all')); expect(body.operations).toHaveLength(25); expect(JSON.stringify(body)).not.toContain('secret'); - const last = GET({ request: new Request('https://ec.test/api/hosts/operations?page=999&filter=all') }); + const last = GET({ + request: new Request('https://ec.test/api/hosts/operations?page=999&filter=all') + }); expect(await last.json()).toEqual(api.listHostOperationHistory(3, 'all')); }); @@ -323,10 +631,17 @@ describe('host operation history', () => { const action = api.add('instance.restart'); const read = api.add('instance.inspect', { result: { healthy: true } }); const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); - const request = (query = '') => GET({ request: new Request(`https://ec.test/api/hosts/operations${query}`) }).json(); + const request = (query = '') => + GET({ request: new Request(`https://ec.test/api/hosts/operations${query}`) }).json(); expect(await request()).toEqual(api.listHostOperationHistory()); - expect((await request()).operations.map((operation: { id: string }) => operation.id)).toEqual([action]); - expect((await request('?filter=routine')).operations.map((operation: { id: string }) => operation.id)).toEqual([read]); + expect( + (await request()).operations.map((operation: { id: string }) => operation.id) + ).toEqual([action]); + expect( + (await request('?filter=routine')).operations.map( + (operation: { id: string }) => operation.id + ) + ).toEqual([read]); expect(await request('?filter=all')).toEqual(api.listHostOperationHistory(1, 'all')); }); @@ -334,7 +649,11 @@ describe('host operation history', () => { const api = await seedHistory(); const operation = api.ids[0]; const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); - const response = GET({ request: new Request(`https://ec.test/api/hosts/operations?filter=all&operation=${operation}`) }); + const response = GET({ + request: new Request( + `https://ec.test/api/hosts/operations?filter=all&operation=${operation}` + ) + }); const body = await response.json(); expect(body).toMatchObject({ page: 1, total: 1, totalPages: 1 }); expect(body.operations.map((row: { id: string }) => row.id)).toEqual([operation]); @@ -345,13 +664,19 @@ describe('host operation history', () => { const { GET } = await import('../../src/routes/api/hosts/operations/+server.js'); const url = 'https://ec.test/api/hosts/operations'; for (const page of ['', '0', '-1', '1.5', 'nope', 'Infinity', '9007199254740992']) { - expect(() => GET({ request: new Request(`${url}?page=${page}`) })).toThrow(expect.objectContaining({ status: 400 })); + expect(() => GET({ request: new Request(`${url}?page=${page}`) })).toThrow( + expect.objectContaining({ status: 400 }) + ); } for (const filter of ['', 'unknown', 'ALL', 'important%27%20OR%201=1']) { - expect(() => GET({ request: new Request(`${url}?filter=${filter}`) })).toThrow(expect.objectContaining({ status: 400 })); + expect(() => GET({ request: new Request(`${url}?filter=${filter}`) })).toThrow( + expect.objectContaining({ status: 400 }) + ); } for (const operation of ['', 'x'.repeat(513), 'operation%00bad', 'operation%0Abad']) { - expect(() => GET({ request: new Request(`${url}?operation=${operation}`) })).toThrow(expect.objectContaining({ status: 400 })); + expect(() => GET({ request: new Request(`${url}?operation=${operation}`) })).toThrow( + expect.objectContaining({ status: 400 }) + ); } const blockedHeaders: HeadersInit[] = [ { origin: 'https://other.test' }, @@ -360,14 +685,19 @@ describe('host operation history', () => { { authorization: 'Bearer ec_enroll_secret' } ]; for (const headers of blockedHeaders) { - expect(() => GET({ request: new Request(url, { headers }) })).toThrow(expect.objectContaining({ status: 403 })); + expect(() => GET({ request: new Request(url, { headers }) })).toThrow( + expect.objectContaining({ status: 403 }) + ); } }); it('keeps operation history out of host detail responses', async () => { const api = await seedHistory(); const { GET } = await import('../../src/routes/api/hosts/[id]/+server.js'); - const body = await GET({ request: new Request(`https://ec.test/api/hosts/${api.first.id}`), params: { id: api.first.id } }).json(); + const body = await GET({ + request: new Request(`https://ec.test/api/hosts/${api.first.id}`), + params: { id: api.first.id } + }).json(); expect(body.host.id).toBe(api.first.id); expect(body).not.toHaveProperty('operations'); expect(JSON.stringify(body)).not.toContain('secret'); diff --git a/tests/unit/host-preparation.test.ts b/tests/unit/host-preparation.test.ts index e952a72..88d8c39 100644 --- a/tests/unit/host-preparation.test.ts +++ b/tests/unit/host-preparation.test.ts @@ -13,8 +13,14 @@ const runtime = 'ghcr.io/example/plutainer:provisioning-test'; const initialTime = Date.parse('2026-10-04T12:00:00Z'); const content = 'a'.repeat(100); const inventory: HostInventory = { - hostname: 'game-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true, - storageAvailable: 100 * 1024 ** 3, runtimeImages: [], libraries: [], capabilities: ['host.decommission'] + hostname: 'game-host', + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: [], + libraries: [], + capabilities: ['host.decommission'] }; beforeEach(async () => { @@ -47,24 +53,40 @@ function heartbeat(id: string, overrides: HostInventory = {}) { } function operations(id: string) { - return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download', 'plutonium.prepare'].includes(operation.kind)); + return hosts + .listHostOperations(id) + .filter((operation) => + ['runtime.pull', 'library.download', 'plutonium.prepare'].includes(operation.kind) + ); } async function publish(game: 't5' | 't6' = 't5', bytes = content) { vi.setSystemTime(Date.now() + 1); const upload = libraries.createLibraryUpload({ - game, files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }] + game, + files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }] }); - await libraries.uploadBrowserFile(upload.id, 'zone/game.ff', { - offset: 0, size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') - }, new Response(bytes).body); + await libraries.uploadBrowserFile( + upload.id, + 'zone/game.ff', + { + offset: 0, + size: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex') + }, + new Response(bytes).body + ); return libraries.publishBrowserUpload(upload.id); } function verified(id: string, version: GameLibraryVersion) { - libraries.updateHostLibraryCaches(id, [{ - libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 - }]); + libraries.updateHostLibraryCaches(id, [ + { + libraryVersionId: version.id, + status: 'ready', + manifestSha256: version.manifestSha256 + } + ]); } describe('automatic host preparation', () => { @@ -72,168 +94,276 @@ describe('automatic host preparation', () => { const host = hosts.createHost({ name: 'Heartbeat host' }); const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js'); - const cleanup = vi.spyOn(libraries, 'cleanupControllerLibraries').mockRejectedValue(new Error('Cleanup is independent of heartbeats')); + const cleanup = vi + .spyOn(libraries, 'cleanupControllerLibraries') + .mockRejectedValue(new Error('Cleanup is independent of heartbeats')); try { for (let i = 0; i < 3; i++) { - const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', { - method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } }) - }) }); + const response = await POST({ + request: new Request('https://controller.test/api/agent/heartbeat', { + method: 'POST', + headers: { + Authorization: `Bearer ${credential}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } }) + }) + }); expect(response.status).toBe(200); } expect(cleanup).not.toHaveBeenCalled(); - } finally { cleanup.mockRestore(); } + } finally { + cleanup.mockRestore(); + } }); - it.each([false, true])('keeps re-enabled hosts disabled until required files are verified, without waiting for the optional cache (supported: %s)', async supported => { - const t5 = await publish('t5'); - const t6 = await publish('t6'); - const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })), - capabilities: [...inventory.capabilities!, ...(supported ? ['plutonium.cache'] : [])] }; - const id = enroll(readyInventory); - verified(id, t5); verified(id, t6); - const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime }); - hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' }); - heartbeat(id, readyInventory); - hosts.updateHost(id, { mode: 'disabled' }); - const requested = hosts.updateHost(id, { mode: 'active' }); - expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) }); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); - expect(() => hosts.requireReadyHost(id)).toThrow('before enabling'); - preparation.reconcileHostPreparation(id); - const pull = operations(id)[0]; - expect(pull.id).not.toBe(oldRuntime.id); - expect(pull.kind).toBe('runtime.pull'); - expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]); - expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled'); - expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince); - hosts.reportHostOperation(id, pull.id, { state: 'completed' }); - preparation.reconcileHostPreparation(id); - expect(hosts.getHost(id)?.mode).toBe('disabled'); - heartbeat(id, readyInventory); - for (let index = 0; index < 2; index++) { + it.each([false, true])( + 'keeps re-enabled hosts disabled until required files are verified, without waiting for the optional cache (supported: %s)', + async (supported) => { + const t5 = await publish('t5'); + const t6 = await publish('t6'); + const readyInventory = { + runtimeImages: [runtime], + libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })), + capabilities: [ + ...inventory.capabilities!, + ...(supported ? ['plutonium.cache'] : []) + ] + }; + const id = enroll(readyInventory); + verified(id, t5); + verified(id, t6); + const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime }); + hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' }); + heartbeat(id, readyInventory); + hosts.updateHost(id, { mode: 'disabled' }); + const requested = hosts.updateHost(id, { mode: 'active' }); + expect(requested).toMatchObject({ + mode: 'disabled', + readiness: 'preparing', + enablingSince: expect.any(String) + }); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + expect(() => hosts.requireReadyHost(id)).toThrow('before enabling'); preparation.reconcileHostPreparation(id); - const download = operations(id)[0]; - expect(download.kind).toBe('library.download'); - expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]); - const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!; - verified(id, version); - hosts.reportHostOperation(id, download.id, { state: 'completed' }); + const pull = operations(id)[0]; + expect(pull.id).not.toBe(oldRuntime.id); + expect(pull.kind).toBe('runtime.pull'); + expect(hosts.pendingHostOperations(id).map((operation) => operation.id)).toEqual([ + pull.id + ]); + expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled'); + expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe( + requested.enablingSince + ); + hosts.reportHostOperation(id, pull.id, { state: 'completed' }); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)?.mode).toBe('disabled'); heartbeat(id, readyInventory); + for (let index = 0; index < 2; index++) { + preparation.reconcileHostPreparation(id); + const download = operations(id)[0]; + expect(download.kind).toBe('library.download'); + expect(hosts.pendingHostOperations(id).map((operation) => operation.id)).toEqual([ + download.id + ]); + const version = [t5, t6].find( + (version) => version.id === download.payload.libraryVersionId + )!; + verified(id, version); + hosts.reportHostOperation(id, download.id, { state: 'completed' }); + preparation.reconcileHostPreparation(id); + expect(hosts.getHost(id)?.mode).toBe('disabled'); + heartbeat(id, readyInventory); + } + preparation.reconcileHostPreparation(id); + expect(hosts.getHost(id)).toMatchObject({ + mode: 'active', + readiness: 'ready', + enablingSince: null + }); + expect( + operations(id).filter((operation) => operation.kind === 'library.download') + ).toHaveLength(2); + expect( + operations(id).filter((operation) => operation.kind === 'plutonium.prepare') + ).toHaveLength(supported ? 1 : 0); } - preparation.reconcileHostPreparation(id); - expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null }); - expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2); - expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(supported ? 1 : 0); - }); + ); it('requires both games when enabling even if one library is already cached', async () => { const version = await publish('t5'); - const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); + const id = enroll({ + runtimeImages: [runtime], + libraries: [{ id: version.id, state: 'ready' }] + }); verified(id, version); hosts.updateHost(id, { mode: 'disabled' }); hosts.updateHost(id, { mode: 'active' }); const status = preparation.reconcileHostPreparation(id); - expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II', state: 'blocked' })); + expect(status.items).toContainEqual( + expect.objectContaining({ label: 'Black Ops II', state: 'blocked' }) + ); expect(hosts.getHost(id)?.mode).toBe('disabled'); }); it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => { const t5 = await publish('t5'); const t6 = await publish('t6'); - const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) }; + const readyInventory = { + runtimeImages: [runtime], + libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })) + }; const id = enroll(readyInventory); - verified(id, t5); verified(id, t6); + verified(id, t5); + verified(id, t6); hosts.updateHost(id, { mode: 'disabled' }); hosts.updateHost(id, { mode: 'active' }); preparation.reconcileHostPreparation(id); - hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' }); + hosts.reportHostOperation(id, operations(id)[0].id, { + state: 'failed', + error: 'Registry unavailable' + }); heartbeat(id, readyInventory); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(() => hosts.requireReadyHost(id)).toThrow('before enabling'); - const queued = operations(id).map(operation => operation.id); + const queued = operations(id).map((operation) => operation.id); vi.setSystemTime(Date.now() + 91_000); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false }); - expect(operations(id).map(operation => operation.id)).toEqual(queued); + expect(operations(id).map((operation) => operation.id)).toEqual(queued); heartbeat(id, readyInventory); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)?.mode).toBe('disabled'); }); it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => { - const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([ + const [ + { POST: enrollAgent }, + { GET: pollOperations }, + { POST: reportOperation }, + { POST: reportHeartbeat } + ] = await Promise.all([ import('../../src/routes/api/agent/enroll/+server.js'), import('../../src/routes/api/agent/operations/+server.js'), import('../../src/routes/api/agent/operations/[id]/+server.js'), import('../../src/routes/api/agent/heartbeat/+server.js') ]); const host = hosts.createHost({ name: 'Agent provisioning integration' }); - const enrollment = await enrollAgent({ request: new Request('https://controller.test/api/agent/enroll', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory }) - }) }); + const enrollment = await enrollAgent({ + request: new Request('https://controller.test/api/agent/enroll', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory }) + }) + }); expect(enrollment.status).toBe(200); const { credential } = await enrollment.json(); - const request = (path: string, body?: object) => new Request(`https://controller.test/api/agent/${path}`, { - method: body ? 'POST' : 'GET', - headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, - ...(body ? { body: JSON.stringify(body) } : {}) - }); + const request = (path: string, body?: object) => + new Request(`https://controller.test/api/agent/${path}`, { + method: body ? 'POST' : 'GET', + headers: { + Authorization: `Bearer ${credential}`, + 'Content-Type': 'application/json' + }, + ...(body ? { body: JSON.stringify(body) } : {}) + }); expect(operations(host.id)).toEqual([]); // Poll lanes only deliver queued work. One inventory heartbeat drives setup. const reconcile = vi.spyOn(preparation, 'reconcileHostPreparation'); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([]); expect(reconcile).not.toHaveBeenCalled(); - expect((await reportHeartbeat({ request: request('heartbeat', { inventory }) })).status).toBe(200); + expect( + (await reportHeartbeat({ request: request('heartbeat', { inventory }) })).status + ).toBe(200); expect(reconcile).toHaveBeenCalledOnce(); reconcile.mockRestore(); const firstPoll = await (await pollOperations({ request: request('operations') })).json(); expect(firstPoll.operations).toHaveLength(1); const runtimeOperation = firstPoll.operations[0]; - expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } }); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]); + expect(runtimeOperation).toMatchObject({ + kind: 'runtime.pull', + payload: { image: runtime } + }); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([runtimeOperation]); const version = await publish(); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([runtimeOperation]); vi.setSystemTime(Date.now() + 1); const completedRuntime = await reportOperation({ - request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id } + request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), + params: { id: runtimeOperation.id } }); expect(completedRuntime.status).toBe(200); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([]); expect(operations(host.id)).toHaveLength(1); vi.setSystemTime(Date.now() + 1); const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] }; - const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) }); + const runtimeHeartbeat = await reportHeartbeat({ + request: request('heartbeat', { inventory: preparedInventory }) + }); expect(runtimeHeartbeat.status).toBe(200); expect(operations(host.id)).toHaveLength(2); const download = operations(host.id)[0]; - expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } }); - expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([download]); + expect(download).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: version.id } + }); + expect( + ( + await reportHeartbeat({ + request: request('heartbeat', { inventory: preparedInventory }) + }) + ).status + ).toBe(200); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([download]); expect(operations(host.id)).toHaveLength(2); vi.setSystemTime(Date.now() + 1); - expect((await reportOperation({ - request: request(`operations/${download.id}`, { state: 'completed' }), params: { id: download.id } - })).status).toBe(200); + expect( + ( + await reportOperation({ + request: request(`operations/${download.id}`, { state: 'completed' }), + params: { id: download.id } + }) + ).status + ).toBe(200); vi.setSystemTime(Date.now() + 1); - const finalHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: { - ...preparedInventory, - libraries: [{ id: version.id, state: 'ready' }], - libraryCaches: [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }] - } }) }); + const finalHeartbeat = await reportHeartbeat({ + request: request('heartbeat', { + inventory: { + ...preparedInventory, + libraries: [{ id: version.id, state: 'ready' }], + libraryCaches: [ + { + libraryVersionId: version.id, + status: 'ready', + manifestSha256: version.manifestSha256 + } + ] + } + }) + }); expect(finalHeartbeat.status).toBe(200); expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id); expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready'); - expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]); + expect( + (await (await pollOperations({ request: request('operations') })).json()).operations + ).toEqual([]); expect(operations(host.id)).toHaveLength(2); }); @@ -242,13 +372,17 @@ describe('automatic host preparation', () => { const id = enroll({ runtimeImages: ['another:runtime'] }); const host = hosts.getHost(id)!; const status = preparation.hostPreparationStatus(host); - expect(status.items.map(item => item.kind)).toEqual(['runtime', 'library']); + expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']); expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status }); expect(operations(id)).toEqual([]); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); const [operation] = operations(id); - expect(operation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime }, state: 'pending' }); + expect(operation).toMatchObject({ + kind: 'runtime.pull', + payload: { image: runtime }, + state: 'pending' + }); expect(libraries.getHostLibraryCache(id, version.id)).toBeNull(); for (let poll = 0; poll < 4; poll++) { heartbeat(id); @@ -256,7 +390,7 @@ describe('automatic host preparation', () => { } hosts.reportHostOperation(id, operation.id, { state: 'unknown' }); preparation.reconcileHostPreparation(id); - expect(operations(id).map(item => item.id)).toEqual([operation.id]); + expect(operations(id).map((item) => item.id)).toEqual([operation.id]); }); it('waits for fresh inventory after runtime completion before starting a library download', async () => { @@ -265,7 +399,10 @@ describe('automatic host preparation', () => { preparation.reconcileHostPreparation(id); const runtimeOperation = operations(id)[0]; vi.setSystemTime(Date.now() + 1); - hosts.reportHostOperation(id, runtimeOperation.id, { state: 'completed', result: { image: runtime } }); + hosts.reportHostOperation(id, runtimeOperation.id, { + state: 'completed', + result: { image: runtime } + }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); @@ -273,10 +410,16 @@ describe('automatic host preparation', () => { preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(2); const download = operations(id)[0]; - expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } }); + expect(download).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: version.id } + }); verified(id, version); hosts.reportHostOperation(id, download.id, { state: 'completed' }); - heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); + heartbeat(id, { + runtimeImages: [runtime], + libraries: [{ id: version.id, state: 'ready' }] + }); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); expect(operations(id)).toHaveLength(2); }); @@ -290,11 +433,13 @@ describe('automatic host preparation', () => { files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }] }); const id = enroll({ runtimeImages: [runtime] }); - const expected = new Map([latest, t6].map(version => [version.id, version])); + const expected = new Map([latest, t6].map((version) => [version.id, version])); const downloaded: string[] = []; for (let index = 0; index < expected.size; index++) { preparation.reconcileHostPreparation(id); - const active = operations(id).filter(operation => !['completed', 'failed'].includes(operation.state)); + const active = operations(id).filter( + (operation) => !['completed', 'failed'].includes(operation.state) + ); expect(active).toHaveLength(1); const operation = active[0]; const versionId = String(operation.payload.libraryVersionId); @@ -306,7 +451,10 @@ describe('automatic host preparation', () => { hosts.reportHostOperation(id, operation.id, { state: 'completed' }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(index + 1); - heartbeat(id, { runtimeImages: [runtime], libraries: downloaded.map(id => ({ id, state: 'ready' })) }); + heartbeat(id, { + runtimeImages: [runtime], + libraries: downloaded.map((id) => ({ id, state: 'ready' })) + }); } expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); expect(downloaded).toHaveLength(expected.size); @@ -340,50 +488,83 @@ describe('automatic host preparation', () => { verified(id, old); const latest = await publish(); preparation.reconcileHostPreparation(id); - expect(hosts.getHostOperation(first.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') }); - expect(operations(id).filter(operation => operation.state === 'pending')).toMatchObject([ + expect(hosts.getHostOperation(first.id)).toMatchObject({ + state: 'failed', + error: expect.stringContaining('Superseded') + }); + expect(operations(id).filter((operation) => operation.state === 'pending')).toMatchObject([ { payload: { libraryVersionId: latest.id, automatic: true } } ]); expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); }); - it.each([false, true])('prepares queued libraries without pulling custom runtimes (automatic placement: %s)', async automatic => { - const old = await publish(); - const image = 'ghcr.io/example/plutainer:queued-version'; - const id = enroll({ runtimeImages: [runtime, image] }); - const other = enroll({ runtimeImages: [runtime] }); - const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - for (let index = 0; index < 2; index++) { - expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id, - hostId: automatic ? undefined : id, image, plutoKey: `saved-key-${index}` }).ok).toBe(true); + it.each([false, true])( + 'prepares queued libraries without pulling custom runtimes (automatic placement: %s)', + async (automatic) => { + const old = await publish(); + const image = 'ghcr.io/example/plutainer:queued-version'; + const id = enroll({ runtimeImages: [runtime, image] }); + const other = enroll({ runtimeImages: [runtime] }); + const { queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); + for (let index = 0; index < 2; index++) { + expect( + queueNativeCreation({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + libraryId: old.id, + hostId: automatic ? undefined : id, + image, + plutoKey: `saved-key-${index}` + }).ok + ).toBe(true); + } + // Also cover pre-existing intents whose image is no longer cached. + heartbeat(id, { runtimeImages: [runtime] }); + const latest = await publish(); + const status = preparation.hostPreparationStatus(hosts.getHost(id)!); + expect(status.items.filter((item) => item.label === image)).toHaveLength(0); + expect( + status.items.filter((item) => item.label.includes('Queued version')) + ).toHaveLength(1); + const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!); + expect(otherStatus.items.some((item) => item.label === image)).toBe(false); + expect(otherStatus.items.some((item) => item.label.includes('Queued version'))).toBe( + automatic + ); + expect(operations(id)).toEqual([]); + + preparation.reconcileHostPreparation(id); + const download = operations(id)[0]; + expect(download).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: old.id, automatic: true } + }); + preparation.reconcileHostPreparation(id); + expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); + expect(operations(id)).toHaveLength(1); + preparation.reconcileHostPreparation(other); + expect( + [...operations(id), ...operations(other)].every( + (operation) => operation.kind !== 'runtime.pull' + ) + ).toBe(true); + + verified(id, old); + vi.setSystemTime(Date.now() + 1); + hosts.reportHostOperation(id, download.id, { state: 'completed' }); + heartbeat(id, { + runtimeImages: [runtime, image], + libraries: [{ id: old.id, state: 'ready' }] + }); + preparation.reconcileHostPreparation(id); + expect(operations(id)[0]).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: latest.id } + }); } - // Also cover pre-existing intents whose image is no longer cached. - heartbeat(id, { runtimeImages: [runtime] }); - const latest = await publish(); - const status = preparation.hostPreparationStatus(hosts.getHost(id)!); - expect(status.items.filter(item => item.label === image)).toHaveLength(0); - expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1); - const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!); - expect(otherStatus.items.some(item => item.label === image)).toBe(false); - expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic); - expect(operations(id)).toEqual([]); - - preparation.reconcileHostPreparation(id); - const download = operations(id)[0]; - expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } }); - preparation.reconcileHostPreparation(id); - expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); - expect(operations(id)).toHaveLength(1); - preparation.reconcileHostPreparation(other); - expect([...operations(id), ...operations(other)].every(operation => operation.kind !== 'runtime.pull')).toBe(true); - - verified(id, old); - vi.setSystemTime(Date.now() + 1); - hosts.reportHostOperation(id, download.id, { state: 'completed' }); - heartbeat(id, { runtimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] }); - preparation.reconcileHostPreparation(id); - expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); - }); + ); it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => { const old = await publish(); @@ -391,8 +572,15 @@ describe('automatic host preparation', () => { preparation.reconcileHostPreparation(id); const download = operations(id)[0]; const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const launch = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id, - hostId: id, image: runtime, plutoKey: 'saved-key' }); + const launch = queueNativeCreation({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + libraryId: old.id, + hostId: id, + image: runtime, + plutoKey: 'saved-key' + }); expect(launch.ok).toBe(true); if (!launch.ok) throw new Error(launch.message); const latest = await publish(); @@ -400,11 +588,18 @@ describe('automatic host preparation', () => { expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); expect(operations(id)).toHaveLength(1); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String)); preparation.reconcileHostPreparation(id); - expect(hosts.getHostOperation(download.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') }); - expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); + expect(hosts.getHostOperation(download.id)).toMatchObject({ + state: 'failed', + error: expect.stringContaining('Superseded') + }); + expect(operations(id)[0]).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: latest.id } + }); }); it('finishes a delivered download before following a newer upload under a new operation ID', async () => { @@ -420,11 +615,14 @@ describe('automatic host preparation', () => { hosts.reportHostOperation(id, first.id, { state: 'completed' }); heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] }); preparation.reconcileHostPreparation(id); - expect(operations(id)[0]).toMatchObject({ state: 'pending', payload: { libraryVersionId: latest.id } }); + expect(operations(id)[0]).toMatchObject({ + state: 'pending', + payload: { libraryVersionId: latest.id } + }); expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); }); - it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', mode => { + it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', (mode) => { const id = enroll(); hosts.updateHost(id, { mode }); preparation.reconcileHostPreparation(id); @@ -463,10 +661,15 @@ describe('automatic host preparation', () => { it('retries runtime failures with exponential delays capped at fifteen minutes', () => { const id = enroll(); preparation.reconcileHostPreparation(id); - for (const [attempt, delay] of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000].entries()) { + for (const [attempt, delay] of [ + 60_000, 120_000, 240_000, 480_000, 900_000, 900_000 + ].entries()) { const operation = operations(id)[0]; vi.setSystemTime(Date.now() + 1); - hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Registry unavailable' }); + hosts.reportHostOperation(id, operation.id, { + state: 'failed', + error: 'Registry unavailable' + }); const failedAt = Date.now(); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(attempt + 1); @@ -484,7 +687,10 @@ describe('automatic host preparation', () => { it('blocks a library that cannot fit while preserving the host free-space reserve', async () => { const version = await publish(); - const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 }); + const id = enroll({ + runtimeImages: [runtime], + storageAvailable: 1_000 + version.sizeBytes - 1 + }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(operations(id)).toEqual([]); @@ -497,7 +703,10 @@ describe('automatic host preparation', () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime] }); const operation = libraries.queueLibraryDownload(id, version.id); - heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes }); + heartbeat(id, { + runtimeImages: [runtime], + storageAvailable: hosts.getHost(id)!.minimumFreeBytes + }); expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); @@ -509,10 +718,12 @@ describe('automatic host preparation', () => { const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); const status = preparation.reconcileHostPreparation(id); - expect(status.items).toEqual(expect.arrayContaining([ - expect.objectContaining({ label: 'Black Ops I', state: 'blocked' }), - expect.objectContaining({ label: 'Black Ops II', state: 'preparing' }) - ])); + expect(status.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ label: 'Black Ops I', state: 'blocked' }), + expect.objectContaining({ label: 'Black Ops II', state: 'preparing' }) + ]) + ); expect(operations(id)).toHaveLength(1); expect(operations(id)[0].payload.libraryVersionId).toBe(small.id); expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull(); @@ -521,12 +732,22 @@ describe('automatic host preparation', () => { it('does not count partial bytes from a different manifest toward download storage', async () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime] }); - libraries.updateHostLibraryCaches(id, [{ - libraryVersionId: version.id, status: 'failed', bytesDownloaded: 80, manifestSha256: '0'.repeat(64) - }]); - heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 }); + libraries.updateHostLibraryCaches(id, [ + { + libraryVersionId: version.id, + status: 'failed', + bytesDownloaded: 80, + manifestSha256: '0'.repeat(64) + } + ]); + heartbeat(id, { + runtimeImages: [runtime], + storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 + }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); - expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(/insufficient storage/); + expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow( + /insufficient storage/ + ); expect(operations(id)).toEqual([]); }); @@ -535,9 +756,14 @@ describe('automatic host preparation', () => { const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const operation = operations(id)[0]; - libraries.updateHostLibraryCaches(id, [{ - libraryVersionId: version.id, status: 'downloading', bytesDownloaded: 80, manifestSha256: version.manifestSha256 - }]); + libraries.updateHostLibraryCaches(id, [ + { + libraryVersionId: version.id, + status: 'downloading', + bytesDownloaded: 80, + manifestSha256: version.manifestSha256 + } + ]); vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' }); const failedAt = Date.now(); @@ -556,34 +782,56 @@ describe('automatic host preparation', () => { describe('optional host Plutonium cache', () => { const capable = { capabilities: ['host.decommission', 'plutonium.cache'] }; - async function release(revision: number, overrides: { checkedAt?: string; error?: string | null } = {}) { + async function release( + revision: number, + overrides: { checkedAt?: string; error?: string | null } = {} + ) { const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js'); - writeSetting('plutonium_release', { revision, checkedAt: new Date().toISOString(), error: null, ...overrides }); + writeSetting('plutonium_release', { + revision, + checkedAt: new Date().toISOString(), + error: null, + ...overrides + }); } - it.each(['pending', 'accepted', 'running', 'unknown'] as const)('waits to warm while a deployment is %s', state => { - const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] }; - const id = enroll(ready); - const deployment = hosts.queueHostOperation(id, 'instance.create', {}); - if (state !== 'pending') hosts.reportHostOperation(id, deployment.id, { state }); - preparation.reconcileHostPreparation(id); - expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true); - expect(operations(id)).toEqual([]); - hosts.reportHostOperation(id, deployment.id, { state: 'completed' }); - heartbeat(id, ready); - preparation.reconcileHostPreparation(id); - expect(operations(id)).toMatchObject([{ kind: 'plutonium.prepare' }]); - }); + it.each(['pending', 'accepted', 'running', 'unknown'] as const)( + 'waits to warm while a deployment is %s', + (state) => { + const ready = { + ...capable, + runtimeImages: [runtime], + libraries: [{ id: 'game-files', state: 'ready' as const }] + }; + const id = enroll(ready); + const deployment = hosts.queueHostOperation(id, 'instance.create', {}); + if (state !== 'pending') hosts.reportHostOperation(id, deployment.id, { state }); + preparation.reconcileHostPreparation(id); + expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true); + expect(operations(id)).toEqual([]); + hosts.reportHostOperation(id, deployment.id, { state: 'completed' }); + heartbeat(id, ready); + preparation.reconcileHostPreparation(id); + expect(operations(id)).toMatchObject([{ kind: 'plutonium.prepare' }]); + } + ); it('delivers newer deployments before queued cache work and before a cache replay', () => { - const id = enroll({ ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' }] }); + const id = enroll({ + ...capable, + runtimeImages: [runtime], + libraries: [{ id: 'game-files', state: 'ready' }] + }); preparation.reconcileHostPreparation(id); const cache = operations(id)[0]; const deployment = hosts.queueHostOperation(id, 'instance.create', {}); - expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([deployment.id, cache.id]); + expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([ + deployment.id, + cache.id + ]); hosts.reportHostOperation(id, cache.id, { state: 'running' }); const change = hosts.queueHostOperation(id, 'files.stage', {}); - const delivery = hosts.pendingHostOperations(id, 'write').map(operation => operation.id); + const delivery = hosts.pendingHostOperations(id, 'write').map((operation) => operation.id); expect(delivery.at(-1)).toBe(cache.id); expect(delivery).toContain(deployment.id); expect(delivery).toContain(change.id); @@ -591,40 +839,74 @@ describe('optional host Plutonium cache', () => { it('signals deferral on heartbeats for mutations while ignoring cache work and all observation lanes', async () => { const host = hosts.createHost({ name: 'Cache yielding host' }); - const ready = { ...inventory, ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] }; + const ready = { + ...inventory, + ...capable, + runtimeImages: [runtime], + libraries: [{ id: 'game-files', state: 'ready' as const }] + }; const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready); preparation.reconcileHostPreparation(host.id); const cache = operations(host.id)[0]; hosts.reportHostOperation(host.id, cache.id, { state: 'running' }); - for (const kind of ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read']) { + for (const kind of [ + 'host.inspect', + 'instance.inspect', + 'rcon.status-batch', + 'files.read', + 'files.list', + 'files.stat', + 'files.manifest', + 'logs.read' + ]) { hosts.queueHostOperation(host.id, kind, {}); } - for (const command of ['status', 'getstatus', 'version']) hosts.queueHostOperation(host.id, 'rcon.execute', { command }); + for (const command of ['status', 'getstatus', 'version']) + hosts.queueHostOperation(host.id, 'rcon.execute', { command }); // Work belonging to another host must not pause this one's cache. const other = enroll(ready); hosts.queueHostOperation(other, 'instance.restart', {}); const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js'); const poll = async () => { vi.setSystemTime(Date.now() + 1); - const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', { - method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ inventory: ready }) - }) }); + const response = await POST({ + request: new Request('https://controller.test/api/agent/heartbeat', { + method: 'POST', + headers: { + Authorization: `Bearer ${credential}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ inventory: ready }) + }) + }); expect(response.status).toBe(200); return response.json(); }; expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false }); - const change = hosts.queueHostOperation(host.id, 'rcon.execute', { command: 'map_restart' }); + const change = hosts.queueHostOperation(host.id, 'rcon.execute', { + command: 'map_restart' + }); expect(await poll()).toMatchObject({ deferPlutoniumPreparation: true }); - hosts.reportHostOperation(host.id, change.id, { state: 'failed', error: 'Command rejected' }); + hosts.reportHostOperation(host.id, change.id, { + state: 'failed', + error: 'Command rejected' + }); expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false }); - expect(operations(host.id).filter(operation => operation.kind === 'plutonium.prepare').map(operation => operation.id)).toEqual([cache.id]); + expect( + operations(host.id) + .filter((operation) => operation.kind === 'plutonium.prepare') + .map((operation) => operation.id) + ).toEqual([cache.id]); }); it('lets newly published game files interrupt cache refresh, then resumes the same cache operation', async () => { const version = await publish(); - const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }], - plutoniumCache: { revision: 5354, bytes: 200 } }; + const ready = { + ...capable, + runtimeImages: [runtime], + libraries: [{ id: version.id, state: 'ready' as const }], + plutoniumCache: { revision: 5354, bytes: 200 } + }; const id = enroll(ready); verified(id, version); await release(5355); @@ -636,16 +918,26 @@ describe('optional host Plutonium cache', () => { heartbeat(id, ready); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; - expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); + expect(download).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: latest.id } + }); expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true); - expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([download.id, cache.id]); + expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([ + download.id, + cache.id + ]); verified(id, latest); hosts.reportHostOperation(id, download.id, { state: 'completed' }); heartbeat(id, { ...ready, libraries: [{ id: latest.id, state: 'ready' }] }); preparation.reconcileHostPreparation(id); expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(false); - expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]); - expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(1); + expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([ + cache.id + ]); + expect( + operations(id).filter((operation) => operation.kind === 'plutonium.prepare') + ).toHaveLength(1); }); it('warms only after required files, preserves launch readiness, and waits for fresh cache inventory', async () => { @@ -662,27 +954,47 @@ describe('optional host Plutonium cache', () => { expect(download.kind).toBe('library.download'); verified(id, version); hosts.reportHostOperation(id, download.id, { state: 'completed' }); - const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] }; + const ready = { + ...capable, + runtimeImages: [runtime], + libraries: [{ id: version.id, state: 'ready' as const }] + }; heartbeat(id, ready); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); const cache = operations(id)[0]; - expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: { - revision: 5354, minimumFreeBytes: hosts.getHost(id)!.minimumFreeBytes, automatic: true - } }); + expect(cache).toMatchObject({ + kind: 'plutonium.prepare', + payload: { + revision: 5354, + minimumFreeBytes: hosts.getHost(id)!.minimumFreeBytes, + automatic: true + } + }); expect(hosts.requireReadyHost(id).readiness).toBe('ready'); expect(hosts.pendingHostOperations(id, 'read')).toEqual([]); - expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]); - expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0]).toMatchObject({ - id: cache.id, routine: false, targetPath: 'Production revision r5354' + expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([ + cache.id + ]); + expect( + hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0] + ).toMatchObject({ + id: cache.id, + routine: false, + targetPath: 'Production revision r5354' }); preparation.reconcileHostPreparation(id); hosts.reportHostOperation(id, cache.id, { state: 'completed' }); const pendingInventory = preparation.reconcileHostPreparation(id); - expect(pendingInventory.items.find(item => item.kind === 'plutonium')).toMatchObject({ state: 'preparing' }); + expect(pendingInventory.items.find((item) => item.kind === 'plutonium')).toMatchObject({ + state: 'preparing' + }); expect(operations(id)).toHaveLength(3); heartbeat(id, { ...ready, plutoniumCache: { revision: 5354, bytes: 25_000 } }); - expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({ - state: 'ready', detail: expect.stringContaining('r5354') + expect( + preparation.reconcileHostPreparation(id).items.find((item) => item.kind === 'plutonium') + ).toMatchObject({ + state: 'ready', + detail: expect.stringContaining('r5354') }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); @@ -691,7 +1003,11 @@ describe('optional host Plutonium cache', () => { }); it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => { - const ready = { ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 25_000 } }; + const ready = { + ...capable, + runtimeImages: [runtime], + plutoniumCache: { revision: 5354, bytes: 25_000 } + }; const id = enroll(ready); await release(5354); preparation.reconcileHostPreparation(id); @@ -705,11 +1021,16 @@ describe('optional host Plutonium cache', () => { heartbeat(id, ready); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); - expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({ - state: 'preparing', detail: expect.stringContaining('reconcile interrupted') + expect( + preparation.reconcileHostPreparation(id).items.find((item) => item.kind === 'plutonium') + ).toMatchObject({ + state: 'preparing', + detail: expect.stringContaining('reconcile interrupted') }); expect(operations(id)).toHaveLength(1); - expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]); + expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([ + cache.id + ]); hosts.reportHostOperation(id, cache.id, { state: 'completed' }); heartbeat(id, { ...ready, plutoniumCache: { revision: 5356, bytes: 26_000 } }); preparation.reconcileHostPreparation(id); @@ -719,40 +1040,69 @@ describe('optional host Plutonium cache', () => { expect(operations(id)).toHaveLength(1); }); - it.each(['missing', 'stale', 'failed'] as const)('uses the agent production manifest when release evidence is %s', async evidence => { - if (evidence === 'stale') await release(5354, { checkedAt: new Date(Date.now() - 60 * 60_000).toISOString() }); - if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' }); - const id = enroll({ ...capable, runtimeImages: [runtime] }); - preparation.reconcileHostPreparation(id); - expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 0 } }); - expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0].targetPath).toBe('Latest production revision'); - const cached = enroll({ ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5353, bytes: 200 } }); - preparation.reconcileHostPreparation(cached); - expect(operations(cached)).toEqual([]); - }); + it.each(['missing', 'stale', 'failed'] as const)( + 'uses the agent production manifest when release evidence is %s', + async (evidence) => { + if (evidence === 'stale') + await release(5354, { + checkedAt: new Date(Date.now() - 60 * 60_000).toISOString() + }); + if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' }); + const id = enroll({ ...capable, runtimeImages: [runtime] }); + preparation.reconcileHostPreparation(id); + expect(operations(id)[0]).toMatchObject({ + kind: 'plutonium.prepare', + payload: { revision: 0 } + }); + expect( + hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0] + .targetPath + ).toBe('Latest production revision'); + const cached = enroll({ + ...capable, + runtimeImages: [runtime], + plutoniumCache: { revision: 5353, bytes: 200 } + }); + preparation.reconcileHostPreparation(cached); + expect(operations(cached)).toEqual([]); + } + ); it('leaves agents without the cache capability unchanged', async () => { await release(5355); - const id = enroll({ runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 200 } }); - expect(preparation.reconcileHostPreparation(id).items.some(item => item.kind === 'plutonium')).toBe(false); + const id = enroll({ + runtimeImages: [runtime], + plutoniumCache: { revision: 5354, bytes: 200 } + }); + expect( + preparation.reconcileHostPreparation(id).items.some((item) => item.kind === 'plutonium') + ).toBe(false); expect(operations(id)).toEqual([]); }); it('backs off failed cache work without blocking launch readiness or newly published game files', async () => { const version = await publish(); - const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] }; + const ready = { + ...capable, + runtimeImages: [runtime], + libraries: [{ id: version.id, state: 'ready' as const }] + }; const id = enroll(ready); verified(id, version); await release(5354); preparation.reconcileHostPreparation(id); const cache = operations(id)[0]; - hosts.reportHostOperation(id, cache.id, { state: 'failed', error: 'Plutonium CDN unavailable' }); + hosts.reportHostOperation(id, cache.id, { + state: 'failed', + error: 'Plutonium CDN unavailable' + }); const failedAt = Date.now(); heartbeat(id, ready); const status = preparation.reconcileHostPreparation(id); expect(status).toMatchObject({ state: 'ready' }); - expect(status.items.find(item => item.kind === 'plutonium')).toMatchObject({ - state: 'blocked', detail: expect.stringContaining('Automatic retry') + expect(status.items.find((item) => item.kind === 'plutonium')).toMatchObject({ + state: 'blocked', + detail: expect.stringContaining('Automatic retry') }); expect(hosts.requireReadyHost(id).readiness).toBe('ready'); expect(operations(id)).toHaveLength(1); @@ -762,7 +1112,10 @@ describe('optional host Plutonium cache', () => { expect(operations(id)).toHaveLength(1); const latest = await publish(); preparation.reconcileHostPreparation(id); - expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); + expect(operations(id)[0]).toMatchObject({ + kind: 'library.download', + payload: { libraryVersionId: latest.id } + }); verified(id, latest); hosts.reportHostOperation(id, operations(id)[0].id, { state: 'completed' }); heartbeat(id, ready); @@ -772,26 +1125,47 @@ describe('optional host Plutonium cache', () => { heartbeat(id, ready); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(3); - expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } }); + expect(operations(id)[0]).toMatchObject({ + kind: 'plutonium.prepare', + payload: { revision: 5355 } + }); }); it('waits for required library capacity before consuming space for the optional cache', async () => { const version = await publish(); - const id = enroll({ ...capable, runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 }); + const id = enroll({ + ...capable, + runtimeImages: [runtime], + storageAvailable: 1_000 + version.sizeBytes - 1 + }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(operations(id)).toEqual([]); }); - it.each([null, [], {}, { revision: 0, bytes: 1 }, { revision: 1.5, bytes: 1 }, { revision: 1, bytes: -1 }, { revision: 1, bytes: 0.5 }])( - 'rejects invalid cache inventory %j', cache => { - const id = enroll(); - expect(() => heartbeat(id, { plutoniumCache: cache as HostInventory['plutoniumCache'] })).toThrow('Invalid Plutonium cache inventory'); - } - ); + it.each([ + null, + [], + {}, + { revision: 0, bytes: 1 }, + { revision: 1.5, bytes: 1 }, + { revision: 1, bytes: -1 }, + { revision: 1, bytes: 0.5 } + ])('rejects invalid cache inventory %j', (cache) => { + const id = enroll(); + expect(() => + heartbeat(id, { plutoniumCache: cache as HostInventory['plutoniumCache'] }) + ).toThrow('Invalid Plutonium cache inventory'); + }); it('retains only revision and byte count from cache inventory and clears an omitted cache', () => { - const id = enroll({ plutoniumCache: { revision: 5354, bytes: 0, localPath: '/private/cache' } as HostInventory['plutoniumCache'] }); + const id = enroll({ + plutoniumCache: { + revision: 5354, + bytes: 0, + localPath: '/private/cache' + } as HostInventory['plutoniumCache'] + }); expect(hosts.getHost(id)?.inventory.plutoniumCache).toEqual({ revision: 5354, bytes: 0 }); heartbeat(id); expect(hosts.getHost(id)?.inventory.plutoniumCache).toBeUndefined(); diff --git a/tests/unit/host-request-cost.test.ts b/tests/unit/host-request-cost.test.ts index 81c0f26..6b6d907 100644 --- a/tests/unit/host-request-cost.test.ts +++ b/tests/unit/host-request-cost.test.ts @@ -5,13 +5,26 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; const epoch = Date.parse('2026-10-05T12:00:00Z'); -const inventory = { hostname: 'request-cache-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true, - storageAvailable: 100 * 1024 ** 3, runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }] }; +const inventory = { + hostname: 'request-cache-host', + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: ['plutainer:test'], + libraries: [{ id: 'test-library', state: 'ready' }] +}; beforeEach(async () => { - vi.resetModules(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); + vi.resetModules(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(epoch); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); }); +afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + await fixture.cleanup(); +}); async function enrolled() { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -27,40 +40,53 @@ describe('agent preparation admission', () => { it('skips fleet preparation lookups for empty, health, read and unlimited library polls', async () => { const api = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({...fleetSettings(),preparationConcurrency:1}); - const prepare = vi.spyOn(db(),'prepare'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); + saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 1 }); + const prepare = vi.spyOn(db(), 'prepare'); expect(api.pendingHostOperations(api.hostId)).toEqual([]); - api.queueHostOperation(api.hostId,'instance.inspect',{}); - api.pendingHostOperations(api.hostId,'health'); - api.queueHostOperation(api.hostId,'files.read',{path:'server.cfg'}); - api.pendingHostOperations(api.hostId,'read'); - saveFleetSettings({...fleetSettings(),preparationConcurrency:0}); - const download = api.queueHostOperation(api.hostId,'library.download',{}); - expect(api.pendingHostOperations(api.hostId,'write').map(op=>op.id)).toContain(download.id); - expect(prepare.mock.calls.some(([sql])=>sql.includes('SELECT DISTINCT host_id'))).toBe(false); + api.queueHostOperation(api.hostId, 'instance.inspect', {}); + api.pendingHostOperations(api.hostId, 'health'); + api.queueHostOperation(api.hostId, 'files.read', { path: 'server.cfg' }); + api.pendingHostOperations(api.hostId, 'read'); + saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 0 }); + const download = api.queueHostOperation(api.hostId, 'library.download', {}); + expect(api.pendingHostOperations(api.hostId, 'write').map((op) => op.id)).toContain( + download.id + ); + expect(prepare.mock.calls.some(([sql]) => sql.includes('SELECT DISTINCT host_id'))).toBe( + false + ); }); it('uses an indexed lookup and counts delivered preparation by host until all its jobs finish', async () => { const api = await enrolled(); - const other = api.createHost({name:'Other host'}); - api.enrollHost(api.createEnrollment(other.id).token,inventory); + const other = api.createHost({ name: 'Other host' }); + api.enrollHost(api.createEnrollment(other.id).token, inventory); const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({...fleetSettings(),preparationConcurrency:1}); - const download = api.queueHostOperation(api.hostId,'library.download',{}); - const imported = api.queueHostOperation(api.hostId,'library.import',{}); - const waiting = api.queueHostOperation(other.id,'library.download',{}); - const prepare = vi.spyOn(db(),'prepare'); - expect(api.pendingHostOperations(api.hostId,'write').map(op=>op.id)).toEqual([download.id,imported.id]); - const sql = prepare.mock.calls.find(([sql])=>sql.includes('SELECT DISTINCT host_id'))![0]; - const plan = db().prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as {detail:string}[]; - expect(plan.map(row=>row.detail).join('\n')).toContain('USING INDEX ec_host_operations_kind_state (kind=? AND state=?)'); - expect(api.pendingHostOperations(other.id,'write')).toEqual([]); - api.reportHostOperation(api.hostId,download.id,{state:'completed'}); - expect(api.pendingHostOperations(other.id,'write')).toEqual([]); - api.reportHostOperation(api.hostId,imported.id,{state:'failed'}); - expect(api.pendingHostOperations(other.id,'write').map(op=>op.id)).toEqual([waiting.id]); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); + saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 1 }); + const download = api.queueHostOperation(api.hostId, 'library.download', {}); + const imported = api.queueHostOperation(api.hostId, 'library.import', {}); + const waiting = api.queueHostOperation(other.id, 'library.download', {}); + const prepare = vi.spyOn(db(), 'prepare'); + expect(api.pendingHostOperations(api.hostId, 'write').map((op) => op.id)).toEqual([ + download.id, + imported.id + ]); + const sql = prepare.mock.calls.find(([sql]) => sql.includes('SELECT DISTINCT host_id'))![0]; + const plan = db().prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]; + expect(plan.map((row) => row.detail).join('\n')).toContain( + 'USING INDEX ec_host_operations_kind_state (kind=? AND state=?)' + ); + expect(api.pendingHostOperations(other.id, 'write')).toEqual([]); + api.reportHostOperation(api.hostId, download.id, { state: 'completed' }); + expect(api.pendingHostOperations(other.id, 'write')).toEqual([]); + api.reportHostOperation(api.hostId, imported.id, { state: 'failed' }); + expect(api.pendingHostOperations(other.id, 'write').map((op) => op.id)).toEqual([ + waiting.id + ]); }); }); @@ -72,16 +98,24 @@ describe('agent credential usage writes', () => { CREATE TRIGGER count_credential_usage AFTER UPDATE OF last_used_at ON ec_host_credentials BEGIN INSERT INTO credential_usage_writes VALUES (NEW.id, NEW.last_used_at); END;`); const writes = () => db().prepare('SELECT at FROM credential_usage_writes').all(); - const authenticate = () => expect(api.authenticateCredential(api.credential)?.id).toBe(api.hostId); - authenticate(); authenticate(); authenticate(); + const authenticate = () => + expect(api.authenticateCredential(api.credential)?.id).toBe(api.hostId); + authenticate(); + authenticate(); + authenticate(); expect(writes()).toEqual([{ at: new Date(epoch).toISOString() }]); vi.setSystemTime(epoch + 59_999); authenticate(); - closeDb(); authenticate(); + closeDb(); + authenticate(); expect(writes()).toHaveLength(1); vi.setSystemTime(epoch + 60_000); - authenticate(); authenticate(); - expect(writes()).toEqual([{ at: new Date(epoch).toISOString() }, { at: new Date(epoch + 60_000).toISOString() }]); + authenticate(); + authenticate(); + expect(writes()).toEqual([ + { at: new Date(epoch).toISOString() }, + { at: new Date(epoch + 60_000).toISOString() } + ]); expect(api.authenticateCredential('invalid')).toBeNull(); expect(writes()).toHaveLength(2); }); @@ -95,8 +129,13 @@ describe('agent credential usage writes', () => { const replacement = api.rotateCredential(api.hostId); expect(api.authenticateCredential(api.credential)).toBeNull(); expect(api.authenticateCredential(replacement)?.id).toBe(api.hostId); - expect(db().prepare('SELECT last_used_at FROM ec_host_credentials WHERE revoked_at IS NULL').get()) - .toEqual({ last_used_at: new Date(epoch + 1000).toISOString() }); + expect( + db() + .prepare( + 'SELECT last_used_at FROM ec_host_credentials WHERE revoked_at IS NULL' + ) + .get() + ).toEqual({ last_used_at: new Date(epoch + 1000).toISOString() }); api.revokeCredentials(api.hostId); expect(api.authenticateCredential(replacement)).toBeNull(); expect(api.getHost(api.hostId)?.credentialActive).toBe(false); @@ -111,11 +150,14 @@ describe('request-scoped host inventory parsing', () => { const parse = vi.spyOn(JSON, 'parse'); await api.withHostRequestContext(async () => { const authenticated = api.authenticateCredential(api.credential)!; - const operation = api.queueHostOperation(api.hostId, 'instance.inspect', { name: 'server' }); + const operation = api.queueHostOperation(api.hostId, 'instance.inspect', { + name: 'server' + }); api.pendingHostOperationRows(api.hostId, 'read', false); api.reportHostOperationStatus(api.hostId, operation.id, { state: 'accepted' }); await Promise.resolve(); - for (let i = 0; i < 10; i++) expect(api.getHost(api.hostId)?.inventory).toBe(authenticated.inventory); + for (let i = 0; i < 10; i++) + expect(api.getHost(api.hostId)?.inventory).toBe(authenticated.inventory); expect(api.requireReadyHost(api.hostId).inventory).toBe(authenticated.inventory); expect(api.listHosts()[0].inventory).toBe(authenticated.inventory); }); @@ -130,17 +172,25 @@ describe('request-scoped host inventory parsing', () => { await api.withHostRequestContext(() => { const original = api.getHost(api.hostId)!; const updated = api.updateHost(api.hostId, { name: 'Renamed host', mode: 'draining' }); - expect(updated).toMatchObject({ name: 'Renamed host', mode: 'draining', readiness: 'draining' }); + expect(updated).toMatchObject({ + name: 'Renamed host', + mode: 'draining', + readiness: 'draining' + }); expect(updated.inventory).toBe(original.inventory); const heartbeat = api.heartbeat(api.hostId, { ...inventory, storageAvailable: 123 }); expect(heartbeat.inventory).not.toBe(original.inventory); expect(api.getHost(api.hostId)?.inventory).toBe(heartbeat.inventory); expect(api.getHost(api.hostId)?.inventory.storageAvailable).toBe(123); - expect(() => db().transaction(() => { - db().prepare('UPDATE ec_hosts SET inventory=? WHERE id=?').run(JSON.stringify({ ...inventory, storageAvailable: 456 }), api.hostId); - expect(api.getHost(api.hostId)?.inventory.storageAvailable).toBe(456); - throw new Error('rollback'); - })()).toThrow('rollback'); + expect(() => + db().transaction(() => { + db() + .prepare('UPDATE ec_hosts SET inventory=? WHERE id=?') + .run(JSON.stringify({ ...inventory, storageAvailable: 456 }), api.hostId); + expect(api.getHost(api.hostId)?.inventory.storageAvailable).toBe(456); + throw new Error('rollback'); + })() + ).toThrow('rollback'); expect(api.getHost(api.hostId)?.inventory.storageAvailable).toBe(123); }); }); @@ -148,7 +198,9 @@ describe('request-scoped host inventory parsing', () => { it('isolates overlapping requests and does not cache reads outside a request', async () => { const api = await enrolled(); let release!: () => void; - const gate = new Promise(resolve => { release = resolve; }); + const gate = new Promise((resolve) => { + release = resolve; + }); let first!: HostInventory; const request = api.withHostRequestContext(async () => { first = api.getHost(api.hostId)!.inventory; @@ -163,25 +215,35 @@ describe('request-scoped host inventory parsing', () => { expect(api.getHost(api.hostId)!.inventory).toBe(second); }); expect(api.getHost(api.hostId)!.inventory).not.toBe(api.getHost(api.hostId)!.inventory); - } finally { release(); await request; } + } finally { + release(); + await request; + } }); - it.each([false, true])('releases cached inventory from inherited work after request completion (failed: %s)', async failed => { - const api = await enrolled(); - let release!: () => void; - const gate = new Promise(resolve => { release = resolve; }); - let detached!: Promise; - let original!: HostInventory; - const request = api.withHostRequestContext(() => { - original = api.getHost(api.hostId)!.inventory; - detached = gate.then(() => { - const later = api.getHost(api.hostId)!.inventory; - expect(later).not.toBe(original); - expect(api.getHost(api.hostId)!.inventory).not.toBe(later); + it.each([false, true])( + 'releases cached inventory from inherited work after request completion (failed: %s)', + async (failed) => { + const api = await enrolled(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); - if (failed) throw new Error('request failed'); - }); - if (failed) await expect(request).rejects.toThrow('request failed'); else await request; - release(); await detached; - }); + let detached!: Promise; + let original!: HostInventory; + const request = api.withHostRequestContext(() => { + original = api.getHost(api.hostId)!.inventory; + detached = gate.then(() => { + const later = api.getHost(api.hostId)!.inventory; + expect(later).not.toBe(original); + expect(api.getHost(api.hostId)!.inventory).not.toBe(later); + }); + if (failed) throw new Error('request failed'); + }); + if (failed) await expect(request).rejects.toThrow('request failed'); + else await request; + release(); + await detached; + } + ); }); diff --git a/tests/unit/host-status-batch.test.ts b/tests/unit/host-status-batch.test.ts index a50e3f5..e1e2b68 100644 --- a/tests/unit/host-status-batch.test.ts +++ b/tests/unit/host-status-batch.test.ts @@ -5,12 +5,17 @@ import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; let fixture: Fixture; let at: number; beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); + vi.resetModules(); + vi.resetAllMocks(); fixture = await newFixture([]); at = Date.now(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(at); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(at); +}); +afterEach(async () => { + vi.useRealTimers(); + await fixture.cleanup(); }); -afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function setup(count = 5, hostCount = 1) { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -20,29 +25,86 @@ async function setup(count = 5, hostCount = 1) { vi.mocked(nativeHealth).mockResolvedValue({ ok: true, state: 'running' }); const hostIds: string[] = []; for (let h = 0; h < hostCount; h++) { - const host = hosts.createHost({ name: `Host ${h}` }); hostIds.push(host.id); - const inventory = { protocolVersion: 1, capabilities: ['instance.heartbeat-state', 'rcon.status-batch', 'operations.health-lane'], - instanceStates: Array.from({ length: count }, (_, i) => ({ instanceId: `instance-${h}-${i}`, generation: 1, - healthy: true, state: 'running', observedAt: new Date(at).toISOString() })) }; + const host = hosts.createHost({ name: `Host ${h}` }); + hostIds.push(host.id); + const inventory = { + protocolVersion: 1, + capabilities: [ + 'instance.heartbeat-state', + 'rcon.status-batch', + 'operations.health-lane' + ], + instanceStates: Array.from({ length: count }, (_, i) => ({ + instanceId: `instance-${h}-${i}`, + generation: 1, + healthy: true, + state: 'running', + observedAt: new Date(at).toISOString() + })) + }; hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); for (let i = 0; i < count; i++) { const name = `server-${h}-${i}`; - db().prepare('INSERT INTO servers(name,created_at) VALUES(?,?)').run(name, new Date(at - 1000).toISOString()); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,'library','image',?,?)`).run(`instance-${h}-${i}`, name, host.id, 28960 + i, new Date(at - 1000).toISOString()); + db() + .prepare('INSERT INTO servers(name,created_at) VALUES(?,?)') + .run(name, new Date(at - 1000).toISOString()); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,'library','image',?,?)` + ) + .run( + `instance-${h}-${i}`, + name, + host.id, + 28960 + i, + new Date(at - 1000).toISOString() + ); } } const health = await import('../../src/lib/server/fleet/health-observations.js'); const worker = await import('../../src/lib/server/pipeline/worker.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const operations = () => hostIds.flatMap(id => hosts.listHostOperations(id)); - const complete = () => operations().filter(op => !['completed', 'failed'].includes(op.state)).forEach(op => { - const targets = op.payload.instances as { instanceId: string; generation: number }[]; - hosts.reportHostOperation(op.hostId, op.id, { state: 'completed', result: op.kind === 'rcon.status-batch' - ? { ok: true, results: targets.map(target => ({ ...target, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } })) } - : { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } }); - }); - return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(nativeHealth), hostIds, operations, complete }; + const operations = () => hostIds.flatMap((id) => hosts.listHostOperations(id)); + const complete = () => + operations() + .filter((op) => !['completed', 'failed'].includes(op.state)) + .forEach((op) => { + const targets = op.payload.instances as { + instanceId: string; + generation: number; + }[]; + hosts.reportHostOperation(op.hostId, op.id, { + state: 'completed', + result: + op.kind === 'rcon.status-batch' + ? { + ok: true, + results: targets.map((target) => ({ + ...target, + reply: { + ok: true, + text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' + } + })) + } + : { + ok: true, + text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' + } + }); + }); + return { + ...hosts, + ...health, + ...worker, + getServer, + db: db(), + inspect: vi.mocked(nativeHealth), + hostIds, + operations, + complete + }; } describe('host status observations', () => { @@ -53,28 +115,51 @@ describe('host status observations', () => { const pass = api.fanoutStatus(); try { await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); - expect(api.operations()[0]).toMatchObject({ kind: 'rcon.status-batch', - payload: { instances: [{ instanceId: 'instance-0-0', generation: 1 }] } }); - } finally { api.complete(); await pass; } + expect(api.operations()[0]).toMatchObject({ + kind: 'rcon.status-batch', + payload: { instances: [{ instanceId: 'instance-0-0', generation: 1 }] } + }); + } finally { + api.complete(); + await pass; + } expect(api.getServer('server-0-1')?.last_seen_at).toBeNull(); }); it('bounds monitoring reads below five per server across thirty instances', async () => { const api = await setup(10, 3); const hosts = await import('../../src/lib/server/hosts/index.js'); - const wait = vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async id => { + const wait = vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) => { const operation = hosts.getHostOperation(id)!; - return { ...operation, state: 'completed', result: { results: (operation.payload.instances as { instanceId: string; generation: number }[]) - .map(target => ({ ...target, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } })) } }; + return { + ...operation, + state: 'completed', + result: { + results: ( + operation.payload.instances as { instanceId: string; generation: number }[] + ).map((target) => ({ + ...target, + reply: { + ok: true, + text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' + } + })) + } + }; }); const prepare = vi.spyOn(api.db, 'prepare'); try { await api.fanoutStatus(); const reads = prepare.mock.calls.filter(([sql]) => /^\s*(SELECT|WITH)\b/i.test(sql)); expect(reads.length).toBeLessThan(30 * 5); - expect(reads.filter(([sql]) => sql.includes('instance_json')).length).toBeLessThanOrEqual(30 * 3 + 5); + expect( + reads.filter(([sql]) => sql.includes('instance_json')).length + ).toBeLessThanOrEqual(30 * 3 + 5); expect(api.getServer('server-2-9')?.last_players).toBe(0); - } finally { prepare.mockRestore(); wait.mockRestore(); } + } finally { + prepare.mockRestore(); + wait.mockRestore(); + } }); it('uses one batch per host for a fifteen-server pass and shares overlapping scans', async () => { @@ -88,12 +173,21 @@ describe('host status observations', () => { expect(operation.kind).toBe('rcon.status-batch'); expect(wait).toHaveBeenCalledWith(operation.id, 25_000); expect(operation.payload.instances).toHaveLength(5); - expect(api.pendingHostOperations(operation.hostId, 'health').map(op => op.id)).toEqual([operation.id]); + expect( + api.pendingHostOperations(operation.hostId, 'health').map((op) => op.id) + ).toEqual([operation.id]); expect(api.pendingHostOperations(operation.hostId, 'write')).toEqual([]); } - } finally { api.complete(); await Promise.all(passes); } + } finally { + api.complete(); + await Promise.all(passes); + } expect(api.inspect).not.toHaveBeenCalled(); - expect(api.getServer('server-2-4')).toMatchObject({ last_players: 0, last_map: 'mp_kino', container_checked_at: new Date(at).toISOString() }); + expect(api.getServer('server-2-4')).toMatchObject({ + last_players: 0, + last_map: 'mp_kino', + container_checked_at: new Date(at).toISOString() + }); expect(api.monitoringStatus('server-2-4')).toMatchObject({ container: { observedAt: new Date(at).toISOString(), result: { ok: true } }, game: { observedAt: api.getServer('server-2-4')?.last_seen_at, result: { ok: true } } @@ -103,80 +197,151 @@ describe('host status observations', () => { expect(api.operations()).toHaveLength(3); }); - it.each(['stale', 'future', 'generation', 'lifecycle', 'missing', 'legacy'])('falls back to inspection for %s heartbeat evidence', async mode => { - const api = await setup(1); - const host = api.getHost(api.hostIds[0])!; - const inventory = structuredClone(host.inventory); - if (mode === 'stale') inventory.instanceStates![0].observedAt = new Date(at - 30_000).toISOString(); - if (mode === 'future') inventory.instanceStates![0].observedAt = new Date(at + 1_000).toISOString(); - if (mode === 'generation') inventory.instanceStates![0].generation = 2; - if (mode === 'missing') inventory.instanceStates = []; - if (mode === 'legacy') inventory.capabilities = []; - if (mode === 'lifecycle') api.db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,finished_at,triggered_by) - VALUES('stop','server-0-0','stop','done','done','started',?,?,'test')`).run(new Date(at + 1).toISOString(), new Date(at + 2).toISOString()); - api.heartbeat(host.id, inventory); - await api.inspectForMonitoring('server-0-0'); - expect(api.inspect).toHaveBeenCalledOnce(); - }); + it.each(['stale', 'future', 'generation', 'lifecycle', 'missing', 'legacy'])( + 'falls back to inspection for %s heartbeat evidence', + async (mode) => { + const api = await setup(1); + const host = api.getHost(api.hostIds[0])!; + const inventory = structuredClone(host.inventory); + if (mode === 'stale') + inventory.instanceStates![0].observedAt = new Date(at - 30_000).toISOString(); + if (mode === 'future') + inventory.instanceStates![0].observedAt = new Date(at + 1_000).toISOString(); + if (mode === 'generation') inventory.instanceStates![0].generation = 2; + if (mode === 'missing') inventory.instanceStates = []; + if (mode === 'legacy') inventory.capabilities = []; + if (mode === 'lifecycle') + api.db + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,finished_at,triggered_by) + VALUES('stop','server-0-0','stop','done','done','started',?,?,'test')` + ) + .run(new Date(at + 1).toISOString(), new Date(at + 2).toISOString()); + api.heartbeat(host.id, inventory); + await api.inspectForMonitoring('server-0-0'); + expect(api.inspect).toHaveBeenCalledOnce(); + } + ); it('retains heartbeat sample time, reports stopped containers and keeps forced probes fresh', async () => { const api = await setup(1); vi.setSystemTime(at + 10_000); - expect(await api.inspectForMonitoring('server-0-0')).toMatchObject({ observedAt: new Date(at).toISOString(), result: { ok: true } }); + expect(await api.inspectForMonitoring('server-0-0')).toMatchObject({ + observedAt: new Date(at).toISOString(), + result: { ok: true } + }); await api.inspectForMonitoring('server-0-0', true); expect(api.inspect).toHaveBeenCalledOnce(); const host = api.getHost(api.hostIds[0])!; - api.heartbeat(host.id, { ...host.inventory, instanceStates: [{ ...host.inventory.instanceStates![0], healthy: false, state: 'exited' }] }); + api.heartbeat(host.id, { + ...host.inventory, + instanceStates: [ + { ...host.inventory.instanceStates![0], healthy: false, state: 'exited' } + ] + }); await api.fanoutStatus(); expect(api.getServer('server-0-0')?.container_state).toBe('exited'); - expect(api.monitoringStatus('server-0-0')).toMatchObject({ container: { - observedAt: new Date(at).toISOString(), result: { ok: false, state: 'exited' } - } }); + expect(api.monitoringStatus('server-0-0')).toMatchObject({ + container: { + observedAt: new Date(at).toISOString(), + result: { ok: false, state: 'exited' } + } + }); expect(api.operations()).toEqual([]); }); - it.each(['generation', 'manual', 'lifecycle'])('discards batch replies overtaken by a %s change, including queued servers', async change => { - const api = await setup(); - const pass = api.fanoutStatus(); - try { - await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); - vi.setSystemTime(at + 1000); - if (change === 'generation') api.db.prepare('UPDATE native_instances SET generation=2').run(); - if (change === 'manual') api.db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual'").run(new Date().toISOString()); - if (change === 'lifecycle') api.db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) - SELECT name,name,'stop','done','done','started',?,'test' FROM servers`).run(new Date().toISOString()); - } finally { api.complete(); await pass; } - for (let i = 0; i < 5; i++) expect(api.getServer(`server-0-${i}`)?.last_seen_at).toBeNull(); - expect(api.operations()).toHaveLength(1); - }); + it.each(['generation', 'manual', 'lifecycle'])( + 'discards batch replies overtaken by a %s change, including queued servers', + async (change) => { + const api = await setup(); + const pass = api.fanoutStatus(); + try { + await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); + vi.setSystemTime(at + 1000); + if (change === 'generation') + api.db.prepare('UPDATE native_instances SET generation=2').run(); + if (change === 'manual') + api.db + .prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual'") + .run(new Date().toISOString()); + if (change === 'lifecycle') + api.db + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) + SELECT name,name,'stop','done','done','started',?,'test' FROM servers` + ) + .run(new Date().toISOString()); + } finally { + api.complete(); + await pass; + } + for (let i = 0; i < 5; i++) + expect(api.getServer(`server-0-${i}`)?.last_seen_at).toBeNull(); + expect(api.operations()).toHaveLength(1); + } + ); it('preserves previous game evidence for deadline skips while recording actual timeouts', async () => { const api = await setup(3); const first = api.fanoutStatus(); await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); - api.complete(); await first; - api.db.prepare("UPDATE servers SET rcon_failures=1,last_error='timeout:first failure'").run(); + api.complete(); + await first; + api.db + .prepare("UPDATE servers SET rcon_failures=1,last_error='timeout:first failure'") + .run(); const previous = api.getServer('server-0-0')!; const previousGame = api.monitoringStatus('server-0-0')?.game; vi.setSystemTime(at + 31_000); const pass = api.fanoutStatus(); try { await vi.waitFor(() => expect(api.operations()).toHaveLength(2)); - const operation = api.operations().find(op => op.state !== 'completed')!; - api.reportHostOperation(operation.hostId, operation.id, { state: 'completed', result: { ok: false, results: [ - { instanceId: 'instance-0-0', generation: 1, reply: { ok: false, reason: 'deadline', text: '' } }, - { instanceId: 'instance-0-1', generation: 1, reply: { ok: false, reason: 'timeout', text: '' } }, - { instanceId: 'instance-0-2', generation: 1, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } } - ] } }); - } finally { api.complete(); await pass; } - expect(api.getServer('server-0-0')).toMatchObject({ last_checked_at: previous.last_checked_at, - last_seen_at: previous.last_seen_at, last_error: previous.last_error, last_players: previous.last_players, - last_reply_snippet: previous.last_reply_snippet, rcon_failures: 1 }); + const operation = api.operations().find((op) => op.state !== 'completed')!; + api.reportHostOperation(operation.hostId, operation.id, { + state: 'completed', + result: { + ok: false, + results: [ + { + instanceId: 'instance-0-0', + generation: 1, + reply: { ok: false, reason: 'deadline', text: '' } + }, + { + instanceId: 'instance-0-1', + generation: 1, + reply: { ok: false, reason: 'timeout', text: '' } + }, + { + instanceId: 'instance-0-2', + generation: 1, + reply: { + ok: true, + text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' + } + } + ] + } + }); + } finally { + api.complete(); + await pass; + } + expect(api.getServer('server-0-0')).toMatchObject({ + last_checked_at: previous.last_checked_at, + last_seen_at: previous.last_seen_at, + last_error: previous.last_error, + last_players: previous.last_players, + last_reply_snippet: previous.last_reply_snippet, + rcon_failures: 1 + }); expect(api.monitoringStatus('server-0-0')?.game).toEqual(previousGame); expect(api.getServer('server-0-1')?.last_error).toContain('timeout:'); expect(api.getServer('server-0-1')?.rcon_failures).toBe(2); expect(api.getServer('server-0-2')?.rcon_failures).toBe(0); - expect(Date.parse(api.getServer('server-0-2')!.last_seen_at!)).toBeGreaterThanOrEqual(at + 31_000); + expect(Date.parse(api.getServer('server-0-2')!.last_seen_at!)).toBeGreaterThanOrEqual( + at + 31_000 + ); }); it('records each failed or omitted result without retrying the batch per server', async () => { @@ -185,26 +350,54 @@ describe('host status observations', () => { try { await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); const operation = api.operations()[0]; - api.reportHostOperation(operation.hostId, operation.id, { state: 'completed', result: { ok: false, results: [ - { instanceId: 'instance-0-0', generation: 1, reply: { ok: false, reason: 'authentication', text: '' } } - ] } }); - } finally { api.complete(); await pass; } + api.reportHostOperation(operation.hostId, operation.id, { + state: 'completed', + result: { + ok: false, + results: [ + { + instanceId: 'instance-0-0', + generation: 1, + reply: { ok: false, reason: 'authentication', text: '' } + } + ] + } + }); + } finally { + api.complete(); + await pass; + } expect(api.getServer('server-0-0')?.last_error).toBe('bad-reply:wrong RCON password'); expect(api.getServer('server-0-4')?.last_error).toContain('timeout:'); - expect(api.monitoringStatus('server-0-4')?.game?.result).toMatchObject({ ok: false, reason: 'timeout' }); + expect(api.monitoringStatus('server-0-4')?.game?.result).toMatchObject({ + ok: false, + reason: 'timeout' + }); expect(api.operations()).toHaveLength(1); }); - it.each([true, false])('uses batch capability=%s while missing heartbeat state still gets inspected', async capable => { - const api = await setup(1); - const host = api.getHost(api.hostIds[0])!; - api.heartbeat(host.id, { ...host.inventory, instanceStates: [], capabilities: capable ? ['rcon.status-batch'] : [] }); - const pass = api.fanoutStatus(); - try { - await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); - expect(api.operations()[0].kind).toBe(capable ? 'rcon.status-batch' : 'rcon.execute'); - } finally { api.complete(); await pass; } - expect(api.inspect).toHaveBeenCalledOnce(); - expect(api.getServer('server-0-0')?.last_players).toBe(0); - }); + it.each([true, false])( + 'uses batch capability=%s while missing heartbeat state still gets inspected', + async (capable) => { + const api = await setup(1); + const host = api.getHost(api.hostIds[0])!; + api.heartbeat(host.id, { + ...host.inventory, + instanceStates: [], + capabilities: capable ? ['rcon.status-batch'] : [] + }); + const pass = api.fanoutStatus(); + try { + await vi.waitFor(() => expect(api.operations()).toHaveLength(1)); + expect(api.operations()[0].kind).toBe( + capable ? 'rcon.status-batch' : 'rcon.execute' + ); + } finally { + api.complete(); + await pass; + } + expect(api.inspect).toHaveBeenCalledOnce(); + expect(api.getServer('server-0-0')?.last_players).toBe(0); + } + ); }); diff --git a/tests/unit/instance-inventory.test.ts b/tests/unit/instance-inventory.test.ts index e0cd8f8..148b6df 100644 --- a/tests/unit/instance-inventory.test.ts +++ b/tests/unit/instance-inventory.test.ts @@ -3,16 +3,31 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; const name = 't5_zombie_theater_1'; beforeEach(async () => { - vi.resetModules(); fixture = await newFixture([name]); + vi.resetModules(); + fixture = await newFixture([name]); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); it('links only inventory whose exact content matches its library version', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const artifactId = Number(db().prepare("INSERT INTO scripts(name, target_rel_path, kind) VALUES ('utils', 'utils.dll', 'plugin')").run().lastInsertRowid); - const versionId = Number(db().prepare("INSERT INTO script_versions(script_id, version, release_version, sha256, size) VALUES (?, 2, 'v1.1.0', 'known', 10)").run(artifactId).lastInsertRowid); + const artifactId = Number( + db() + .prepare( + "INSERT INTO scripts(name, target_rel_path, kind) VALUES ('utils', 'utils.dll', 'plugin')" + ) + .run().lastInsertRowid + ); + const versionId = Number( + db() + .prepare( + "INSERT INTO script_versions(script_id, version, release_version, sha256, size) VALUES (?, 2, 'v1.1.0', 'known', 10)" + ) + .run(artifactId).lastInsertRowid + ); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); recordScript(name, 'utils.dll', versionId, 'known', 10, 'native-create'); recordScript(name, 'changed.dll', versionId, 'different', 12, 'native-create'); @@ -21,7 +36,13 @@ it('links only inventory whose exact content matches its library version', async const response = await GET({ params: { name } }); expect(response.status).toBe(200); const { scripts } = await response.json(); - expect(scripts.find((item: { relPath: string }) => item.relPath === 'utils.dll')).toMatchObject({ artifactId, version: 2, releaseVersion: 'v1.1.0', kind: 'plugin' }); - expect(scripts.find((item: { relPath: string }) => item.relPath === 'changed.dll')).toMatchObject({ artifactId: null, version: null, releaseVersion: null }); - expect(scripts.find((item: { relPath: string }) => item.relPath === 'unknown.gsc')).toMatchObject({ artifactId: null, version: null, kind: 'script' }); + expect(scripts.find((item: { relPath: string }) => item.relPath === 'utils.dll')).toMatchObject( + { artifactId, version: 2, releaseVersion: 'v1.1.0', kind: 'plugin' } + ); + expect( + scripts.find((item: { relPath: string }) => item.relPath === 'changed.dll') + ).toMatchObject({ artifactId: null, version: null, releaseVersion: null }); + expect( + scripts.find((item: { relPath: string }) => item.relPath === 'unknown.gsc') + ).toMatchObject({ artifactId: null, version: null, kind: 'script' }); }); diff --git a/tests/unit/launch-recipe-summary-migration.test.ts b/tests/unit/launch-recipe-summary-migration.test.ts index e9d44d9..97d1d27 100644 --- a/tests/unit/launch-recipe-summary-migration.test.ts +++ b/tests/unit/launch-recipe-summary-migration.test.ts @@ -5,16 +5,39 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import type { LaunchRecipe } from '../../src/lib/server/fleet/launch-recipes.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); describe('persisted launch recipe summaries', () => { it('lazily upgrades legacy snapshots once without changing fingerprints or retained bytes', async () => { const recipe: LaunchRecipe = { - format: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino der Toten', expectedMap: 'zombie_theater', - capturedAt: '2025-01-01T00:00:00.000Z', image: 'plutainer:test', configFile: 'dedicated_sp.cfg', configSource: 'Defaults', - files: [{ name: 'Old plugin', kind: 'plugin', path: 'legacy.dll', versionId: 1, version: 1, - sha256: 'a'.repeat(64), size: 2 * 1024 ** 2, content: Buffer.alloc(2 * 1024 ** 2, 42).toString('base64') }] + format: 1, + game: 't5', + mapSlug: 'theater', + mapName: 'Kino der Toten', + expectedMap: 'zombie_theater', + capturedAt: '2025-01-01T00:00:00.000Z', + image: 'plutainer:test', + configFile: 'dedicated_sp.cfg', + configSource: 'Defaults', + files: [ + { + name: 'Old plugin', + kind: 'plugin', + path: 'legacy.dll', + versionId: 1, + version: 1, + sha256: 'a'.repeat(64), + size: 2 * 1024 ** 2, + content: Buffer.alloc(2 * 1024 ** 2, 42).toString('base64') + } + ] }; const legacy = new Database(fixture.dbPath); const encoded = JSON.stringify(recipe, null, 2); @@ -22,13 +45,21 @@ describe('persisted launch recipe summaries', () => { legacy.exec(schema.replace(' recipe_summary TEXT,\n', '')); legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('legacy','retired-server','create','done','Complete','2025-01-01','fixture');`); - legacy.prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)') + legacy + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)' + ) .run('legacy', encoded, 'Old collection'); - } finally { legacy.close(); } + } finally { + legacy.close(); + } const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { recipeSummaryForDeployment, recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { recipeSummaryForDeployment, recipeForDeployment, summarizeRecipe } = + await import('../../src/lib/server/fleet/launch-recipes.js'); const expected = { recipe: summarizeRecipe(recipe), blueprintName: 'Old collection' }; - expect(db().prepare('SELECT recipe_summary FROM server_launch_recipes').get()).toEqual({ recipe_summary: null }); + expect(db().prepare('SELECT recipe_summary FROM server_launch_recipes').get()).toEqual({ + recipe_summary: null + }); expect(recipeSummaryForDeployment('legacy')).toEqual(expected); closeDb(); const parse = JSON.parse; @@ -39,9 +70,13 @@ describe('persisted launch recipe summaries', () => { try { expect(recipeSummaryForDeployment('legacy')).toEqual(expected); expect(recipeSummaryForDeployment('missing')).toBeNull(); - } finally { parsing.mockRestore(); } + } finally { + parsing.mockRestore(); + } expect(recipeForDeployment('legacy')).toEqual({ recipe, blueprintName: 'Old collection' }); - const stored = db().prepare('SELECT recipe FROM server_launch_recipes').get() as { recipe: string }; + const stored = db().prepare('SELECT recipe FROM server_launch_recipes').get() as { + recipe: string; + }; expect(stored.recipe.length).toBeLessThan(1024); expect(JSON.parse(stored.recipe).files[0]).not.toHaveProperty('content'); }); diff --git a/tests/unit/launch-recipes.test.ts b/tests/unit/launch-recipes.test.ts index 14442e1..4582b83 100644 --- a/tests/unit/launch-recipes.test.ts +++ b/tests/unit/launch-recipes.test.ts @@ -1,49 +1,83 @@ import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })), - rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: null, roster: [] } })) + rconStatusResult: vi.fn(async () => ({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: null, roster: [] } + })) +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; const selection = { game: 't5' as const, mapSlug: 'theater' }; const input = { ...selection, gameMode: 'zombie' as const, plutoKey: 'private-pluto-key' }; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); + vi.resetModules(); + vi.clearAllMocks(); fixture = await newFixture([]); await prepareNativeHost(); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); + vi.stubGlobal( + 'fetch', + vi.fn(async () => + Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }) + ) + ); vi.stubEnv('RCON_PASSWORD', 'private-rcon-password'); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); async function seedArtifacts() { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const baseline = createScript({ name: 'Classic settings', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }); - const script = createScript({ name: 'Classic perks', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'perks.gsc', content: Buffer.from('main(){ old_perks(); }') }); - for (const item of [baseline, script]) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(item.version.id, item.script.id); + const baseline = createScript({ + name: 'Classic settings', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('set sv_voice "1"\n') + }); + const script = createScript({ + name: 'Classic perks', + kind: 'script', + scopeType: 'game', + scopeValue: 't5', + targetRelPath: 'perks.gsc', + content: Buffer.from('main(){ old_perks(); }') + }); + for (const item of [baseline, script]) + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(item.version.id, item.script.id); return { baseline, script }; } describe('launch recipes and previews', () => { - it('previews without reserving capacity, saving keys, exposing bytes, or submitting host commands', async () => { await seedArtifacts(); const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); const { db } = await import('../../src/lib/server/db/index.js'); const first = previewCreation(selection); const second = previewCreation(selection); - expect(first).toMatchObject({ name: 't5_zombie_theater_1', port: 28960, hasSavedKey: false, ready: true }); + expect(first).toMatchObject({ + name: 't5_zombie_theater_1', + port: 28960, + hasSavedKey: false, + ready: true + }); expect(second.name).toBe(first.name); expect(second.port).toBe(first.port); expect(first.recipe.files).toHaveLength(3); @@ -54,42 +88,74 @@ describe('launch recipes and previews', () => { }); it('returns the same operation for repeated submission, even after expiry and restart', async () => { - const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); + const { previewCreation, queueCreateServer } = + await import('../../src/lib/server/fleet/create.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const preview = previewCreation(selection); const queued = queueCreateServer({ ...input, previewId: preview.id }); expect(queued.ok).toBe(true); - db().prepare("UPDATE server_launch_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(preview.id); + db() + .prepare( + "UPDATE server_launch_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?" + ) + .run(preview.id); closeDb(); - expect(queueCreateServer({ ...input, previewId: preview.id, plutoKey: 'replacement-key' })).toEqual(queued); + expect( + queueCreateServer({ ...input, previewId: preview.id, plutoKey: 'replacement-key' }) + ).toEqual(queued); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1); expect(getServerKey(preview.name!)).toBe('private-pluto-key'); }); it('carries preview identity through the public creation API and rejects malformed requests', async () => { - const { POST: preview } = await import('../../src/routes/api/deployments/preview/+server.js'); + const { POST: preview } = + await import('../../src/routes/api/deployments/preview/+server.js'); const { POST: launch } = await import('../../src/routes/api/servers/+server.js'); - const request = (body: unknown) => ({ request: new Request('http://localhost/api', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) }); - for (const body of [null, [], { game: 't7', mapSlug: 'theater' }]) expect((await preview(request(body))).status).toBe(400); + const request = (body: unknown) => ({ + request: new Request('http://localhost/api', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + }); + for (const body of [null, [], { game: 't7', mapSlug: 'theater' }]) + expect((await preview(request(body))).status).toBe(400); const response = await preview(request(selection)); const plan = await response.json(); expect(response.status).toBe(200); expect((await launch(request({ ...input, previewId: '' }))).status).toBe(400); - const first = await launch(request({ ...input, previewId: plan.id, expectedName: plan.name })); + const first = await launch( + request({ ...input, previewId: plan.id, expectedName: plan.name }) + ); expect(first.status).toBe(202); - const retry = await launch(request({ ...input, previewId: plan.id, expectedName: plan.name })); + const retry = await launch( + request({ ...input, previewId: plan.id, expectedName: plan.name }) + ); expect(await retry.json()).toEqual(await first.json()); }); it('rejects expired, missing, and mismatched previews', async () => { - const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); + const { previewCreation, queueCreateServer } = + await import('../../src/lib/server/fleet/create.js'); const { db } = await import('../../src/lib/server/db/index.js'); const preview = previewCreation(selection); - expect(queueCreateServer({ ...input, mapSlug: 'moon', previewId: preview.id })).toMatchObject({ ok: false, kind: 'bad-request' }); - db().prepare("UPDATE server_launch_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(preview.id); - expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'bad-request' }); - expect(queueCreateServer({ ...input, previewId: 'missing' })).toMatchObject({ ok: false, kind: 'bad-request' }); + expect( + queueCreateServer({ ...input, mapSlug: 'moon', previewId: preview.id }) + ).toMatchObject({ ok: false, kind: 'bad-request' }); + db() + .prepare( + "UPDATE server_launch_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?" + ) + .run(preview.id); + expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ + ok: false, + kind: 'bad-request' + }); + expect(queueCreateServer({ ...input, previewId: 'missing' })).toMatchObject({ + ok: false, + kind: 'bad-request' + }); expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]); }); @@ -98,9 +164,13 @@ describe('launch recipes and previews', () => { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const previewId = previewCreation(selection).id; expect(() => saveBlueprint({ name: '', previewId })).toThrow(/name/); - expect(() => saveBlueprint({ name: 'Valid', previewId, deploymentId: 'also' })).toThrow(/exactly one/); + expect(() => saveBlueprint({ name: 'Valid', previewId, deploymentId: 'also' })).toThrow( + /exactly one/ + ); expect(() => saveBlueprint({ name: 'Valid', previewId: 'missing' })).toThrow(/unavailable/); saveBlueprint({ name: 'Classic Night', previewId }); - expect(() => saveBlueprint({ name: ' classic night ', previewId })).toThrow(/already exists/); + expect(() => saveBlueprint({ name: ' classic night ', previewId })).toThrow( + /already exists/ + ); }); }); diff --git a/tests/unit/leaderboard-efficiency-query.test.ts b/tests/unit/leaderboard-efficiency-query.test.ts index 94e908b..1f5d1b5 100644 --- a/tests/unit/leaderboard-efficiency-query.test.ts +++ b/tests/unit/leaderboard-efficiency-query.test.ts @@ -4,31 +4,59 @@ import type { LeaderboardMetric } from '../../src/lib/leaderboard.js'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); afterEach(async () => fixture.cleanup()); interface Sample { - kills?: number; headshots?: number; downs?: number; rounds?: number; completed?: number; - game?: string; map?: string; status?: 'active' | 'completed' | 'interrupted'; + kills?: number; + headshots?: number; + downs?: number; + rounds?: number; + completed?: number; + game?: string; + map?: string; + status?: 'active' | 'completed' | 'interrupted'; } function addSamples(handle: Database.Database, guid: string, samples: Sample[]) { for (const sample of samples) { - const { kills = 20, headshots = 10, downs = 1, rounds = 2, completed = 1, - game = 't5', map = 'theater', status = 'completed' } = sample; - const inserted = handle.prepare(`INSERT INTO leaderboard_matches + const { + kills = 20, + headshots = 10, + downs = 1, + rounds = 2, + completed = 1, + game = 't5', + map = 'theater', + status = 'completed' + } = sample; + const inserted = handle + .prepare( + `INSERT INTO leaderboard_matches (server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, highest_round, last_event, last_fingerprint) - VALUES ('removed-server', ?, ?, ?, ?, '2026-09-01', '2026-09-02', '2026-09-02', ?, 20, 'game_end', 'test')`) + VALUES ('removed-server', ?, ?, ?, ?, '2026-09-01', '2026-09-02', '2026-09-02', ?, 20, 'game_end', 'test')` + ) .run(game, map, `${game}:${map}`, map, status); - handle.prepare(`INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES (?, ?, ?, '2026-09-01', '2026-09-02')`).run(game, guid, guid); - handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, headshots, + handle + .prepare( + `INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES (?, ?, ?, '2026-09-01', '2026-09-02')` + ) + .run(game, guid, guid); + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, headshots, downs, rounds_survived, completed, has_stats, first_seen_at, last_seen_at) - VALUES (?, ?, 20, ?, ?, ?, ?, ?, 1, '2026-09-01', '2026-09-02')`) + VALUES (?, ?, 20, ?, ?, ?, ?, ?, 1, '2026-09-01', '2026-09-02')` + ) .run(inserted.lastInsertRowid, guid, kills, headshots, downs, rounds, completed); } } -const samples = (count: number, sample: Sample = {}) => Array.from({ length: count }, () => ({ ...sample })); +const samples = (count: number, sample: Sample = {}) => + Array.from({ length: count }, () => ({ ...sample })); it('calculates every efficiency measure from aggregate counts and preserves raw ratios', async () => { const { db } = await import('../../src/lib/server/db/index.js'); @@ -39,10 +67,21 @@ it('calculates every efficiency measure from aggregate counts and preserves raw ]); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const entry = getLeaderboard().entries[0]; - expect(entry).toMatchObject({ kills: 100, headshots: 10, downs: 10, roundsSurvived: 10, - gamesPlayed: 5, gamesCompleted: 1, headshotRate: 0.1, killsPerDown: 10, - averageKills: 20, averageRounds: 2, completionRate: 0.2 }); - const { efficiencyMetrics, efficiencyValue } = await import('../../src/lib/leaderboard-efficiency.js'); + expect(entry).toMatchObject({ + kills: 100, + headshots: 10, + downs: 10, + roundsSurvived: 10, + gamesPlayed: 5, + gamesCompleted: 1, + headshotRate: 0.1, + killsPerDown: 10, + averageKills: 20, + averageRounds: 2, + completionRate: 0.2 + }); + const { efficiencyMetrics, efficiencyValue } = + await import('../../src/lib/leaderboard-efficiency.js'); for (const { key } of efficiencyMetrics) { expect(entry[key]).toBe(efficiencyValue(entry, key)); expect(getLeaderboard({ metric: key }).entries).toEqual([entry]); @@ -57,7 +96,9 @@ it('qualifies at five games, one hundred kills and five downs, with each metric addSamples(db(), 'four-downs', [{ downs: 0 }, ...samples(4)]); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const ids = (metric: LeaderboardMetric) => - getLeaderboard({ metric }).entries.map(entry => entry.guid).sort(); + getLeaderboard({ metric }) + .entries.map((entry) => entry.guid) + .sort(); expect(ids('headshotRate')).toEqual(['exact', 'four-downs']); expect(ids('killsPerDown')).toEqual(['exact', 'ninety-nine-kills']); for (const metric of ['averageKills', 'averageRounds', 'completionRate'] as const) { @@ -69,16 +110,31 @@ it('qualifies at five games, one hundred kills and five downs, with each metric it('keeps undefined ratios null and unranked while ranking legitimate zero efficiencies', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - addSamples(db(), 'zero', samples(5, { kills: 0, headshots: 0, downs: 0, rounds: 0, completed: 0 })); + addSamples( + db(), + 'zero', + samples(5, { kills: 0, headshots: 0, downs: 0, rounds: 0, completed: 0 }) + ); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard().entries[0]).toMatchObject({ headshotRate: null, killsPerDown: null, - averageKills: 0, averageRounds: 0, completionRate: 0 }); + expect(getLeaderboard().entries[0]).toMatchObject({ + headshotRate: null, + killsPerDown: null, + averageKills: 0, + averageRounds: 0, + completionRate: 0 + }); for (const metric of ['headshotRate', 'killsPerDown'] as const) { - expect(getLeaderboard({ metric })).toMatchObject({ entries: [], total: 0, - summary: { players: 1, matches: 5, kills: 0 } }); + expect(getLeaderboard({ metric })).toMatchObject({ + entries: [], + total: 0, + summary: { players: 1, matches: 5, kills: 0 } + }); } for (const metric of ['averageKills', 'averageRounds', 'completionRate'] as const) { - expect(getLeaderboard({ metric })).toMatchObject({ total: 1, entries: [{ guid: 'zero', rank: 1 }] }); + expect(getLeaderboard({ metric })).toMatchObject({ + total: 1, + entries: [{ guid: 'zero', rank: 1 }] + }); } addSamples(db(), 'no-kills', samples(5, { kills: 0, headshots: 0, downs: 1 })); expect(getLeaderboard({ metric: 'killsPerDown' }).entries).toEqual([ @@ -89,19 +145,31 @@ it('keeps undefined ratios null and unranked while ranking legitimate zero effic it('recomputes qualification within selected games and maps using completed matches only', async () => { const { db } = await import('../../src/lib/server/db/index.js'); addSamples(db(), 'split', [...samples(3), ...samples(2, { game: 't6', map: 'town' })]); - addSamples(db(), 'unfinished', [...samples(4), { status: 'active', kills: 10000 }, - { status: 'interrupted', kills: 10000 }]); - const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard({ metric: 'averageKills' })).toMatchObject({ total: 1, - entries: [{ guid: 'split', gamesPlayed: 5, averageKills: 20 }], summary: { players: 2, matches: 9, kills: 180 } }); - expect(getLeaderboard({ metric: 'averageKills', game: 't5' })).toMatchObject({ total: 0, entries: [], - summary: { players: 2, matches: 7, kills: 140 } }); - expect(getLeaderboard({ metric: 'averageKills', map: 't6:town' })).toMatchObject({ total: 0, entries: [], - summary: { players: 1, matches: 2, kills: 40 } }); - addSamples(db(), 'split', samples(2)); - expect(getLeaderboard({ metric: 'averageKills', game: 't5', map: 't5:theater' }).entries).toEqual([ - expect.objectContaining({ guid: 'split', gamesPlayed: 5, rank: 1 }) + addSamples(db(), 'unfinished', [ + ...samples(4), + { status: 'active', kills: 10000 }, + { status: 'interrupted', kills: 10000 } ]); + const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); + expect(getLeaderboard({ metric: 'averageKills' })).toMatchObject({ + total: 1, + entries: [{ guid: 'split', gamesPlayed: 5, averageKills: 20 }], + summary: { players: 2, matches: 9, kills: 180 } + }); + expect(getLeaderboard({ metric: 'averageKills', game: 't5' })).toMatchObject({ + total: 0, + entries: [], + summary: { players: 2, matches: 7, kills: 140 } + }); + expect(getLeaderboard({ metric: 'averageKills', map: 't6:town' })).toMatchObject({ + total: 0, + entries: [], + summary: { players: 1, matches: 2, kills: 40 } + }); + addSamples(db(), 'split', samples(2)); + expect( + getLeaderboard({ metric: 'averageKills', game: 't5', map: 't5:theater' }).entries + ).toEqual([expect.objectContaining({ guid: 'split', gamesPlayed: 5, rank: 1 })]); }); it('qualifies before competition ranking, then searches and paginates without changing scope summaries', async () => { @@ -111,22 +179,59 @@ it('qualifies before competition ranking, then searches and paginates without ch addSamples(handle, 'Blake', samples(5, { kills: 50 })); addSamples(handle, 'Casey', samples(5, { kills: 10 })); addSamples(handle, 'Small sample', samples(4, { kills: 1000 })); - handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES ('t6', 'Casey', 'Current name', '2026-09-01', '2026-09-03')`).run(); + handle + .prepare( + `INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES ('t6', 'Casey', 'Current name', '2026-09-01', '2026-09-03')` + ) + .run(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const full = getLeaderboard({ metric: 'averageKills', game: 't5' }); - expect(full.entries.map(entry => [entry.name, entry.rank])).toEqual([['Ace', 1], ['Blake', 1], ['Current name', 3]]); - const searched = getLeaderboard({ metric: 'averageKills', game: 't5', q: 'current NAME', limit: 1 }); - expect(searched).toMatchObject({ total: 1, entries: [{ guid: 'Casey', rank: 3 }], summary: full.summary }); - expect(getLeaderboard({ metric: 'averageKills', game: 't5', q: 'Small' })).toMatchObject({ total: 0, entries: [], summary: full.summary }); + expect(full.entries.map((entry) => [entry.name, entry.rank])).toEqual([ + ['Ace', 1], + ['Blake', 1], + ['Current name', 3] + ]); + const searched = getLeaderboard({ + metric: 'averageKills', + game: 't5', + q: 'current NAME', + limit: 1 + }); + expect(searched).toMatchObject({ + total: 1, + entries: [{ guid: 'Casey', rank: 3 }], + summary: full.summary + }); + expect(getLeaderboard({ metric: 'averageKills', game: 't5', q: 'Small' })).toMatchObject({ + total: 0, + entries: [], + summary: full.summary + }); const page = getLeaderboard({ metric: 'averageKills', game: 't5', limit: 1, offset: 1 }); - expect(page).toMatchObject({ total: 3, entries: [{ guid: 'Blake', rank: 1 }], summary: full.summary }); - expect(getLeaderboard({ metric: 'averageKills', offset: 3 })).toMatchObject({ total: 3, entries: [], summary: full.summary }); + expect(page).toMatchObject({ + total: 3, + entries: [{ guid: 'Blake', rank: 1 }], + summary: full.summary + }); + expect(getLeaderboard({ metric: 'averageKills', offset: 3 })).toMatchObject({ + total: 3, + entries: [], + summary: full.summary + }); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - const response = await GET({ url: new URL('http://localhost/api/leaderboard?metric=averageKills&game=t5&q=current&limit=1') }); + const response = await GET({ + url: new URL( + 'http://localhost/api/leaderboard?metric=averageKills&game=t5&q=current&limit=1' + ) + }); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ metric: 'averageKills', total: 1, - entries: [{ guid: 'Casey', rank: 3, averageKills: 10, gamesPlayed: 5 }], summary: full.summary }); + expect(await response.json()).toMatchObject({ + metric: 'averageKills', + total: 1, + entries: [{ guid: 'Casey', rank: 3, averageKills: 10, gamesPlayed: 5 }], + summary: full.summary + }); }); it('accepts each efficiency metric through the API when the qualified group is empty', async () => { @@ -135,21 +240,37 @@ it('accepts each efficiency metric through the API when the qualified group is e const { efficiencyMetrics } = await import('../../src/lib/leaderboard-efficiency.js'); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); for (const { key } of efficiencyMetrics) { - const response = await GET({ url: new URL(`http://localhost/api/leaderboard?metric=${key}`) }); + const response = await GET({ + url: new URL(`http://localhost/api/leaderboard?metric=${key}`) + }); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ metric: key, entries: [], total: 0, - summary: { players: 1, matches: 4, kills: 80 } }); + expect(await response.json()).toMatchObject({ + metric: key, + entries: [], + total: 0, + summary: { players: 1, matches: 4, kills: 80 } + }); } }); it('does not use same-round departures to qualify for efficiency rankings', async () => { const { db } = await import('../../src/lib/server/db/index.js'); addSamples(db(), 'short-visit', [...samples(4), { completed: 0 }]); - db().prepare(`UPDATE leaderboard_match_players SET first_observed_round = highest_round - WHERE completed = 0`).run(); + db() + .prepare( + `UPDATE leaderboard_match_players SET first_observed_round = highest_round + WHERE completed = 0` + ) + .run(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard().entries[0]).toMatchObject({ gamesPlayed: 4, gamesCompleted: 4, - kills: 100, averageKills: 25, completionRate: 1 }); + expect(getLeaderboard().entries[0]).toMatchObject({ + gamesPlayed: 4, + gamesCompleted: 4, + kills: 100, + averageKills: 25, + completionRate: 1 + }); const { efficiencyMetrics } = await import('../../src/lib/leaderboard-efficiency.js'); - for (const { key } of efficiencyMetrics) expect(getLeaderboard({ metric: key }).entries).toEqual([]); + for (const { key } of efficiencyMetrics) + expect(getLeaderboard({ metric: key }).entries).toEqual([]); }); diff --git a/tests/unit/leaderboard-efficiency.test.ts b/tests/unit/leaderboard-efficiency.test.ts index fd88574..68c2cb6 100644 --- a/tests/unit/leaderboard-efficiency.test.ts +++ b/tests/unit/leaderboard-efficiency.test.ts @@ -1,48 +1,95 @@ import { describe, expect, it } from 'vitest'; import type { PlayerStats } from '../../src/lib/leaderboard.js'; import { - EFFICIENCY_MIN_DOWNS, EFFICIENCY_MIN_GAMES, EFFICIENCY_MIN_KILLS, - efficiencyEligible, efficiencyMetrics, efficiencyRequirement, efficiencySample, efficiencyValue, - isEfficiencyMetric, type EfficiencyMetric + EFFICIENCY_MIN_DOWNS, + EFFICIENCY_MIN_GAMES, + EFFICIENCY_MIN_KILLS, + efficiencyEligible, + efficiencyMetrics, + efficiencyRequirement, + efficiencySample, + efficiencyValue, + isEfficiencyMetric, + type EfficiencyMetric } from '../../src/lib/leaderboard-efficiency.js'; const stats = (overrides: Partial = {}): PlayerStats => ({ - highestRound: 20, kills: 100, roundsSurvived: 47, headshots: 31, revives: 7, downs: 5, - gamesPlayed: 5, gamesCompleted: 3, bestKills: 50, bestScore: 12000, ...overrides + highestRound: 20, + kills: 100, + roundsSurvived: 47, + headshots: 31, + revives: 7, + downs: 5, + gamesPlayed: 5, + gamesCompleted: 3, + bestKills: 50, + bestScore: 12000, + ...overrides }); describe('efficiency ranking qualification', () => { - it.each(efficiencyMetrics)('requires five ended games for $key even when other counters are large', ({ key }) => { - expect(efficiencyEligible(stats({ gamesPlayed: EFFICIENCY_MIN_GAMES - 1, kills: 10000, downs: 100 }), key)).toBe(false); - expect(efficiencyEligible(stats({ gamesPlayed: EFFICIENCY_MIN_GAMES }), key)).toBe(true); - }); + it.each(efficiencyMetrics)( + 'requires five ended games for $key even when other counters are large', + ({ key }) => { + expect( + efficiencyEligible( + stats({ gamesPlayed: EFFICIENCY_MIN_GAMES - 1, kills: 10000, downs: 100 }), + key + ) + ).toBe(false); + expect(efficiencyEligible(stats({ gamesPlayed: EFFICIENCY_MIN_GAMES }), key)).toBe( + true + ); + } + ); it('requires enough kills for headshot rate and enough downs for kills per down', () => { - expect(efficiencyEligible(stats({ kills: EFFICIENCY_MIN_KILLS - 1 }), 'headshotRate')).toBe(false); - expect(efficiencyEligible(stats({ kills: EFFICIENCY_MIN_KILLS }), 'headshotRate')).toBe(true); - expect(efficiencyEligible(stats({ downs: EFFICIENCY_MIN_DOWNS - 1 }), 'killsPerDown')).toBe(false); - expect(efficiencyEligible(stats({ downs: EFFICIENCY_MIN_DOWNS }), 'killsPerDown')).toBe(true); + expect(efficiencyEligible(stats({ kills: EFFICIENCY_MIN_KILLS - 1 }), 'headshotRate')).toBe( + false + ); + expect(efficiencyEligible(stats({ kills: EFFICIENCY_MIN_KILLS }), 'headshotRate')).toBe( + true + ); + expect(efficiencyEligible(stats({ downs: EFFICIENCY_MIN_DOWNS - 1 }), 'killsPerDown')).toBe( + false + ); + expect(efficiencyEligible(stats({ downs: EFFICIENCY_MIN_DOWNS }), 'killsPerDown')).toBe( + true + ); expect(efficiencyEligible(stats({ kills: 0 }), 'killsPerDown')).toBe(true); }); it('counts departed players in ended-game samples and applies no extra counter minimum to game averages', () => { - const departed = stats({ kills: 0, headshots: 0, downs: 0, roundsSurvived: 0, gamesCompleted: 0 }); + const departed = stats({ + kills: 0, + headshots: 0, + downs: 0, + roundsSurvived: 0, + gamesCompleted: 0 + }); for (const metric of ['averageKills', 'averageRounds', 'completionRate'] as const) { expect(efficiencyEligible(departed, metric)).toBe(true); expect(efficiencyValue(departed, metric)).toBe(0); } - expect(efficiencyValue(stats({ gamesPlayed: 5, gamesCompleted: 3 }), 'completionRate')).toBe(3 / 5); + expect( + efficiencyValue(stats({ gamesPlayed: 5, gamesCompleted: 3 }), 'completionRate') + ).toBe(3 / 5); }); }); describe('efficiency values', () => { it.each([ - ['headshotRate', { kills: 0 }], ['killsPerDown', { downs: 0 }], - ['averageKills', { gamesPlayed: 0 }], ['averageRounds', { gamesPlayed: 0 }], + ['headshotRate', { kills: 0 }], + ['killsPerDown', { downs: 0 }], + ['averageKills', { gamesPlayed: 0 }], + ['averageRounds', { gamesPlayed: 0 }], ['completionRate', { gamesPlayed: 0 }] - ] as Array<[EfficiencyMetric, Partial]>)('leaves %s undefined with no denominator', (metric, zeroDenominator) => { - expect(efficiencyValue(stats(zeroDenominator), metric)).toBeNull(); - }); + ] as Array<[EfficiencyMetric, Partial]>)( + 'leaves %s undefined with no denominator', + (metric, zeroDenominator) => { + expect(efficiencyValue(stats(zeroDenominator), metric)).toBeNull(); + } + ); it('uses ratios of totals instead of equally weighting games with different sample sizes', () => { // One game has 1/1 headshots and 1 kill/down; another has 9/99 and 99/9. @@ -55,7 +102,14 @@ describe('efficiency values', () => { }); it('preserves fractional ratios and low samples without rounding, capping or inventing a denominator', () => { - const small = stats({ gamesPlayed: 3, gamesCompleted: 1, kills: 2, headshots: 1, downs: 3, roundsSurvived: 7 }); + const small = stats({ + gamesPlayed: 3, + gamesCompleted: 1, + kills: 2, + headshots: 1, + downs: 3, + roundsSurvived: 7 + }); expect(efficiencyValue(small, 'headshotRate')).toBe(1 / 2); expect(efficiencyValue(small, 'killsPerDown')).toBe(2 / 3); expect(efficiencyValue(small, 'averageKills')).toBe(2 / 3); @@ -71,9 +125,13 @@ describe('efficiency explanations', () => { expect(efficiencySample(player, 'headshotRate')).toBe('31 headshots / 1,200 kills'); expect(efficiencySample(player, 'killsPerDown')).toBe('1,200 kills / 5 downs'); expect(efficiencySample(player, 'averageKills')).toBe('1,200 kills / 5 ended games'); - expect(efficiencySample(player, 'averageRounds')).toBe('47 observed rounds / 5 ended games'); + expect(efficiencySample(player, 'averageRounds')).toBe( + '47 observed rounds / 5 ended games' + ); expect(efficiencySample(player, 'completionRate')).toBe('3 finishes / 5 ended games'); - expect(efficiencySample(stats({ kills: 1, downs: 0 }), 'killsPerDown')).toBe('1 kill / 0 downs'); + expect(efficiencySample(stats({ kills: 1, downs: 0 }), 'killsPerDown')).toBe( + '1 kill / 0 downs' + ); }); it('identifies ratio metrics and publishes the thresholds used to qualify them', () => { @@ -84,8 +142,11 @@ describe('efficiency explanations', () => { } expect(efficiencyRequirement('headshotRate')).toContain(`${EFFICIENCY_MIN_KILLS} kills`); expect(efficiencyRequirement('killsPerDown')).toContain(`${EFFICIENCY_MIN_DOWNS} downs`); - expect(efficiencyMetrics.filter(metric => metric.format === 'percent').map(metric => metric.key)) - .toEqual(['headshotRate', 'completionRate']); + expect( + efficiencyMetrics + .filter((metric) => metric.format === 'percent') + .map((metric) => metric.key) + ).toEqual(['headshotRate', 'completionRate']); expect(isEfficiencyMetric('kills')).toBe(false); expect(isEfficiencyMetric('')).toBe(false); }); diff --git a/tests/unit/leaderboard-ingest.test.ts b/tests/unit/leaderboard-ingest.test.ts index 8a662f0..0e76ffa 100644 --- a/tests/unit/leaderboard-ingest.test.ts +++ b/tests/unit/leaderboard-ingest.test.ts @@ -5,8 +5,18 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; let received = 0; const server = 't5_zombie_theater_1'; -const player = (guid = '2741626', stats: Partial = {}): TelemetryPlayerStats => ({ - guid, name: 'JoshWright', kills: 0, headshots: 0, downs: 0, revives: 0, score: 0, ...stats +const player = ( + guid = '2741626', + stats: Partial = {} +): TelemetryPlayerStats => ({ + guid, + name: 'JoshWright', + kills: 0, + headshots: 0, + downs: 0, + revives: 0, + score: 0, + ...stats }); beforeEach(async () => { @@ -18,25 +28,50 @@ beforeEach(async () => { }); afterEach(async () => fixture.cleanup()); -async function report(event: GameEvent['event'], round = 0, players = [player()], source = server, map = 'zombie_theater') { +async function report( + event: GameEvent['event'], + round = 0, + players = [player()], + source = server, + map = 'zombie_theater' +) { const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); const receivedAt = new Date(Date.UTC(2026, 8, 18, 20, 0, received++)).toISOString(); - saveTelemetry(event === 'game_start' - ? { event, server: source, map, players: players.map(({ name, guid }) => ({ name, guid })), receivedAt } - : { event, server: source, map, round, players, receivedAt }); + saveTelemetry( + event === 'game_start' + ? { + event, + server: source, + map, + players: players.map(({ name, guid }) => ({ name, guid })), + receivedAt + } + : { event, server: source, map, round, players, receivedAt } + ); } async function matches() { const { db } = await import('../../src/lib/server/db/index.js'); return db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all() as Array<{ - id: number; game: string; map: string; map_key: string; map_name: string; - status: string; highest_round: number; start_observed: number; + id: number; + game: string; + map: string; + map_key: string; + map_name: string; + status: string; + highest_round: number; + start_observed: number; }>; } it('keeps report revisions stable during a match and invalidates completed data and renamed players', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const revision = () => (db().prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; + const revision = () => + ( + db() + .prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'") + .get() as { revision: number } + ).revision; await report('game_start'); const started = revision(); for (let round = 1; round <= 30; round++) { @@ -66,11 +101,21 @@ it('publishes round 2 or higher games only after game end, preserving early stat const departed = player('round-one-only', { kills: 5 }); await report('game_start', 0, [player(), departed]); await report('round_change', 1, [player(undefined, { kills: 6, score: 1000 }), departed]); - expect(getLeaderboard()).toMatchObject({ entries: [], maps: [], total: 0, summary: { players: 0, matches: 0, kills: 0 } }); + expect(getLeaderboard()).toMatchObject({ + entries: [], + maps: [], + total: 0, + summary: { players: 0, matches: 0, kills: 0 } + }); expect(getPlayerProfile(player().guid)).toBeNull(); expect(serverMatchHistory()).toEqual({ total: 0, matches: [], maps: [], page: 1, limit: 25 }); await report('round_change', 2, [player(undefined, { kills: 6, score: 500 })]); - expect(getLeaderboard()).toMatchObject({ entries: [], maps: [], total: 0, summary: { players: 0, matches: 0, kills: 0 } }); + expect(getLeaderboard()).toMatchObject({ + entries: [], + maps: [], + total: 0, + summary: { players: 0, matches: 0, kills: 0 } + }); expect(getPlayerProfile(player().guid)).toBeNull(); expect(serverMatchHistory()).toMatchObject({ total: 1, matches: [{ status: 'active' }] }); expect(getPlayerProfile(departed.guid)).toBeNull(); @@ -78,11 +123,25 @@ it('publishes round 2 or higher games only after game end, preserving early stat const board = getLeaderboard(); const profile = getPlayerProfile(player().guid); const history = serverMatchHistory(); - expect(board).toMatchObject({ total: 1, entries: [expect.objectContaining({ - highestRound: 2, kills: 10, bestScore: 1000, roundsSurvived: 1, gamesPlayed: 1, gamesCompleted: 1 - })], summary: { players: 1, matches: 1, kills: 10 } }); + expect(board).toMatchObject({ + total: 1, + entries: [ + expect.objectContaining({ + highestRound: 2, + kills: 10, + bestScore: 1000, + roundsSurvived: 1, + gamesPlayed: 1, + gamesCompleted: 1 + }) + ], + summary: { players: 1, matches: 1, kills: 10 } + }); expect(getPlayerProfile(departed.guid)).toBeNull(); - expect(history).toMatchObject({ total: 1, matches: [{ startPlayers: 2, uniquePlayers: 2, durationMs: 3000, finalRound: 2 }] }); + expect(history).toMatchObject({ + total: 1, + matches: [{ startPlayers: 2, uniquePlayers: 2, durationMs: 3000, finalRound: 2 }] + }); await report('game_start'); await report('game_end', 1, [player(undefined, { kills: 5000 })]); await report('game_start', 0, []); @@ -91,12 +150,25 @@ it('publishes round 2 or higher games only after game end, preserving early stat closeDb(); expect(getLeaderboard()).toEqual(board); // Identity timestamps still reflect live presence, but gameplay records do not change. - expect(getPlayerProfile(player().guid)).toMatchObject({ stats: profile!.stats, maps: profile!.maps, matches: profile!.matches }); + expect(getPlayerProfile(player().guid)).toMatchObject({ + stats: profile!.stats, + maps: profile!.maps, + matches: profile!.matches + }); expect(serverMatchHistory()).toEqual(history); const { GET } = await import('../../src/routes/api/matches/+server.js'); - expect(await GET({ url: new URL('http://localhost/api/matches?limit=1') }).json()).toEqual({ ...history, limit: 1 }); - const { GET: serverGET } = await import('../../src/routes/api/servers/[name]/matches/+server.js'); - expect(await serverGET({ params: { name: server }, url: new URL('http://localhost/?limit=1') }).json()).toEqual({ ...history, limit: 1 }); + expect(await GET({ url: new URL('http://localhost/api/matches?limit=1') }).json()).toEqual({ + ...history, + limit: 1 + }); + const { GET: serverGET } = + await import('../../src/routes/api/servers/[name]/matches/+server.js'); + expect( + await serverGET({ + params: { name: server }, + url: new URL('http://localhost/?limit=1') + }).json() + ).toEqual({ ...history, limit: 1 }); }); it('records server match counts, unique rejoins, duration, and final round durably', async () => { @@ -108,11 +180,27 @@ it('records server match counts, unique rejoins, duration, and final round durab const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); - expect(serverMatchHistory(server)).toMatchObject({ total: 1, matches: [expect.objectContaining({ - status: 'completed', startPlayers: 2, uniquePlayers: 3, endPlayers: 1, - durationMs: 3000, finalRound: 7, endedAt: '2026-09-18T20:00:03.000Z' - })] }); - expect(serverMatchHistory('other')).toEqual({ total: 0, matches: [], maps: [], page: 1, limit: 25 }); + expect(serverMatchHistory(server)).toMatchObject({ + total: 1, + matches: [ + expect.objectContaining({ + status: 'completed', + startPlayers: 2, + uniquePlayers: 3, + endPlayers: 1, + durationMs: 3000, + finalRound: 7, + endedAt: '2026-09-18T20:00:03.000Z' + }) + ] + }); + expect(serverMatchHistory('other')).toEqual({ + total: 0, + matches: [], + maps: [], + page: 1, + limit: 25 + }); }); it('preserves unknown boundaries and distinguishes empty rosters from missing observations', async () => { @@ -124,16 +212,39 @@ it('preserves unknown boundaries and distinguishes empty rosters from missing ob await report('round_change', 2); const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); expect(serverMatchHistory(server).matches).toEqual([ - expect.objectContaining({ status: 'active', startPlayers: 1, endPlayers: null, durationMs: null, finalRound: null, endedAt: null }), - expect.objectContaining({ status: 'interrupted', startPlayers: 0, endPlayers: null, durationMs: null, finalRound: null, endedAt: null }), - expect.objectContaining({ status: 'completed', startPlayers: null, uniquePlayers: 1, endPlayers: 0, durationMs: null, finalRound: 6 }) + expect.objectContaining({ + status: 'active', + startPlayers: 1, + endPlayers: null, + durationMs: null, + finalRound: null, + endedAt: null + }), + expect.objectContaining({ + status: 'interrupted', + startPlayers: 0, + endPlayers: null, + durationMs: null, + finalRound: null, + endedAt: null + }), + expect.objectContaining({ + status: 'completed', + startPlayers: null, + uniquePlayers: 1, + endPlayers: 0, + durationMs: null, + finalRound: 6 + }) ]); const { GET } = await import('../../src/routes/api/servers/[name]/matches/+server.js'); const response = GET({ params: { name: server }, url: new URL('http://localhost/?limit=1') }); const body = await response.json(); expect(body.total).toBe(3); expect(body.matches).toHaveLength(1); - expect(GET({ params: { name: server }, url: new URL('http://localhost/?limit=-1') }).status).toBe(400); + expect( + GET({ params: { name: server }, url: new URL('http://localhost/?limit=-1') }).status + ).toBe(400); }); it('lists global matches newest first across servers with totals and ending timestamps', async () => { @@ -156,10 +267,20 @@ it('lists global matches newest first across servers with totals and ending time async function records(guid = '2741626') { const { db } = await import('../../src/lib/server/db/index.js'); - return db().prepare('SELECT * FROM leaderboard_match_players WHERE guid = ? ORDER BY match_id').all(guid) as Array<{ - match_id: number; guid: string; highest_round: number; rounds_survived: number; - kills: number; headshots: number; downs: number; revives: number; best_score: number; - completed: number; present: number; + return db() + .prepare('SELECT * FROM leaderboard_match_players WHERE guid = ? ORDER BY match_id') + .all(guid) as Array<{ + match_id: number; + guid: string; + highest_round: number; + rounds_survived: number; + kills: number; + headshots: number; + downs: number; + revives: number; + best_score: number; + completed: number; + present: number; }>; } @@ -168,30 +289,60 @@ it('refreshes stats during a round without adding rounds or completing the match await report('round_change', 30, [player(undefined, { kills: 100 })]); await report('heartbeat', 30, [player(undefined, { kills: 150 })]); await report('heartbeat', 30, [player(undefined, { kills: 150 })]); - expect(await matches()).toEqual([expect.objectContaining({ status: 'active', highest_round: 30 })]); - expect(await records()).toEqual([expect.objectContaining({ kills: 150, rounds_survived: 29, completed: 0 })]); + expect(await matches()).toEqual([ + expect.objectContaining({ status: 'active', highest_round: 30 }) + ]); + expect(await records()).toEqual([ + expect.objectContaining({ kills: 150, rounds_survived: 29, completed: 0 }) + ]); await report('game_end', 30); await report('heartbeat', 30); - expect((await matches()).map(match => match.status)).toEqual(['completed', 'active']); + expect((await matches()).map((match) => match.status)).toEqual(['completed', 'active']); }); it('persists independent stat high-water marks without adding cumulative snapshots or retries', async () => { await report('game_start'); await report('round_change', 1); - const first = player(undefined, { kills: 100, headshots: 60, downs: 2, revives: 3, score: 5000 }); + const first = player(undefined, { + kills: 100, + headshots: 60, + downs: 2, + revives: 3, + score: 5000 + }); await report('round_change', 3, [first]); await report('round_change', 3, [first]); - await report('round_change', 4, [player(undefined, { - kills: 120, headshots: 55, downs: 1, revives: 5, score: 1000 - })]); - await report('game_end', 4, [player(undefined, { - kills: 125, headshots: 62, downs: 3, revives: 5, score: 2000 - })]); + await report('round_change', 4, [ + player(undefined, { + kills: 120, + headshots: 55, + downs: 1, + revives: 5, + score: 1000 + }) + ]); + await report('game_end', 4, [ + player(undefined, { + kills: 125, + headshots: 62, + downs: 3, + revives: 5, + score: 2000 + }) + ]); expect(await matches()).toHaveLength(1); - expect(await records()).toEqual([expect.objectContaining({ - highest_round: 4, rounds_survived: 3, kills: 125, headshots: 62, - downs: 3, revives: 5, best_score: 5000, completed: 1 - })]); + expect(await records()).toEqual([ + expect.objectContaining({ + highest_round: 4, + rounds_survived: 3, + kills: 125, + headshots: 62, + downs: 3, + revives: 5, + best_score: 5000, + completed: 1 + }) + ]); }); it('combines the same GUID into one profile and ranking across games', async () => { @@ -199,33 +350,85 @@ it('combines the same GUID into one profile and ranking across games', async () await report('game_end', 8, [player(undefined, { kills: 120, name: 'New name' })]); await report('game_start'); await report('game_end', 2, [player(undefined, { kills: 20, name: 'Latest name' })]); - await report('game_end', 5, [player(undefined, { kills: 200, name: 'Other game' })], 't6_zombie_town_1', 'zm_transit'); + await report( + 'game_end', + 5, + [player(undefined, { kills: 200, name: 'Other game' })], + 't6_zombie_town_1', + 'zm_transit' + ); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT game, guid, name FROM leaderboard_players ORDER BY game').all()).toEqual([ + expect( + db().prepare('SELECT game, guid, name FROM leaderboard_players ORDER BY game').all() + ).toEqual([ { game: 't5', guid: '2741626', name: 'Latest name' }, { game: 't6', guid: '2741626', name: 'Other game' } ]); - expect((await records()).map(row => row.kills)).toEqual([120, 20, 200]); - expect(db().prepare(`SELECT SUM(p.rounds_survived) AS total FROM leaderboard_match_players p - JOIN leaderboard_matches m ON m.id = p.match_id WHERE m.game = 't5' AND p.guid = ?`).get('2741626')) - .toEqual({ total: 1 }); + expect((await records()).map((row) => row.kills)).toEqual([120, 20, 200]); + expect( + db() + .prepare( + `SELECT SUM(p.rounds_survived) AS total FROM leaderboard_match_players p + JOIN leaderboard_matches m ON m.id = p.match_id WHERE m.game = 't5' AND p.guid = ?` + ) + .get('2741626') + ).toEqual({ total: 1 }); closeDb(); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const profile = getPlayerProfile(player().guid)!; - expect(profile).toMatchObject({ guid: player().guid, name: 'Other game', - firstSeen: '2026-09-18T20:00:00.000Z', lastSeen: '2026-09-18T20:00:04.000Z', - stats: { gamesPlayed: 3, gamesCompleted: 3, kills: 340, highestRound: 8, bestKills: 200, roundsSurvived: 1 } }); - expect(profile.matches.map(match => match.game)).toEqual(['t6', 't5', 't5']); + expect(profile).toMatchObject({ + guid: player().guid, + name: 'Other game', + firstSeen: '2026-09-18T20:00:00.000Z', + lastSeen: '2026-09-18T20:00:04.000Z', + stats: { + gamesPlayed: 3, + gamesCompleted: 3, + kills: 340, + highestRound: 8, + bestKills: 200, + roundsSurvived: 1 + } + }); + expect(profile.matches.map((match) => match.game)).toEqual(['t6', 't5', 't5']); const board = getLeaderboard(); expect(board).toMatchObject({ total: 1, summary: { players: 1, matches: 3, kills: 340 } }); - expect(board.entries).toEqual([expect.objectContaining({ ...profile.stats, guid: player().guid, name: profile.name, rank: 1 })]); - expect(getLeaderboard({ game: 't5' }).entries).toEqual([expect.objectContaining({ guid: player().guid, name: 'Other game', kills: 140, gamesPlayed: 2 })]); - expect(getLeaderboard({ game: 't6' }).entries).toEqual([expect.objectContaining({ guid: player().guid, name: 'Other game', kills: 200, gamesPlayed: 1 })]); + expect(board.entries).toEqual([ + expect.objectContaining({ + ...profile.stats, + guid: player().guid, + name: profile.name, + rank: 1 + }) + ]); + expect(getLeaderboard({ game: 't5' }).entries).toEqual([ + expect.objectContaining({ + guid: player().guid, + name: 'Other game', + kills: 140, + gamesPlayed: 2 + }) + ]); + expect(getLeaderboard({ game: 't6' }).entries).toEqual([ + expect.objectContaining({ + guid: player().guid, + name: 'Other game', + kills: 200, + gamesPlayed: 1 + }) + ]); await report('game_start', 0, [player(undefined, { name: 'Live name' })]); - expect(getPlayerProfile(player().guid)).toMatchObject({ name: 'Live name', lastSeen: '2026-09-18T20:00:05.000Z', - stats: profile.stats, maps: profile.maps, matches: profile.matches }); - expect(getLeaderboard().entries).toEqual([expect.objectContaining({ ...profile.stats, name: 'Live name' })]); + expect(getPlayerProfile(player().guid)).toMatchObject({ + name: 'Live name', + lastSeen: '2026-09-18T20:00:05.000Z', + stats: profile.stats, + maps: profile.maps, + matches: profile.matches + }); + expect(getLeaderboard().entries).toEqual([ + expect.objectContaining({ ...profile.stats, name: 'Live name' }) + ]); }); it('credits the highest round but only observed survival for a late join or a return after an absent roster', async () => { @@ -236,21 +439,36 @@ it('credits the highest round but only observed survival for a late join or a re await report('round_change', 5, [player(), late]); await report('round_change', 6, [late, player()]); await report('game_end', 6, [player()]); - expect(await records()).toEqual([expect.objectContaining({ - highest_round: 6, rounds_survived: 2, completed: 1 - })]); - expect(await records(late.guid)).toEqual([expect.objectContaining({ - highest_round: 6, rounds_survived: 1, completed: 0, present: 0 - })]); + expect(await records()).toEqual([ + expect.objectContaining({ + highest_round: 6, + rounds_survived: 2, + completed: 1 + }) + ]); + expect(await records(late.guid)).toEqual([ + expect.objectContaining({ + highest_round: 6, + rounds_survived: 1, + completed: 0, + present: 0 + }) + ]); }); it('credits the highest round without inventing survival before the first midgame snapshot', async () => { await report('round_change', 8, [player(undefined, { kills: 111 })]); - expect(await records()).toEqual([expect.objectContaining({ highest_round: 8, rounds_survived: 0, kills: 111 })]); + expect(await records()).toEqual([ + expect.objectContaining({ highest_round: 8, rounds_survived: 0, kills: 111 }) + ]); await report('round_change', 10, [player(undefined, { kills: 160 })]); await report('game_end', 10, [player(undefined, { kills: 170 })]); - expect(await records()).toEqual([expect.objectContaining({ highest_round: 10, rounds_survived: 2, completed: 1 })]); - expect(await matches()).toEqual([expect.objectContaining({ start_observed: 0, status: 'completed' })]); + expect(await records()).toEqual([ + expect.objectContaining({ highest_round: 10, rounds_survived: 2, completed: 1 }) + ]); + expect(await matches()).toEqual([ + expect.objectContaining({ start_observed: 0, status: 'completed' }) + ]); }); it('keeps disconnected player records while completing only the final roster', async () => { @@ -262,11 +480,20 @@ it('keeps disconnected player records while completing only the final roster', a expect(getLeaderboard()).toMatchObject({ entries: [], total: 0 }); expect(getPlayerProfile(other.guid)).toBeNull(); await report('game_end', 5, [player(undefined, { kills: 100 })]); - expect(await records(other.guid)).toEqual([expect.objectContaining({ kills: 40, highest_round: 4, rounds_survived: 3, completed: 0 })]); - expect(await records()).toEqual([expect.objectContaining({ kills: 100, highest_round: 5, rounds_survived: 4, completed: 1 })]); - expect(getLeaderboard()).toMatchObject({ total: 2, summary: { matches: 1, players: 2, kills: 140 } }); - expect(getPlayerProfile(other.guid)).toMatchObject({ stats: { kills: 40, highestRound: 4, - roundsSurvived: 3, gamesPlayed: 1, gamesCompleted: 0 }, matches: [{ status: 'completed', completed: 0 }] }); + expect(await records(other.guid)).toEqual([ + expect.objectContaining({ kills: 40, highest_round: 4, rounds_survived: 3, completed: 0 }) + ]); + expect(await records()).toEqual([ + expect.objectContaining({ kills: 100, highest_round: 5, rounds_survived: 4, completed: 1 }) + ]); + expect(getLeaderboard()).toMatchObject({ + total: 2, + summary: { matches: 1, players: 2, kills: 140 } + }); + expect(getPlayerProfile(other.guid)).toMatchObject({ + stats: { kills: 40, highestRound: 4, roundsSurvived: 3, gamesPlayed: 1, gamesCompleted: 0 }, + matches: [{ status: 'completed', completed: 0 }] + }); }); it.each([ @@ -288,19 +515,37 @@ it.each([ const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const board = getLeaderboard({ game }); - expect(board.entries.find(entry => entry.guid === departed.guid)).toMatchObject({ - gamesPlayed: 0, gamesCompleted: 0, kills: 15, highestRound: 5, - averageKills: null, averageRounds: null, completionRate: null + expect(board.entries.find((entry) => entry.guid === departed.guid)).toMatchObject({ + gamesPlayed: 0, + gamesCompleted: 0, + kills: 15, + highestRound: 5, + averageKills: null, + averageRounds: null, + completionRate: null + }); + expect(board.entries.find((entry) => entry.guid === advanced.guid)).toMatchObject({ + gamesPlayed: 1, + gamesCompleted: 0 + }); + expect(board.entries.find((entry) => entry.guid === finished.guid)).toMatchObject({ + gamesPlayed: 1, + gamesCompleted: 1 }); - expect(board.entries.find(entry => entry.guid === advanced.guid)).toMatchObject({ gamesPlayed: 1, gamesCompleted: 0 }); - expect(board.entries.find(entry => entry.guid === finished.guid)).toMatchObject({ gamesPlayed: 1, gamesCompleted: 1 }); const profile = getPlayerProfile(departed.guid)!; - expect(profile).toMatchObject({ stats: { gamesPlayed: 0, kills: 15 }, - maps: [{ gamesPlayed: 0, kills: 15 }], matchesTotal: 1, - matches: [{ kills: 15, completed: 0 }], records: { bestKills: { value: 15 } } }); - expect(getLeaderboard({ game, map: profile.maps[0].key, metric: 'gamesPlayed' }).entries) - .toContainEqual(expect.objectContaining({ guid: departed.guid, gamesPlayed: 0 })); - expect(getPlayerProfile(departed.guid, 0, { game, map: profile.maps[0].key })?.matchesTotal).toBe(1); + expect(profile).toMatchObject({ + stats: { gamesPlayed: 0, kills: 15 }, + maps: [{ gamesPlayed: 0, kills: 15 }], + matchesTotal: 1, + matches: [{ kills: 15, completed: 0 }], + records: { bestKills: { value: 15 } } + }); + expect( + getLeaderboard({ game, map: profile.maps[0].key, metric: 'gamesPlayed' }).entries + ).toContainEqual(expect.objectContaining({ guid: departed.guid, gamesPlayed: 0 })); + expect( + getPlayerProfile(departed.guid, 0, { game, map: profile.maps[0].key })?.matchesTotal + ).toBe(1); }); it('starts separate matches for game starts, map changes, round resets, and new rounds after an end', async () => { @@ -312,10 +557,14 @@ it('starts separate matches for game starts, map changes, round resets, and new await report('round_change', 1, [player(undefined, { kills: 10 })], server, 'zombie_moon'); await report('game_end', 1, [player(undefined, { kills: 15 })], server, 'zombie_moon'); await report('round_change', 1, [player(undefined, { kills: 5 })], server, 'zombie_moon'); - expect((await matches()).map(row => [row.status, row.highest_round])).toEqual([ - ['interrupted', 5], ['interrupted', 2], ['interrupted', 3], ['completed', 1], ['active', 1] + expect((await matches()).map((row) => [row.status, row.highest_round])).toEqual([ + ['interrupted', 5], + ['interrupted', 2], + ['interrupted', 3], + ['completed', 1], + ['active', 1] ]); - expect((await records()).map(row => row.kills)).toEqual([50, 20, 30, 15, 5]); + expect((await records()).map((row) => row.kills)).toEqual([50, 20, 30, 15, 5]); }); it('deduplicates consecutive start and end retries even with new receipt timestamps and reordered players', async () => { @@ -326,15 +575,24 @@ it('deduplicates consecutive start and end retries even with new receipt timesta await report('game_end', 3, [player(), other]); await report('game_end', 3, [other, player()]); expect(await matches()).toHaveLength(1); - expect(await records()).toEqual([expect.objectContaining({ rounds_survived: 2, completed: 1 })]); + expect(await records()).toEqual([ + expect.objectContaining({ rounds_survived: 2, completed: 1 }) + ]); }); it('rejects a GUID repeated in the same telemetry roster before it can double-count participation', async () => { const { parseGameEvent } = await import('../../src/lib/telemetry.js'); const snapshot = player(undefined, { kills: 100, headshots: 50 }); for (const event of ['game_start', 'round_change', 'game_end']) { - expect(() => parseGameEvent({ event, server, map: 'zombie_theater', round: 5, players: [snapshot, snapshot] })) - .toThrow(/guid must be unique/); + expect(() => + parseGameEvent({ + event, + server, + map: 'zombie_theater', + round: 5, + players: [snapshot, snapshot] + }) + ).toThrow(/guid must be unique/); } }); @@ -342,10 +600,10 @@ it('keeps T6 location variants distinct when they share the same engine map', as await report('game_end', 8, [player()], 't6_zombie_town_1', 'zm_transit'); await report('game_end', 12, [player()], 't6_zombie_farm_1', 'zm_transit'); const rows = await matches(); - expect(rows.map(row => row.game)).toEqual(['t6', 't6']); - expect(rows.map(row => row.map)).toEqual(['zm_transit', 'zm_transit']); - expect(new Set(rows.map(row => row.map_key)).size).toBe(2); - expect(rows.map(row => row.map_name)).toEqual(['Town Survival', 'Farm Survival']); + expect(rows.map((row) => row.game)).toEqual(['t6', 't6']); + expect(rows.map((row) => row.map)).toEqual(['zm_transit', 'zm_transit']); + expect(new Set(rows.map((row) => row.map_key)).size).toBe(2); + expect(rows.map((row) => row.map_name)).toEqual(['Town Survival', 'Farm Survival']); }); it('retains historical records across app restarts and server removal', async () => { @@ -356,29 +614,47 @@ it('retains historical records across app restarts and server removal', async () await report('game_end', 5, [player(undefined, { kills: 150 })]); db().prepare('DELETE FROM servers WHERE name = ?').run(server); closeDb(); - expect(await records()).toEqual([expect.objectContaining({ kills: 150, rounds_survived: 4, completed: 1 })]); + expect(await records()).toEqual([ + expect.objectContaining({ kills: 150, rounds_survived: 4, completed: 1 }) + ]); expect(await matches()).toHaveLength(1); }); it('backfills valid saved telemetry once while skipping malformed and legacy snapshots', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const receivedAt = '2026-09-18T20:01:39.729Z'; - const saved = { event: 'round_change', server, map: 'zombie_theater', round: 8, - players: [player(undefined, { kills: 111, headshots: 61 })], receivedAt }; - const insert = db().prepare('INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)'); + const saved = { + event: 'round_change', + server, + map: 'zombie_theater', + round: 8, + players: [player(undefined, { kills: 111, headshots: 61 })], + receivedAt + }; + const insert = db().prepare( + 'INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)' + ); insert.run(server, 8, 1, receivedAt, JSON.stringify(saved)); insert.run('t6_zombie_town_1', 99, 4, receivedAt, '{malformed'); insert.run('t6_zombie_farm_1', 99, 4, receivedAt, null); db().prepare("DELETE FROM settings WHERE key = 'leaderboard_v1'").run(); closeDb(); - expect(await matches()).toEqual([expect.objectContaining({ highest_round: 8, start_observed: 0 })]); - expect(await records()).toEqual([expect.objectContaining({ kills: 111, headshots: 61, rounds_survived: 0 })]); - expect(db().prepare("SELECT value FROM settings WHERE key = 'leaderboard_v1'").get()).toEqual({ value: '1' }); + expect(await matches()).toEqual([ + expect.objectContaining({ highest_round: 8, start_observed: 0 }) + ]); + expect(await records()).toEqual([ + expect.objectContaining({ kills: 111, headshots: 61, rounds_survived: 0 }) + ]); + expect(db().prepare("SELECT value FROM settings WHERE key = 'leaderboard_v1'").get()).toEqual({ + value: '1' + }); closeDb(); await report('round_change', 9, [player(undefined, { kills: 150, headshots: 80 })]); closeDb(); expect(await matches()).toHaveLength(1); - expect(await records()).toEqual([expect.objectContaining({ kills: 150, headshots: 80, rounds_survived: 1 })]); + expect(await records()).toEqual([ + expect.objectContaining({ kills: 150, headshots: 80, rounds_survived: 1 }) + ]); }); it('keeps deleted match identities and starts a separate match when the server name is reused', async () => { @@ -395,31 +671,48 @@ it('keeps deleted match identities and starts a separate match when the server n closeDb(); const history = serverMatchHistory(server).matches; expect(history).toHaveLength(2); - expect(history.find(match => match.id === original.id)).toMatchObject({ - server, server_deleted_at: expect.any(String), status: 'interrupted' + expect(history.find((match) => match.id === original.id)).toMatchObject({ + server, + server_deleted_at: expect.any(String), + status: 'interrupted' + }); + expect(history.find((match) => match.id !== original.id)).toMatchObject({ + server, + server_deleted_at: null, + status: 'active' }); - expect(history.find(match => match.id !== original.id)).toMatchObject({ server, server_deleted_at: null, status: 'active' }); expect(getPlayerProfile(player().guid)).toBeNull(); await report('game_end', 8, [player(undefined, { kills: 120 })]); const profile = getPlayerProfile(player().guid)!; expect(profile.stats).toMatchObject({ gamesPlayed: 1, kills: 120 }); - expect(profile.matches).toEqual([expect.objectContaining({ id: history.find(match => match.id !== original.id)!.id, - server_deleted_at: null, status: 'completed' })]); + expect(profile.matches).toEqual([ + expect.objectContaining({ + id: history.find((match) => match.id !== original.id)!.id, + server_deleted_at: null, + status: 'completed' + }) + ]); }); it('does not fabricate players for an empty match', async () => { await report('game_start', 0, []); await report('game_end', 0, []); expect(await records()).toEqual([]); - expect(await matches()).toEqual([expect.objectContaining({ highest_round: 0, status: 'completed' })]); + expect(await matches()).toEqual([ + expect.objectContaining({ highest_round: 0, status: 'completed' }) + ]); }); it('rolls back history if writing the latest scoreboard fails', async () => { const { db } = await import('../../src/lib/server/db/index.js'); db().exec(`CREATE TRIGGER reject_scoreboard BEFORE INSERT ON server_telemetry BEGIN SELECT RAISE(ABORT, 'scoreboard write failed'); END`); - await expect(report('round_change', 8, [player(undefined, { kills: 111 })])).rejects.toThrow('scoreboard write failed'); + await expect(report('round_change', 8, [player(undefined, { kills: 111 })])).rejects.toThrow( + 'scoreboard write failed' + ); expect(await records()).toEqual([]); expect(await matches()).toEqual([]); - expect(db().prepare('SELECT COUNT(*) AS count FROM leaderboard_players').get()).toEqual({ count: 0 }); + expect(db().prepare('SELECT COUNT(*) AS count FROM leaderboard_players').get()).toEqual({ + count: 0 + }); }); diff --git a/tests/unit/leaderboard-navigation.test.ts b/tests/unit/leaderboard-navigation.test.ts index 78b9726..1e810e5 100644 --- a/tests/unit/leaderboard-navigation.test.ts +++ b/tests/unit/leaderboard-navigation.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { leaderboardReturnHref, playerContextHref, profileViewHref } from '../../src/lib/leaderboard-navigation.js'; +import { + leaderboardReturnHref, + playerContextHref, + profileViewHref +} from '../../src/lib/leaderboard-navigation.js'; describe('leaderboard profile navigation', () => { it('preserves leaderboard scope, search and page while starting profile history on its first page', () => { @@ -15,7 +19,9 @@ describe('leaderboard profile navigation', () => { }); it('keeps profile filters and return context through pagination and resets history pages on filter changes', () => { - const current = new URL('http://localhost/leaderboard/players/101?returnTo=%2Fleaderboard%3Fq%3DAce&game=t5&map=t5%3Atheater&metric=highestRound&offset=25'); + const current = new URL( + 'http://localhost/leaderboard/players/101?returnTo=%2Fleaderboard%3Fq%3DAce&game=t5&map=t5%3Atheater&metric=highestRound&offset=25' + ); const next = new URL(profileViewHref(current, { offset: '50' }), current); expect(next.searchParams.get('offset')).toBe('50'); expect(next.searchParams.get('returnTo')).toBe('/leaderboard?q=Ace'); @@ -39,10 +45,20 @@ describe('leaderboard profile navigation', () => { }); it('restricts supplied return destinations to the local leaderboard', () => { - for (const value of [null, '', 'https://example.com/leaderboard', '//example.com/leaderboard', '/leaderboard/players/101', '/leaderboard/../settings', '/leaderboard\\@example.com']) { + for (const value of [ + null, + '', + 'https://example.com/leaderboard', + '//example.com/leaderboard', + '/leaderboard/players/101', + '/leaderboard/../settings', + '/leaderboard\\@example.com' + ]) { expect(leaderboardReturnHref(value)).toBe('/leaderboard'); } - expect(leaderboardReturnHref('/leaderboard?q=A%26B&unrelated=drop#fragment')).toBe('/leaderboard?q=A%26B'); + expect(leaderboardReturnHref('/leaderboard?q=A%26B&unrelated=drop#fragment')).toBe( + '/leaderboard?q=A%26B' + ); }); it('uses the displayed fallback scope when a shared leaderboard URL contains unknown filters', () => { diff --git a/tests/unit/leaderboard-page-load.test.ts b/tests/unit/leaderboard-page-load.test.ts index b1778fb..74cc172 100644 --- a/tests/unit/leaderboard-page-load.test.ts +++ b/tests/unit/leaderboard-page-load.test.ts @@ -2,34 +2,53 @@ import { beforeEach, expect, it, vi } from 'vitest'; import { GET } from '../../src/routes/api/leaderboard/+server.js'; import { load } from '../../src/routes/leaderboard/+page.server.js'; vi.mock('../../src/routes/api/leaderboard/+server.js', () => ({ GET: vi.fn() })); -beforeEach(() => { vi.clearAllMocks(); }); +beforeEach(() => { + vi.clearAllMocks(); +}); it('requests the same 25-player page on initial load and preserves filters without mutating the page URL', async () => { - vi.mocked(GET).mockResolvedValueOnce(new Response(JSON.stringify({ entries: [], limit: 25, offset: 25 }))); - const url = new URL('http://localhost/leaderboard?game=t5&map=theater&players=2&metric=kills&q=123&offset=25&limit=100'); + vi.mocked(GET).mockResolvedValueOnce( + new Response(JSON.stringify({ entries: [], limit: 25, offset: 25 })) + ); + const url = new URL( + 'http://localhost/leaderboard?game=t5&map=theater&players=2&metric=kills&q=123&offset=25&limit=100' + ); const original = url.href; const data = await load({ url }); const requested = vi.mocked(GET).mock.calls[0][0].url; expect(requested.searchParams.get('limit')).toBe('25'); - for (const key of ['game', 'map', 'players', 'metric', 'q', 'offset']) expect(requested.searchParams.get(key)).toBe(url.searchParams.get(key)); + for (const key of ['game', 'map', 'players', 'metric', 'q', 'offset']) + expect(requested.searchParams.get(key)).toBe(url.searchParams.get(key)); expect(url.href).toBe(original); expect(data.leaderboard.limit).toBe(25); }); -it.each(['', '?game=t6&players=2&metric=kills&q=Cherry&offset=25'])('defaults to All maps without redirecting or changing other filters: %s', async query => { - vi.mocked(GET).mockResolvedValueOnce(new Response(JSON.stringify({ entries: [], maps: [{ key: 't6:town' }], limit: 25 }))); - const url = new URL(`http://localhost/leaderboard${query}`); - const original = url.href; - expect(await load({ url })).toMatchObject({ leaderboard: { entries: [], limit: 25 } }); - const requested = vi.mocked(GET).mock.calls[0][0].url; - expect(requested.searchParams.has('map')).toBe(false); - for (const key of ['game', 'players', 'metric', 'q', 'offset']) expect(requested.searchParams.get(key)).toBe(url.searchParams.get(key)); - expect(url.href).toBe(original); -}); +it.each(['', '?game=t6&players=2&metric=kills&q=Cherry&offset=25'])( + 'defaults to All maps without redirecting or changing other filters: %s', + async (query) => { + vi.mocked(GET).mockResolvedValueOnce( + new Response(JSON.stringify({ entries: [], maps: [{ key: 't6:town' }], limit: 25 })) + ); + const url = new URL(`http://localhost/leaderboard${query}`); + const original = url.href; + expect(await load({ url })).toMatchObject({ leaderboard: { entries: [], limit: 25 } }); + const requested = vi.mocked(GET).mock.calls[0][0].url; + expect(requested.searchParams.has('map')).toBe(false); + for (const key of ['game', 'players', 'metric', 'q', 'offset']) + expect(requested.searchParams.get(key)).toBe(url.searchParams.get(key)); + expect(url.href).toBe(original); + } +); it('honors explicit All maps and handles an empty recorded-map catalog without redirecting', async () => { - vi.mocked(GET).mockResolvedValue(new Response(JSON.stringify({ entries: [], maps: [], limit: 25 }))); + vi.mocked(GET).mockResolvedValue( + new Response(JSON.stringify({ entries: [], maps: [], limit: 25 })) + ); await load({ url: new URL('http://localhost/leaderboard?map=&players=3') }); expect(vi.mocked(GET).mock.calls[0][0].url.searchParams.get('map')).toBe(''); - vi.mocked(GET).mockResolvedValueOnce(new Response(JSON.stringify({ entries: [], maps: [], limit: 25 }))); - expect(await load({ url: new URL('http://localhost/leaderboard') })).toMatchObject({ leaderboard: { entries: [], maps: [] } }); + vi.mocked(GET).mockResolvedValueOnce( + new Response(JSON.stringify({ entries: [], maps: [], limit: 25 })) + ); + expect(await load({ url: new URL('http://localhost/leaderboard') })).toMatchObject({ + leaderboard: { entries: [], maps: [] } + }); }); diff --git a/tests/unit/leaderboard-players.test.ts b/tests/unit/leaderboard-players.test.ts index c444725..9298d47 100644 --- a/tests/unit/leaderboard-players.test.ts +++ b/tests/unit/leaderboard-players.test.ts @@ -3,46 +3,95 @@ import type Database from 'better-sqlite3'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); -function seed(handle: Database.Database, id: number, players: number | null, options: { - guid?: string; round?: number; game?: string; map?: string; mapName?: string; status?: string; hasStats?: number; -} = {}) { - const { guid = 'ace', round = 20, game = 't5', map = 'theater', mapName = 'Kino Der Toten', status = 'completed', hasStats = 1 } = options; +function seed( + handle: Database.Database, + id: number, + players: number | null, + options: { + guid?: string; + round?: number; + game?: string; + map?: string; + mapName?: string; + status?: string; + hasStats?: number; + } = {} +) { + const { + guid = 'ace', + round = 20, + game = 't5', + map = 'theater', + mapName = 'Kino Der Toten', + status = 'completed', + hasStats = 1 + } = options; const timestamp = new Date(Date.UTC(2026, 9, 1, 0, id)).toISOString(); - handle.prepare(`INSERT INTO leaderboard_matches + handle + .prepare( + `INSERT INTO leaderboard_matches (id,server,game,map,map_key,map_name,started_at,ended_at,last_received_at,status,start_players,highest_round,last_event,last_fingerprint) - VALUES (?,'removed-server',?,?,?,?,?,?,?,?,?,?,'game_end','fixture')`) - .run(id, game, map, `${game}:${map}`, mapName, timestamp, timestamp, timestamp, status, players, round); - handle.prepare(`INSERT OR IGNORE INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?)`) + VALUES (?,'removed-server',?,?,?,?,?,?,?,?,?,?,'game_end','fixture')` + ) + .run( + id, + game, + map, + `${game}:${map}`, + mapName, + timestamp, + timestamp, + timestamp, + status, + players, + round + ); + handle + .prepare( + `INSERT OR IGNORE INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?)` + ) .run(game, guid, guid, timestamp, timestamp); - handle.prepare(`INSERT INTO leaderboard_match_players + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id,guid,highest_round,kills,best_score,rounds_survived,has_stats,completed,first_seen_at,last_seen_at) - VALUES(?,?,?,100,1000,?, ?,1,?,?)`).run(id, guid, round, round, hasStats, timestamp, timestamp); + VALUES(?,?,?,100,1000,?, ?,1,?,?)` + ) + .run(id, guid, round, round, hasStats, timestamp, timestamp); } describe('leaderboard starting-player scopes', () => { it('separates each squad and its cache, with unknown starts available only in All players', async () => { const { db } = await import('../../src/lib/server/db/index.js'); seed(db(), 1, null); - for (const players of [1, 2, 3, 4] as const) seed(db(), players + 1, players, { round: players < 3 ? 20 : players * 10 }); + for (const players of [1, 2, 3, 4] as const) + seed(db(), players + 1, players, { round: players < 3 ? 20 : players * 10 }); seed(db(), 6, 1, { guid: 'blake', round: 30 }); seed(db(), 7, 2, { guid: 'blake', round: 10 }); seed(db(), 8, null, { guid: 'legacy', round: 99 }); - const { getLeaderboard, getPlayerLeaderboardEntry } = await import('../../src/lib/server/fleet/leaderboard.js'); + const { getLeaderboard, getPlayerLeaderboardEntry } = + await import('../../src/lib/server/fleet/leaderboard.js'); const all = getLeaderboard(); expect(all).toMatchObject({ total: 3, summary: { matches: 8, highestRound: 99 } }); for (const players of [1, 2, 3, 4, 2, 1] as const) { const board = getLeaderboard({ players }); expect(board.summary.matches).toBe(players < 3 ? 2 : 1); - expect(board.entries.some(entry => entry.guid === 'legacy')).toBe(false); - const ace = board.entries.find(entry => entry.guid === 'ace')!; + expect(board.entries.some((entry) => entry.guid === 'legacy')).toBe(false); + const ace = board.entries.find((entry) => entry.guid === 'ace')!; expect(ace.gamesPlayed).toBe(1); expect(ace.rank).toBe(players === 1 ? 2 : 1); // All three equal-valued records must come from this squad, not // the earlier unknown-start or solo match with matching values. - for (const record of Object.values(ace.records)) expect(record.matchId).toBe(players + 1); + for (const record of Object.values(ace.records)) + expect(record.matchId).toBe(players + 1); expect(getPlayerLeaderboardEntry('ace', { players })).toEqual(ace); } expect(getLeaderboard()).toEqual(all); @@ -51,36 +100,75 @@ describe('leaderboard starting-player scopes', () => { it('filters profile history and rank while keeping explicitly lifetime records intact', async () => { const { db } = await import('../../src/lib/server/db/index.js'); for (let id = 1; id <= 30; id++) seed(db(), id, 2); - seed(db(), 31, 1, { round: 40 }); seed(db(), 32, null, { round: 99 }); + seed(db(), 31, 1, { round: 40 }); + seed(db(), 32, null, { round: 99 }); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const lifetime = getPlayerProfile('ace')!; const first = getPlayerProfile('ace', 0, { players: 2 })!; const last = getPlayerProfile('ace', 999, { players: 2, map: 't5:theater', game: 't5' })!; - expect(first).toMatchObject({ matchesTotal: 30, offset: 0 }); expect(first.matches).toHaveLength(25); - expect(last).toMatchObject({ matchesTotal: 30, offset: 25 }); expect(last.matches).toHaveLength(5); - expect(new Set([...first.matches, ...last.matches].map(match => match.id))).toEqual(new Set(Array.from({ length: 30 }, (_, i) => i + 1))); - expect(first.stats).toEqual(lifetime.stats); expect(first.records).toEqual(lifetime.records); expect(first.maps).toEqual(lifetime.maps); - expect(getPlayerProfile('ace', 25, { players: 4 })).toMatchObject({ matchesTotal: 0, matches: [], offset: 0, stats: lifetime.stats }); - const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - const scoped = await load({ params: { guid: 'ace' }, url: new URL('http://localhost/leaderboard/players/ace?players=2&map=t5%3Atheater') }); - expect(scoped).toMatchObject({ profile: { matchesTotal: 30, stats: { gamesPlayed: 32, highestRound: 99 } }, - scope: { players: '2', entry: { gamesPlayed: 30, highestRound: 20, records: { highestRound: { matchId: 1 } } } } }); + expect(first).toMatchObject({ matchesTotal: 30, offset: 0 }); + expect(first.matches).toHaveLength(25); + expect(last).toMatchObject({ matchesTotal: 30, offset: 25 }); + expect(last.matches).toHaveLength(5); + expect(new Set([...first.matches, ...last.matches].map((match) => match.id))).toEqual( + new Set(Array.from({ length: 30 }, (_, i) => i + 1)) + ); + expect(first.stats).toEqual(lifetime.stats); + expect(first.records).toEqual(lifetime.records); + expect(first.maps).toEqual(lifetime.maps); + expect(getPlayerProfile('ace', 25, { players: 4 })).toMatchObject({ + matchesTotal: 0, + matches: [], + offset: 0, + stats: lifetime.stats + }); + const { load } = + await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); + const scoped = await load({ + params: { guid: 'ace' }, + url: new URL('http://localhost/leaderboard/players/ace?players=2&map=t5%3Atheater') + }); + expect(scoped).toMatchObject({ + profile: { matchesTotal: 30, stats: { gamesPlayed: 32, highestRound: 99 } }, + scope: { + players: '2', + entry: { + gamesPlayed: 30, + highestRound: 20, + records: { highestRound: { matchId: 1 } } + } + } + }); }); it('keeps API absent-map searches global while filtering all contributions by the selected squad', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - seed(db(), 1, 2); seed(db(), 2, 2, { map: 'moon', mapName: 'Moon' }); seed(db(), 3, null, { round: 99 }); + seed(db(), 1, 2); + seed(db(), 2, 2, { map: 'moon', mapName: 'Moon' }); + seed(db(), 3, null, { round: 99 }); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - expect(await (await GET({ url: new URL('http://localhost/api/leaderboard?players=2') })).json()) - .toMatchObject({ summary: { matches: 2, highestRound: 20 }, entries: [{ gamesPlayed: 2 }] }); - expect(await (await GET({ url: new URL('http://localhost/api/leaderboard') })).json()) - .toMatchObject({ summary: { matches: 3, highestRound: 99 } }); + expect( + await (await GET({ url: new URL('http://localhost/api/leaderboard?players=2') })).json() + ).toMatchObject({ + summary: { matches: 2, highestRound: 20 }, + entries: [{ gamesPlayed: 2 }] + }); + expect( + await (await GET({ url: new URL('http://localhost/api/leaderboard') })).json() + ).toMatchObject({ summary: { matches: 3, highestRound: 99 } }); for (const players of ['0', '5', '', '2.0', '-1', 'two', '01']) { - const response = await GET({ url: new URL(`http://localhost/api/leaderboard?players=${players}`) }); + const response = await GET({ + url: new URL(`http://localhost/api/leaderboard?players=${players}`) + }); expect(response.status).toBe(400); } - const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - await expect(load({ params: { guid: 'ace' }, url: new URL('http://localhost/leaderboard/players/ace?players=5') })) - .rejects.toMatchObject({ status: 400 }); + const { load } = + await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); + await expect( + load({ + params: { guid: 'ace' }, + url: new URL('http://localhost/leaderboard/players/ace?players=5') + }) + ).rejects.toMatchObject({ status: 400 }); }); }); diff --git a/tests/unit/leaderboard-query.test.ts b/tests/unit/leaderboard-query.test.ts index a0b006b..48e3323 100644 --- a/tests/unit/leaderboard-query.test.ts +++ b/tests/unit/leaderboard-query.test.ts @@ -3,8 +3,13 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import type { LeaderboardResponse } from '../../src/lib/leaderboard.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); async function seedRecords() { const { db } = await import('../../src/lib/server/db/index.js'); @@ -12,16 +17,47 @@ async function seedRecords() { const match = handle.prepare(`INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint) VALUES (?, 'removed-server', ?, ?, ?, ?, ?, ?, ?, ?, 'round_change', 'test')`); - const player = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + const player = + handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) VALUES (?, ?, ?, '2026-09-01T00:00:00.000Z', '2026-09-05T00:00:00.000Z')`); const record = handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, kills, highest_round, rounds_survived, headshots, revives, downs, best_score, completed, has_stats, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '2026-09-01T00:00:00.000Z', ?)`); const iso = (day: number) => `2026-09-0${day}T00:00:00.000Z`; - match.run(1, 't5', 'zombie_theater', 't5:theater', 'Kino Der Toten', iso(1), iso(2), 'completed', 10); - match.run(2, 't5', 'zombie_theater', 't5:theater', 'Kino Der Toten', iso(2), iso(3), 'completed', 8); - match.run(3, 't5', 'zombie_coast', 't5:coast', 'Call of the Dead', iso(3), iso(4), 'completed', 4); + match.run( + 1, + 't5', + 'zombie_theater', + 't5:theater', + 'Kino Der Toten', + iso(1), + iso(2), + 'completed', + 10 + ); + match.run( + 2, + 't5', + 'zombie_theater', + 't5:theater', + 'Kino Der Toten', + iso(2), + iso(3), + 'completed', + 8 + ); + match.run( + 3, + 't5', + 'zombie_coast', + 't5:coast', + 'Call of the Dead', + iso(3), + iso(4), + 'completed', + 4 + ); match.run(4, 't6', 'zm_transit', 't6:town', 'Town Survival', iso(4), iso(5), 'completed', 20); match.run(5, 't6', 'zm_transit', 't6:farm', 'Farm Survival', iso(5), iso(6), 'completed', 15); match.run(6, 't5', 'custom_map', 't5:custom_map', 'Custom Map', iso(1), iso(7), 'completed', 2); @@ -50,30 +86,56 @@ describe('persisted leaderboard queries', () => { const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const { leaderboardMetrics } = await import('../../src/lib/leaderboard.js'); - const boards = leaderboardMetrics.map(({ key }) => getLeaderboard({ metric: key, limit: 2, offset: 1 })); + const boards = leaderboardMetrics.map(({ key }) => + getLeaderboard({ metric: key, limit: 2, offset: 1 }) + ); const profile = getPlayerProfile('101'); for (const [index, status] of ['active', 'interrupted'].entries()) { const id = index + 10; - handle.prepare(`INSERT INTO leaderboard_matches + handle + .prepare( + `INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint) VALUES (?, 'removed-server', 't5', ?, ?, 'Unfinished Map', - '2026-09-20', '2026-09-20', ?, 99, 'heartbeat', 'unfinished')`) + '2026-09-20', '2026-09-20', ?, 99, 'heartbeat', 'unfinished')` + ) .run(id, status, `t5:${status}`, status); - handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES ('t5', ?, ?, '2026-09-20', '2026-09-20')`).run(status, status); + handle + .prepare( + `INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES ('t5', ?, ?, '2026-09-20', '2026-09-20')` + ) + .run(status, status); for (const guid of ['101', status]) { - handle.prepare(`INSERT INTO leaderboard_match_players + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, highest_round, rounds_survived, kills, headshots, revives, downs, best_score, has_stats, completed, first_seen_at, last_seen_at) - VALUES (?, ?, 99, 99, 99999, 99999, 99999, 99999, 99999, 1, 1, '2026-09-20', '2026-09-20')`) + VALUES (?, ?, 99, 99, 99999, 99999, 99999, 99999, 99999, 1, 1, '2026-09-20', '2026-09-20')` + ) .run(id, guid); } expect(getPlayerProfile(status)).toBeNull(); - expect(getLeaderboard({ map: `t5:${status}` })).toMatchObject({ entries: [], total: 0, - summary: { players: 0, matches: 0, kills: 0, highestRound: 0, firstTrackedAt: null, updatedAt: null } }); + expect(getLeaderboard({ map: `t5:${status}` })).toMatchObject({ + entries: [], + total: 0, + summary: { + players: 0, + matches: 0, + kills: 0, + highestRound: 0, + firstTrackedAt: null, + updatedAt: null + } + }); } - expect(leaderboardMetrics.map(({ key }) => getLeaderboard({ metric: key, limit: 2, offset: 1 }))).toEqual(boards); + expect( + leaderboardMetrics.map(({ key }) => + getLeaderboard({ metric: key, limit: 2, offset: 1 }) + ) + ).toEqual(boards); expect(getPlayerProfile('101')).toEqual(profile); }); @@ -83,7 +145,9 @@ describe('persisted leaderboard queries', () => { const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); const { leaderboardMetrics } = await import('../../src/lib/leaderboard.js'); - const boards = leaderboardMetrics.map(({ key }) => getLeaderboard({ metric: key, limit: 2, offset: 1 })); + const boards = leaderboardMetrics.map(({ key }) => + getLeaderboard({ metric: key, limit: 2, offset: 1 }) + ); const profile = getPlayerProfile('101'); const history = serverMatchHistory(undefined, 2); const scopedHistory = serverMatchHistory('removed-server', 2); @@ -92,24 +156,45 @@ describe('persisted leaderboard queries', () => { for (const game of ['t5', 't6']) { for (const status of ['active', 'completed', 'interrupted']) { for (const round of [0, 1]) { - handle.prepare(`INSERT INTO leaderboard_matches + handle + .prepare( + `INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint) VALUES (?, 'removed-server', ?, 'reset_map', ?, 'Reset Map', - '2026-09-20', '2026-09-20', ?, ?, 'game_end', 'reset')`).run(id, game, `${game}:reset`, status, round); + '2026-09-20', '2026-09-20', ?, ?, 'game_end', 'reset')` + ) + .run(id, game, `${game}:reset`, status, round); // A historical record with large counters must not change rankings or bests. - handle.prepare(`INSERT INTO leaderboard_match_players + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, headshots, revives, downs, best_score, has_stats, completed, first_seen_at, last_seen_at) - VALUES (?, '101', ?, 99999, 99999, 99999, 99999, 99999, 1, 1, '2026-09-20', '2026-09-20')`) + VALUES (?, '101', ?, 99999, 99999, 99999, 99999, 99999, 1, 1, '2026-09-20', '2026-09-20')` + ) .run(id++, round); } } } - expect(leaderboardMetrics.map(({ key }) => getLeaderboard({ metric: key, limit: 2, offset: 1 }))).toEqual(boards); + expect( + leaderboardMetrics.map(({ key }) => + getLeaderboard({ metric: key, limit: 2, offset: 1 }) + ) + ).toEqual(boards); expect(getLeaderboard({ game: 't5', map: 't5:theater' })).toEqual(scopedBoard); - expect(getLeaderboard({ map: 't5:reset' })).toMatchObject({ entries: [], total: 0, - summary: { players: 0, matches: 0, kills: 0, highestRound: 0, firstTrackedAt: null, updatedAt: null } }); + expect(getLeaderboard({ map: 't5:reset' })).toMatchObject({ + entries: [], + total: 0, + summary: { + players: 0, + matches: 0, + kills: 0, + highestRound: 0, + firstTrackedAt: null, + updatedAt: null + } + }); expect(getPlayerProfile('101')).toEqual(profile); expect(serverMatchHistory(undefined, 2)).toEqual(history); expect(serverMatchHistory('removed-server', 2)).toEqual(scopedHistory); @@ -119,23 +204,51 @@ describe('persisted leaderboard queries', () => { await seedRecords(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const board = getLeaderboard(); - expect(board.entries.filter(entry => entry.guid === '101')).toEqual([{ - rank: 1, guid: '101', name: 'Ace', highestRound: 20, kills: 770, - roundsSurvived: 38, headshots: 320, revives: 8, downs: 4, - gamesPlayed: 4, gamesCompleted: 3, bestKills: 500, bestScore: 5000, - headshotRate: 320 / 770, killsPerDown: 770 / 4, averageKills: 770 / 4, - averageRounds: 38 / 4, completionRate: 3 / 4, - lastSeen: '2026-09-05T00:00:00.000Z', records: { - highestRound: expect.objectContaining({ matchId: 4, value: 20 }), - bestKills: expect.objectContaining({ matchId: 4, value: 500 }), - bestScore: expect.objectContaining({ matchId: 4, value: 5000 }) + expect(board.entries.filter((entry) => entry.guid === '101')).toEqual([ + { + rank: 1, + guid: '101', + name: 'Ace', + highestRound: 20, + kills: 770, + roundsSurvived: 38, + headshots: 320, + revives: 8, + downs: 4, + gamesPlayed: 4, + gamesCompleted: 3, + bestKills: 500, + bestScore: 5000, + headshotRate: 320 / 770, + killsPerDown: 770 / 4, + averageKills: 770 / 4, + averageRounds: 38 / 4, + completionRate: 3 / 4, + lastSeen: '2026-09-05T00:00:00.000Z', + records: { + highestRound: expect.objectContaining({ matchId: 4, value: 20 }), + bestKills: expect.objectContaining({ matchId: 4, value: 500 }), + bestScore: expect.objectContaining({ matchId: 4, value: 5000 }) + } } - }]); + ]); expect(board.total).toBe(4); - expect(board.entries.map(entry => [entry.guid, entry.rank])).toEqual([['101', 1], ['505', 2], ['202', 3], ['303', 4]]); - expect(board.entries.some(entry => entry.guid === '404')).toBe(false); - expect(board.summary).toEqual({ players: 4, matches: 5, kills: 1140, roundsSurvived: 68, - highestRound: 20, firstTrackedAt: '2026-09-01T00:00:00.000Z', updatedAt: '2026-09-06T00:00:00.000Z' }); + expect(board.entries.map((entry) => [entry.guid, entry.rank])).toEqual([ + ['101', 1], + ['505', 2], + ['202', 3], + ['303', 4] + ]); + expect(board.entries.some((entry) => entry.guid === '404')).toBe(false); + expect(board.summary).toEqual({ + players: 4, + matches: 5, + kills: 1140, + roundsSurvived: 68, + highestRound: 20, + firstTrackedAt: '2026-09-01T00:00:00.000Z', + updatedAt: '2026-09-06T00:00:00.000Z' + }); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); expect(board.entries[0]).toMatchObject(getPlayerProfile('101')!.stats); }); @@ -144,77 +257,161 @@ describe('persisted leaderboard queries', () => { await seedRecords(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const t5 = getLeaderboard({ game: 't5' }); - expect(t5.entries.find(entry => entry.guid === '101')).toEqual({ - rank: 1, guid: '101', name: 'Ace', highestRound: 10, kills: 270, - roundsSurvived: 19, headshots: 120, revives: 4, downs: 3, gamesPlayed: 3, - headshotRate: 120 / 270, killsPerDown: 270 / 3, averageKills: 270 / 3, - averageRounds: 19 / 3, completionRate: 2 / 3, - gamesCompleted: 2, bestKills: 150, bestScore: 3000, lastSeen: '2026-09-04T00:00:00.000Z', records: { + expect(t5.entries.find((entry) => entry.guid === '101')).toEqual({ + rank: 1, + guid: '101', + name: 'Ace', + highestRound: 10, + kills: 270, + roundsSurvived: 19, + headshots: 120, + revives: 4, + downs: 3, + gamesPlayed: 3, + headshotRate: 120 / 270, + killsPerDown: 270 / 3, + averageKills: 270 / 3, + averageRounds: 19 / 3, + completionRate: 2 / 3, + gamesCompleted: 2, + bestKills: 150, + bestScore: 3000, + lastSeen: '2026-09-04T00:00:00.000Z', + records: { highestRound: expect.objectContaining({ matchId: 1, value: 10 }), bestKills: expect.objectContaining({ matchId: 2, value: 150 }), bestScore: expect.objectContaining({ matchId: 2, value: 3000 }) } }); - expect(t5).toMatchObject({ total: 3, summary: { players: 3, matches: 3, kills: 440, roundsSurvived: 35 } }); + expect(t5).toMatchObject({ + total: 3, + summary: { players: 3, matches: 3, kills: 440, roundsSurvived: 35 } + }); const t6 = getLeaderboard({ game: 't6' }); - expect(t6.entries.find(entry => entry.guid === '101')).toEqual({ - rank: 1, guid: '101', name: 'Ace', highestRound: 20, kills: 500, - roundsSurvived: 19, headshots: 200, revives: 4, downs: 1, gamesPlayed: 1, - headshotRate: 200 / 500, killsPerDown: 500, averageKills: 500, - averageRounds: 19, completionRate: 1, - gamesCompleted: 1, bestKills: 500, bestScore: 5000, lastSeen: '2026-09-05T00:00:00.000Z', records: { + expect(t6.entries.find((entry) => entry.guid === '101')).toEqual({ + rank: 1, + guid: '101', + name: 'Ace', + highestRound: 20, + kills: 500, + roundsSurvived: 19, + headshots: 200, + revives: 4, + downs: 1, + gamesPlayed: 1, + headshotRate: 200 / 500, + killsPerDown: 500, + averageKills: 500, + averageRounds: 19, + completionRate: 1, + gamesCompleted: 1, + bestKills: 500, + bestScore: 5000, + lastSeen: '2026-09-05T00:00:00.000Z', + records: { highestRound: expect.objectContaining({ matchId: 4, value: 20 }), bestKills: expect.objectContaining({ matchId: 4, value: 500 }), bestScore: expect.objectContaining({ matchId: 4, value: 5000 }) } }); - expect(t6).toMatchObject({ total: 2, summary: { players: 2, matches: 2, kills: 700, roundsSurvived: 33 } }); + expect(t6).toMatchObject({ + total: 2, + summary: { players: 2, matches: 2, kills: 700, roundsSurvived: 33 } + }); }); it('filters exact map variants and retains historical and custom map options independently of selection', async () => { await seedRecords(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const theater = getLeaderboard({ game: 't5', map: 't5:theater', metric: 'kills' }); - expect(theater.entries.map(entry => [entry.name, entry.kills])).toEqual([['Ace', 250], ['Blake', 100], ['Casey', 70]]); - expect(theater.summary).toMatchObject({ players: 3, matches: 2, kills: 420, roundsSurvived: 32, highestRound: 10 }); - expect(theater.maps.map(map => map.key)).toEqual(['t5:coast', 't5:custom_map', 't5:theater']); + expect(theater.entries.map((entry) => [entry.name, entry.kills])).toEqual([ + ['Ace', 250], + ['Blake', 100], + ['Casey', 70] + ]); + expect(theater.summary).toMatchObject({ + players: 3, + matches: 2, + kills: 420, + roundsSurvived: 32, + highestRound: 10 + }); + expect(theater.maps.map((map) => map.key)).toEqual([ + 't5:coast', + 't5:custom_map', + 't5:theater' + ]); const town = getLeaderboard({ map: 't6:town' }); - expect(town.entries.map(entry => [entry.guid, entry.name, entry.kills])).toEqual([['101', 'Ace', 500]]); - expect(getLeaderboard({ game: 't5', map: 't6:town' })).toMatchObject({ entries: [], total: 0, summary: { players: 0, kills: 0 } }); - expect(town.maps).toContainEqual({ key: 't6:farm', game: 't6', map: 'zm_transit', name: 'Farm Survival' }); + expect(town.entries.map((entry) => [entry.guid, entry.name, entry.kills])).toEqual([ + ['101', 'Ace', 500] + ]); + expect(getLeaderboard({ game: 't5', map: 't6:town' })).toMatchObject({ + entries: [], + total: 0, + summary: { players: 0, kills: 0 } + }); + expect(town.maps).toContainEqual({ + key: 't6:farm', + game: 't6', + map: 'zm_transit', + name: 'Farm Survival' + }); }); it('uses competition ranks with deterministic ties and searches after ranking before pagination', async () => { await seedRecords(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const board = getLeaderboard({ game: 't5' }); - expect(board.entries.map(entry => [entry.name, entry.rank])).toEqual([['Ace', 1], ['Blake', 1], ['Casey', 3]]); + expect(board.entries.map((entry) => [entry.name, entry.rank])).toEqual([ + ['Ace', 1], + ['Blake', 1], + ['Casey', 3] + ]); const search = getLeaderboard({ game: 't5', q: 'CASE', limit: 1 }); - expect(search.entries.map(entry => [entry.name, entry.rank])).toEqual([['Casey', 3]]); + expect(search.entries.map((entry) => [entry.name, entry.rank])).toEqual([['Casey', 3]]); expect(search.total).toBe(1); expect(search.summary).toEqual(board.summary); expect(getLeaderboard({ game: 't5', q: 'no matching player' })).toMatchObject({ - entries: [], total: 0, summary: board.summary, maps: board.maps + entries: [], + total: 0, + summary: board.summary, + maps: board.maps }); - expect(getLeaderboard({ game: 't5', metric: 'kills', limit: 1, offset: 1 }).entries[0]) - .toMatchObject({ name: 'Blake', rank: 2 }); + expect( + getLeaderboard({ game: 't5', metric: 'kills', limit: 1, offset: 1 }).entries[0] + ).toMatchObject({ name: 'Blake', rank: 2 }); expect(getLeaderboard({ q: '303' }).entries[0]).toMatchObject({ name: 'Casey', rank: 4 }); }); it('uses and searches the latest shared name regardless of game or map filters without merging namesakes', async () => { const handle = await seedRecords(); - handle.prepare("UPDATE leaderboard_players SET last_seen_at = '2026-09-09' WHERE game = 't6' AND guid = '101'").run(); + handle + .prepare( + "UPDATE leaderboard_players SET last_seen_at = '2026-09-09' WHERE game = 't6' AND guid = '101'" + ) + .run(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - for (const options of [{}, { game: 't5' }, { game: 't6' }, { map: 't5:theater' }, { game: 't6', map: 't6:town' }]) { + for (const options of [ + {}, + { game: 't5' }, + { game: 't6' }, + { map: 't5:theater' }, + { game: 't6', map: 't6:town' } + ]) { const board = getLeaderboard(options); - expect(board.entries.find(entry => entry.guid === '101')?.name).toBe('Other game'); - expect(getLeaderboard({ ...options, q: 'other GAME' }).entries.map(entry => entry.guid)).toEqual(['101']); + expect(board.entries.find((entry) => entry.guid === '101')?.name).toBe('Other game'); + expect( + getLeaderboard({ ...options, q: 'other GAME' }).entries.map((entry) => entry.guid) + ).toEqual(['101']); expect(getLeaderboard({ ...options, q: 'Ace' }).entries).toEqual([]); } - handle.prepare("UPDATE leaderboard_players SET name = 'Other game' WHERE guid = '202'").run(); + handle + .prepare("UPDATE leaderboard_players SET name = 'Other game' WHERE guid = '202'") + .run(); const namesakes = getLeaderboard({ q: 'Other game' }); - expect(namesakes.entries.map(entry => [entry.guid, entry.name, entry.rank])).toEqual([ - ['101', 'Other game', 1], ['202', 'Other game', 3] + expect(namesakes.entries.map((entry) => [entry.guid, entry.name, entry.rank])).toEqual([ + ['101', 'Other game', 1], + ['202', 'Other game', 3] ]); expect(namesakes).toMatchObject({ total: 2, summary: { players: 4 } }); }); @@ -224,13 +421,22 @@ describe('persisted leaderboard queries', () => { const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const first = getLeaderboard({ metric: 'kills', limit: 2 }); const second = getLeaderboard({ metric: 'kills', limit: 2, offset: 2 }); - expect(first.entries.map(entry => [entry.guid, entry.kills, entry.rank])).toEqual([['101', 770, 1], ['505', 200, 2]]); - expect(second.entries.map(entry => [entry.guid, entry.kills, entry.rank])).toEqual([['202', 100, 3], ['303', 70, 4]]); + expect(first.entries.map((entry) => [entry.guid, entry.kills, entry.rank])).toEqual([ + ['101', 770, 1], + ['505', 200, 2] + ]); + expect(second.entries.map((entry) => [entry.guid, entry.kills, entry.rank])).toEqual([ + ['202', 100, 3], + ['303', 70, 4] + ]); expect(first.total).toBe(4); expect(second.total).toBe(4); expect(second.summary).toEqual(first.summary); expect(getLeaderboard({ metric: 'kills', limit: 2, offset: 4 })).toMatchObject({ - entries: [], total: first.total, summary: first.summary, maps: first.maps + entries: [], + total: first.total, + summary: first.summary, + maps: first.maps }); }); @@ -241,49 +447,96 @@ describe('persisted leaderboard queries', () => { handle.prepare("DELETE FROM leaderboard_players WHERE guid = '101'").run(); const board = getLeaderboard(); expect(board.total).toBe(original.total - 1); - expect(board.entries.some(entry => entry.guid === '101')).toBe(false); + expect(board.entries.some((entry) => entry.guid === '101')).toBe(false); expect(board.summary).toEqual(original.summary); expect(board.maps).toEqual(original.maps); - expect(getLeaderboard({ q: '101' })).toMatchObject({ entries: [], total: 0, summary: original.summary }); + expect(getLeaderboard({ q: '101' })).toMatchObject({ + entries: [], + total: 0, + summary: original.summary + }); }); it('ignores unsupported games in identities, totals, rankings and map options', async () => { const handle = await seedRecords(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const original = getLeaderboard(); - handle.prepare(`INSERT INTO leaderboard_matches + handle + .prepare( + `INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint) - VALUES (7, 'removed-server', 't7', 'excluded', 't7:excluded', 'Excluded', '2020-01-01', '2030-01-01', 'completed', 99, 'game_end', 'test')`).run(); + VALUES (7, 'removed-server', 't7', 'excluded', 't7:excluded', 'Excluded', '2020-01-01', '2030-01-01', 'completed', 99, 'game_end', 'test')` + ) + .run(); for (const guid of ['101', 'unsupported-only']) { - handle.prepare(`INSERT INTO leaderboard_match_players + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, kills, highest_round, has_stats, first_seen_at, last_seen_at) - VALUES (7, ?, 99999, 99, 1, '2020-01-01', '2030-01-01')`).run(guid); - handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES ('t7', ?, 'Unsupported name', '2020-01-01', '2030-01-01')`).run(guid); + VALUES (7, ?, 99999, 99, 1, '2020-01-01', '2030-01-01')` + ) + .run(guid); + handle + .prepare( + `INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES ('t7', ?, 'Unsupported name', '2020-01-01', '2030-01-01')` + ) + .run(guid); } expect(getLeaderboard()).toEqual(original); expect(getLeaderboard({ q: 'Unsupported' }).entries).toEqual([]); - expect(getLeaderboard({ game: 't7' })).toMatchObject({ entries: [], maps: [], total: 0, - summary: { players: 0, matches: 0, kills: 0 } }); + expect(getLeaderboard({ game: 't7' })).toMatchObject({ + entries: [], + maps: [], + total: 0, + summary: { players: 0, matches: 0, kills: 0 } + }); }); it('treats SQL and LIKE punctuation as literal input and allows only known ranking columns', async () => { const handle = await seedRecords(); - handle.prepare("UPDATE leaderboard_players SET name = '100%_Blake' WHERE guid = '202'").run(); + handle + .prepare("UPDATE leaderboard_players SET name = '100%_Blake' WHERE guid = '202'") + .run(); const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard({ q: '%' }).entries.map(entry => entry.guid)).toEqual(['202']); + expect(getLeaderboard({ q: '%' }).entries.map((entry) => entry.guid)).toEqual(['202']); expect(getLeaderboard({ map: "' OR 1=1 --" }).entries).toEqual([]); expect(getLeaderboard({ q: "' OR 1=1 --" }).entries).toEqual([]); - expect(getLeaderboard({ metric: 'kills; DROP TABLE leaderboard_players' as never }).metric).toBe('highestRound'); - expect(getLeaderboard({ metric: 'downs', game: 't5' }).entries.map(entry => entry.downs)).toEqual([3, 2, 0]); + expect( + getLeaderboard({ metric: 'kills; DROP TABLE leaderboard_players' as never }).metric + ).toBe('highestRound'); + expect( + getLeaderboard({ metric: 'downs', game: 't5' }).entries.map((entry) => entry.downs) + ).toEqual([3, 2, 0]); }); it('returns an empty summary and bounds pagination for direct callers', async () => { const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard()).toEqual({ entries: [], maps: [], total: 0, limit: 50, offset: 0, metric: 'highestRound', - summary: { players: 0, matches: 0, kills: 0, roundsSurvived: 0, highestRound: 0, firstTrackedAt: null, updatedAt: null } }); - expect(getLeaderboard({ limit: 1000, offset: -1 })).toMatchObject({ limit: 100, offset: 0 }); - expect(getLeaderboard({ limit: NaN, offset: 2_000_000 })).toMatchObject({ limit: 50, offset: 1_000_000 }); + expect(getLeaderboard()).toEqual({ + entries: [], + maps: [], + total: 0, + limit: 50, + offset: 0, + metric: 'highestRound', + summary: { + players: 0, + matches: 0, + kills: 0, + roundsSurvived: 0, + highestRound: 0, + firstTrackedAt: null, + updatedAt: null + } + }); + expect(getLeaderboard({ limit: 1000, offset: -1 })).toMatchObject({ + limit: 100, + offset: 0 + }); + expect(getLeaderboard({ limit: NaN, offset: 2_000_000 })).toMatchObject({ + limit: 50, + offset: 1_000_000 + }); }); }); @@ -294,39 +547,102 @@ describe('player profiles', () => { const profile = getPlayerProfile('101')!; expect(profile).toMatchObject({ name: 'Ace', guid: '101', offset: 0, limit: 25 }); expect(profile).not.toHaveProperty('game'); - expect(profile.stats).toEqual({ highestRound: 20, kills: 770, roundsSurvived: 38, - headshots: 320, revives: 8, downs: 4, gamesPlayed: 4, gamesCompleted: 3, - bestKills: 500, bestScore: 5000 }); + expect(profile.stats).toEqual({ + highestRound: 20, + kills: 770, + roundsSurvived: 38, + headshots: 320, + revives: 8, + downs: 4, + gamesPlayed: 4, + gamesCompleted: 3, + bestKills: 500, + bestScore: 5000 + }); expect(profile.maps).toHaveLength(3); - expect(profile.maps[0]).toMatchObject({ game: 't6', key: 't6:town', highestRound: 20, kills: 500, gamesPlayed: 1, gamesCompleted: 1 }); - expect(profile.maps[1]).toMatchObject({ game: 't5', key: 't5:theater', highestRound: 10, kills: 250, gamesPlayed: 2, gamesCompleted: 1 }); - expect(profile.matches.map(match => [match.id, match.game])).toEqual([[4, 't6'], [3, 't5'], [2, 't5'], [1, 't5']]); - expect(profile.matches[2]).toMatchObject({ status: 'completed', completed: 0, server: 'removed-server', kills: 150 }); - expect(getPlayerProfile('505')).toMatchObject({ name: 'Eve', stats: { kills: 200, gamesPlayed: 1 }, - maps: [expect.objectContaining({ game: 't6' })], matches: [expect.objectContaining({ game: 't6' })] }); + expect(profile.maps[0]).toMatchObject({ + game: 't6', + key: 't6:town', + highestRound: 20, + kills: 500, + gamesPlayed: 1, + gamesCompleted: 1 + }); + expect(profile.maps[1]).toMatchObject({ + game: 't5', + key: 't5:theater', + highestRound: 10, + kills: 250, + gamesPlayed: 2, + gamesCompleted: 1 + }); + expect(profile.matches.map((match) => [match.id, match.game])).toEqual([ + [4, 't6'], + [3, 't5'], + [2, 't5'], + [1, 't5'] + ]); + expect(profile.matches[2]).toMatchObject({ + status: 'completed', + completed: 0, + server: 'removed-server', + kills: 150 + }); + expect(getPlayerProfile('505')).toMatchObject({ + name: 'Eve', + stats: { kills: 200, gamesPlayed: 1 }, + maps: [expect.objectContaining({ game: 't6' })], + matches: [expect.objectContaining({ game: 't6' })] + }); expect(getPlayerProfile('404')).toBeNull(); expect(getPlayerProfile("' OR 1=1 --")).toBeNull(); }); it('uses the latest reported name and the full identity date range across supported games', async () => { const handle = await seedRecords(); - handle.prepare(`UPDATE leaderboard_players SET first_seen_at = '2026-08-01', last_seen_at = '2026-09-07' - WHERE game = 't5' AND guid = '101'`).run(); - handle.prepare(`UPDATE leaderboard_players SET first_seen_at = '2026-07-01', last_seen_at = '2026-09-08' - WHERE game = 't6' AND guid = '101'`).run(); + handle + .prepare( + `UPDATE leaderboard_players SET first_seen_at = '2026-08-01', last_seen_at = '2026-09-07' + WHERE game = 't5' AND guid = '101'` + ) + .run(); + handle + .prepare( + `UPDATE leaderboard_players SET first_seen_at = '2026-07-01', last_seen_at = '2026-09-08' + WHERE game = 't6' AND guid = '101'` + ) + .run(); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); - expect(getPlayerProfile('101')).toMatchObject({ name: 'Other game', firstSeen: '2026-07-01', lastSeen: '2026-09-08' }); - handle.prepare("UPDATE leaderboard_players SET last_seen_at = '2026-09-08' WHERE game = 't5' AND guid = '101'").run(); + expect(getPlayerProfile('101')).toMatchObject({ + name: 'Other game', + firstSeen: '2026-07-01', + lastSeen: '2026-09-08' + }); + handle + .prepare( + "UPDATE leaderboard_players SET last_seen_at = '2026-09-08' WHERE game = 't5' AND guid = '101'" + ) + .run(); expect(getPlayerProfile('101')?.name).toBe('Ace'); - handle.prepare("UPDATE leaderboard_players SET last_seen_at = '2026-09-09' WHERE game = 't5' AND guid = '101'").run(); - expect(getPlayerProfile('101')).toMatchObject({ name: 'Ace', firstSeen: '2026-07-01', lastSeen: '2026-09-09' }); + handle + .prepare( + "UPDATE leaderboard_players SET last_seen_at = '2026-09-09' WHERE game = 't5' AND guid = '101'" + ) + .run(); + expect(getPlayerProfile('101')).toMatchObject({ + name: 'Ace', + firstSeen: '2026-07-01', + lastSeen: '2026-09-09' + }); }); it('keeps map records separate across games even when their stored keys are equal', async () => { const handle = await seedRecords(); - handle.prepare("UPDATE leaderboard_matches SET map_key = 'shared-map' WHERE id IN (1, 2, 4)").run(); + handle + .prepare("UPDATE leaderboard_matches SET map_key = 'shared-map' WHERE id IN (1, 2, 4)") + .run(); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); - const records = getPlayerProfile('101')!.maps.filter(map => map.key === 'shared-map'); + const records = getPlayerProfile('101')!.maps.filter((map) => map.key === 'shared-map'); expect(records).toEqual([ expect.objectContaining({ game: 't6', highestRound: 20, kills: 500, gamesPlayed: 1 }), expect.objectContaining({ game: 't5', highestRound: 10, kills: 250, gamesPlayed: 2 }) @@ -347,8 +663,12 @@ describe('player profiles', () => { record.run(10, '101', 99, 1); record.run(10, 'unsupported-only', 99, 1); for (const guid of ['101', 'unsupported-only']) { - handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES ('t7', ?, 'Unsupported name', '2020-01-01', '2030-01-01')`).run(guid); + handle + .prepare( + `INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES ('t7', ?, 'Unsupported name', '2020-01-01', '2030-01-01')` + ) + .run(guid); } match.run(11, 't5', 99); record.run(11, '101', 1, 1); @@ -363,12 +683,20 @@ describe('player profiles', () => { it('paginates one history across both games without changing lifetime totals and clamps invalid offsets', async () => { const handle = await seedRecords(); for (let id = 10; id < 40; id++) { - handle.prepare(`INSERT INTO leaderboard_matches + handle + .prepare( + `INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint) SELECT ?, server, game, map, map_key, map_name, started_at, last_received_at, status, highest_round, last_event, last_fingerprint - FROM leaderboard_matches WHERE id = ?`).run(id, id % 2 === 0 ? 1 : 4); - handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, has_stats, highest_round, first_seen_at, last_seen_at) - VALUES (?, '101', 1, 2, '2026-09-01', '2026-09-06')`).run(id); + FROM leaderboard_matches WHERE id = ?` + ) + .run(id, id % 2 === 0 ? 1 : 4); + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, has_stats, highest_round, first_seen_at, last_seen_at) + VALUES (?, '101', 1, 2, '2026-09-01', '2026-09-06')` + ) + .run(id); } const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const first = getPlayerProfile('101')!; @@ -377,69 +705,128 @@ describe('player profiles', () => { expect(second.matches).toHaveLength(9); expect(second.stats).toEqual(first.stats); expect(first.stats).toMatchObject({ gamesPlayed: 34, kills: 770 }); - expect(first.matches.map(match => [match.id, match.game])).toEqual(Array.from({ length: 25 }, (_, index) => { - const id = 39 - index; - return [id, id % 2 === 0 ? 't5' : 't6']; - })); - expect(second.matches.map(match => match.id)).toEqual([14, 13, 12, 11, 10, 4, 3, 2, 1]); - expect(new Set([...first.matches, ...second.matches].map(match => match.id)).size).toBe(34); + expect(first.matches.map((match) => [match.id, match.game])).toEqual( + Array.from({ length: 25 }, (_, index) => { + const id = 39 - index; + return [id, id % 2 === 0 ? 't5' : 't6']; + }) + ); + expect(second.matches.map((match) => match.id)).toEqual([14, 13, 12, 11, 10, 4, 3, 2, 1]); + expect(new Set([...first.matches, ...second.matches].map((match) => match.id)).size).toBe( + 34 + ); expect(getPlayerProfile('101', 9999)?.offset).toBe(25); - for (const offset of [-1, NaN, 0.5]) expect(getPlayerProfile('101', offset)?.offset).toBe(0); - const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - expect((await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=25') })).profile).toEqual(second); + for (const offset of [-1, NaN, 0.5]) + expect(getPlayerProfile('101', offset)?.offset).toBe(0); + const { load } = + await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); + expect( + ( + await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101?offset=25') + }) + ).profile + ).toEqual(second); // Historical same-round departures retain both pages even with no counted games. - handle.prepare(`UPDATE leaderboard_match_players SET first_observed_round = highest_round, - completed = 0 WHERE guid = '101'`).run(); + handle + .prepare( + `UPDATE leaderboard_match_players SET first_observed_round = highest_round, + completed = 0 WHERE guid = '101'` + ) + .run(); const departures = getPlayerProfile('101', 25)!; - expect(departures).toMatchObject({ stats: { gamesPlayed: 0, kills: 770 }, matchesTotal: 34, offset: 25 }); + expect(departures).toMatchObject({ + stats: { gamesPlayed: 0, kills: 770 }, + matchesTotal: 34, + offset: 25 + }); expect(departures.matches).toHaveLength(9); - expect(departures.maps.every(map => map.gamesPlayed === 0)).toBe(true); + expect(departures.maps.every((map) => map.gamesPlayed === 0)).toBe(true); }); it('loads canonical profile URLs, encodes GUIDs and returns 404 for missing players', async () => { await seedRecords(); - const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); + const { load } = + await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); const url = new URL('http://localhost/leaderboard/players/101'); - expect((await load({ params: { guid: '101' }, url })).profile).toMatchObject({ name: 'Ace', stats: { kills: 770, gamesPlayed: 4 } }); - await expect(load({ params: { guid: 'missing' }, url })).rejects.toThrow(expect.objectContaining({ status: 404 })); + expect((await load({ params: { guid: '101' }, url })).profile).toMatchObject({ + name: 'Ace', + stats: { kills: 770, gamesPlayed: 4 } + }); + await expect(load({ params: { guid: 'missing' }, url })).rejects.toThrow( + expect.objectContaining({ status: 404 }) + ); const { playerProfileHref } = await import('../../src/lib/leaderboard.js'); expect(playerProfileHref('001/2?#')).toBe('/leaderboard/players/001%2F2%3F%23'); }); it('permanently redirects both legacy game URLs while preserving the GUID and pagination query', async () => { - const { load } = await import('../../src/routes/leaderboard/players/[game]/[guid]/+page.server.js'); + const { load } = + await import('../../src/routes/leaderboard/players/[game]/[guid]/+page.server.js'); for (const game of ['t5', 't6']) { - const url = new URL(`http://localhost/leaderboard/players/${game}/001%2F2%3F%23?offset=25&source=old%20link`); - expect(() => load({ params: { game, guid: '001/2?#' }, url })).toThrow(expect.objectContaining({ - status: 308, location: '/leaderboard/players/001%2F2%3F%23?offset=25&source=old%20link' - })); + const url = new URL( + `http://localhost/leaderboard/players/${game}/001%2F2%3F%23?offset=25&source=old%20link` + ); + expect(() => load({ params: { game, guid: '001/2?#' }, url })).toThrow( + expect.objectContaining({ + status: 308, + location: '/leaderboard/players/001%2F2%3F%23?offset=25&source=old%20link' + }) + ); } - expect(() => load({ params: { game: 't7', guid: '101' }, url: new URL('http://localhost/leaderboard/players/t7/101') })) - .toThrow(expect.objectContaining({ status: 404 })); + expect(() => + load({ + params: { game: 't7', guid: '101' }, + url: new URL('http://localhost/leaderboard/players/t7/101') + }) + ).toThrow(expect.objectContaining({ status: 404 })); }); }); describe('leaderboard API', () => { it('returns one combined ranking row when a search matches the same player in both games', async () => { const handle = await seedRecords(); - handle.prepare("UPDATE leaderboard_players SET name = 'JoshWright' WHERE guid = '101'").run(); + handle + .prepare("UPDATE leaderboard_players SET name = 'JoshWright' WHERE guid = '101'") + .run(); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - const response = await GET({ url: new URL('http://localhost/api/leaderboard?q=JoshWright&metric=kills') }); + const response = await GET({ + url: new URL('http://localhost/api/leaderboard?q=JoshWright&metric=kills') + }); expect(response.status).toBe(200); - const result = await response.json() as LeaderboardResponse; - expect(result).toMatchObject({ total: 1, summary: { players: 4, matches: 5, kills: 1140 } }); - expect(result.entries).toEqual([expect.objectContaining({ guid: '101', name: 'JoshWright', rank: 1, - kills: 770, gamesPlayed: 4, gamesCompleted: 3, highestRound: 20, bestKills: 500, bestScore: 5000 })]); + const result = (await response.json()) as LeaderboardResponse; + expect(result).toMatchObject({ + total: 1, + summary: { players: 4, matches: 5, kills: 1140 } + }); + expect(result.entries).toEqual([ + expect.objectContaining({ + guid: '101', + name: 'JoshWright', + rank: 1, + kills: 770, + gamesPlayed: 4, + gamesCompleted: 3, + highestRound: 20, + bestKills: 500, + bestScore: 5000 + }) + ]); expect(result.entries[0]).not.toHaveProperty('game'); }); it('returns the requested ranking and pagination', async () => { await seedRecords(); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - const response = await GET({ url: new URL('http://localhost/api/leaderboard?game=t5&map=t5%3Atheater&metric=kills&limit=1&offset=1') }); + const response = await GET({ + url: new URL( + 'http://localhost/api/leaderboard?game=t5&map=t5%3Atheater&metric=kills&limit=1&offset=1' + ) + }); expect(response.status).toBe(200); - const result = await response.json() as LeaderboardResponse; + const result = (await response.json()) as LeaderboardResponse; expect(result).toMatchObject({ metric: 'kills', limit: 1, offset: 1, total: 3 }); expect(result.entries).toHaveLength(1); expect(result.entries[0]).toMatchObject({ guid: '202', rank: 2, kills: 100 }); @@ -447,10 +834,25 @@ describe('leaderboard API', () => { it('rejects invalid filters and non-integer or out-of-range pagination', async () => { const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - for (const query of ['game=t7', 'game=', 'metric=unknown', 'limit=0', 'limit=101', 'limit=', - 'limit=1.5', 'limit=1e1', 'offset=-1', 'offset=1000001', 'offset=Infinity', 'offset=0.5', - `q=${'a'.repeat(101)}`, `map=${'a'.repeat(1025)}`]) { - const response = await GET({ url: new URL(`http://localhost/api/leaderboard?${query}`) }); + for (const query of [ + 'game=t7', + 'game=', + 'metric=unknown', + 'limit=0', + 'limit=101', + 'limit=', + 'limit=1.5', + 'limit=1e1', + 'offset=-1', + 'offset=1000001', + 'offset=Infinity', + 'offset=0.5', + `q=${'a'.repeat(101)}`, + `map=${'a'.repeat(1025)}` + ]) { + const response = await GET({ + url: new URL(`http://localhost/api/leaderboard?${query}`) + }); expect(response.status, query).toBe(400); expect(await response.json()).toMatchObject({ ok: false, kind: 'bad-request' }); } @@ -460,13 +862,23 @@ describe('leaderboard API', () => { const handle = await seedRecords(); const map = 'a'.repeat(255); const key = `t5:${map}`; - handle.prepare('UPDATE leaderboard_matches SET map = ?, map_key = ?, map_name = ? WHERE id = 1') + handle + .prepare( + 'UPDATE leaderboard_matches SET map = ?, map_key = ?, map_name = ? WHERE id = 1' + ) .run(map, key, map); const { GET } = await import('../../src/routes/api/leaderboard/+server.js'); - const all = await (await GET({ url: new URL('http://localhost/api/leaderboard') })).json() as LeaderboardResponse; - const returnedKey = all.maps.find(option => option.map === map)!.key; - const response = await GET({ url: new URL(`http://localhost/api/leaderboard?map=${encodeURIComponent(returnedKey)}`) }); + const all = (await ( + await GET({ url: new URL('http://localhost/api/leaderboard') }) + ).json()) as LeaderboardResponse; + const returnedKey = all.maps.find((option) => option.map === map)!.key; + const response = await GET({ + url: new URL(`http://localhost/api/leaderboard?map=${encodeURIComponent(returnedKey)}`) + }); expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ total: 2, summary: { matches: 1, kills: 200 } }); + expect(await response.json()).toMatchObject({ + total: 2, + summary: { matches: 1, kills: 200 } + }); }); }); diff --git a/tests/unit/leaderboard-records.test.ts b/tests/unit/leaderboard-records.test.ts index e391dd4..2803184 100644 --- a/tests/unit/leaderboard-records.test.ts +++ b/tests/unit/leaderboard-records.test.ts @@ -4,28 +4,84 @@ import type { GameTelemetry, TelemetryPlayerStats } from '../../src/lib/telemetr import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); afterEach(async () => fixture.cleanup()); const timestamp = (minute: number) => new Date(Date.UTC(2026, 8, 18, 20, minute)).toISOString(); const player = (guid: string, kills = 0): TelemetryPlayerStats => ({ - guid, name: guid, kills, headshots: 0, downs: 0, revives: 0, score: 0 + guid, + name: guid, + kills, + headshots: 0, + downs: 0, + revives: 0, + score: 0 }); -function seed(handle: Database.Database, id: number, options: { - game?: string; map?: string; guid?: string; round?: number; kills?: number; score?: number; - ended?: string | null; received?: string; status?: string; hasStats?: number; observed?: number | null; -} = {}) { - const { game = 't5', map = 'theater', guid = '101', round = 10, kills = 100, score = 1000, - ended = timestamp(id), received = timestamp(id), status = 'completed', hasStats = 1, observed = null } = options; - handle.prepare(`INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, +function seed( + handle: Database.Database, + id: number, + options: { + game?: string; + map?: string; + guid?: string; + round?: number; + kills?: number; + score?: number; + ended?: string | null; + received?: string; + status?: string; + hasStats?: number; + observed?: number | null; + } = {} +) { + const { + game = 't5', + map = 'theater', + guid = '101', + round = 10, + kills = 100, + score = 1000, + ended = timestamp(id), + received = timestamp(id), + status = 'completed', + hasStats = 1, + observed = null + } = options; + handle + .prepare( + `INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, highest_round, last_event, last_fingerprint) - VALUES (?, 't5_zombie_theater_1', ?, ?, ?, ?, ?, ?, ?, ?, ?, 'game_end', ?)`) - .run(id, game, map, `${game}:${map}`, `${map} name`, timestamp(0), ended, received, status, round, String(id)); - handle.prepare(`INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES (?, ?, ?, ?, ?)`).run(game, guid, guid, timestamp(0), received); - handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, best_score, + VALUES (?, 't5_zombie_theater_1', ?, ?, ?, ?, ?, ?, ?, ?, ?, 'game_end', ?)` + ) + .run( + id, + game, + map, + `${game}:${map}`, + `${map} name`, + timestamp(0), + ended, + received, + status, + round, + String(id) + ); + handle + .prepare( + `INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES (?, ?, ?, ?, ?)` + ) + .run(game, guid, guid, timestamp(0), received); + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, best_score, has_stats, completed, first_seen_at, last_seen_at, first_observed_round) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?)`).run(id, guid, round, kills, score, hasStats, timestamp(0), received, observed); + VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?)` + ) + .run(id, guid, round, kills, score, hasStats, timestamp(0), received, observed); } it('links each best to its exact player/game/map scope, including profile map records', async () => { @@ -40,9 +96,19 @@ it('links each best to its exact player/game/map scope, including profile map re const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const overall = getLeaderboard({ q: '101' }).entries[0]; expect(overall.rank).toBe(2); - expect(overall.records.highestRound).toEqual({ matchId: 4, game: 't6', mapKey: 't6:town', mapName: 'town name', - server: 't5_zombie_theater_1', startedAt: timestamp(0), endedAt: timestamp(4), startObserved: false, - presentAtStart: null, firstObservedRound: null, value: 40 }); + expect(overall.records.highestRound).toEqual({ + matchId: 4, + game: 't6', + mapKey: 't6:town', + mapName: 'town name', + server: 't5_zombie_theater_1', + startedAt: timestamp(0), + endedAt: timestamp(4), + startObserved: false, + presentAtStart: null, + firstObservedRound: null, + value: 40 + }); for (const metric of ['highestRound', 'bestKills', 'bestScore'] as const) { expect(overall.records[metric]).toMatchObject({ matchId: 4, value: overall[metric] }); } @@ -57,8 +123,11 @@ it('links each best to its exact player/game/map scope, including profile map re expect(getLeaderboard({ offset: 1, limit: 1 }).entries[0].records).toEqual(overall.records); const profile = getPlayerProfile('101')!; expect(profile.records).toEqual(overall.records); - expect(profile.maps.find(map => map.key === 't5:theater')!.records).toEqual(theater); - expect(profile.maps.find(map => map.key === 't5:moon')!.records.bestScore).toMatchObject({ matchId: 3, value: 1000 }); + expect(profile.maps.find((map) => map.key === 't5:theater')!.records).toEqual(theater); + expect(profile.maps.find((map) => map.key === 't5:moon')!.records.bestScore).toMatchObject({ + matchId: 3, + value: 1000 + }); }); it('resolves tied bests by earliest completion, receipt time for unknown endings, then match ID', async () => { @@ -71,7 +140,8 @@ it('resolves tied bests by earliest completion, receipt time for unknown endings const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); for (const records of [getLeaderboard().entries[0].records, getPlayerProfile('101')!.records]) { - for (const record of Object.values(records)) expect(record).toMatchObject({ matchId: 3, endedAt: null }); + for (const record of Object.values(records)) + expect(record).toMatchObject({ matchId: 3, endedAt: null }); } }); @@ -87,7 +157,13 @@ it('ranks each map best against distinct players using leaderboard ties, qualifi seed(handle, 7, { game: 't6', round: 6, kills: 6, score: 6 }); seed(handle, 8, { game: 't6', guid: '202', round: 7, kills: 7, score: 7 }); seed(handle, 9, { guid: 'active', status: 'active', round: 99, kills: 9999, score: 9999 }); - seed(handle, 10, { guid: 'interrupted', status: 'interrupted', round: 99, kills: 9999, score: 9999 }); + seed(handle, 10, { + guid: 'interrupted', + status: 'interrupted', + round: 99, + kills: 9999, + score: 9999 + }); seed(handle, 11, { guid: 'unrecorded', hasStats: 0, round: 99, kills: 9999, score: 9999 }); seed(handle, 12, { guid: 'unqualified', round: 1, kills: 9999, score: 9999 }); handle.prepare('UPDATE leaderboard_matches SET start_players=2 WHERE id IN (1,6,7)').run(); @@ -95,16 +171,36 @@ it('ranks each map best against distinct players using leaderboard ties, qualifi const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); const { getPlayerLeaderboardEntry } = await import('../../src/lib/server/fleet/leaderboard.js'); const profile = getPlayerProfile('101')!; - expect(profile.maps.find(map => map.key === 't5:theater')?.ranks).toEqual({ highestRound: 3, bestKills: 2, bestScore: 2 }); - expect(profile.maps.find(map => map.key === 't5:moon')?.ranks).toEqual({ highestRound: 1, bestKills: 1, bestScore: 1 }); - expect(profile.maps.find(map => map.key === 't6:theater')?.ranks).toEqual({ highestRound: 2, bestKills: 2, bestScore: 2 }); - for (const map of profile.maps) for (const metric of ['highestRound', 'bestKills', 'bestScore'] as const) { - expect(map.ranks[metric]).toBe(getPlayerLeaderboardEntry('101', { game: map.game, map: map.key, metric })?.rank); - } - expect(getPlayerProfile('101', 0, { game: 't6', map: 't6:theater', players: 2 })?.maps).toEqual(profile.maps); + expect(profile.maps.find((map) => map.key === 't5:theater')?.ranks).toEqual({ + highestRound: 3, + bestKills: 2, + bestScore: 2 + }); + expect(profile.maps.find((map) => map.key === 't5:moon')?.ranks).toEqual({ + highestRound: 1, + bestKills: 1, + bestScore: 1 + }); + expect(profile.maps.find((map) => map.key === 't6:theater')?.ranks).toEqual({ + highestRound: 2, + bestKills: 2, + bestScore: 2 + }); + for (const map of profile.maps) + for (const metric of ['highestRound', 'bestKills', 'bestScore'] as const) { + expect(map.ranks[metric]).toBe( + getPlayerLeaderboardEntry('101', { game: map.game, map: map.key, metric })?.rank + ); + } + expect(getPlayerProfile('101', 0, { game: 't6', map: 't6:theater', players: 2 })?.maps).toEqual( + profile.maps + ); seed(handle, 13, { guid: 'new-leader', round: 40, kills: 400, score: 4000 }); - expect(getPlayerProfile('101')?.maps.find(map => map.key === 't5:theater')?.ranks) - .toEqual({ highestRound: 4, bestKills: 3, bestScore: 3 }); + expect(getPlayerProfile('101')?.maps.find((map) => map.key === 't5:theater')?.ranks).toEqual({ + highestRound: 4, + bestKills: 3, + bestScore: 3 + }); }); it('never uses unfinished, unsupported, nonqualifying or unrecorded stats as evidence', async () => { @@ -126,12 +222,21 @@ it('never uses unfinished, unsupported, nonqualifying or unrecorded stats as evi it('records first observed rounds for start participants and late arrivals without overwriting on return', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js'); + const { recordLeaderboardTelemetry } = + await import('../../src/lib/server/fleet/leaderboard-ingest.js'); const common = { server: 't5_zombie_theater_1', map: 'zombie_theater' }; - const report = (event: GameTelemetry['event'], round: number, players: TelemetryPlayerStats[], minute: number) => { - recordLeaderboardTelemetry(db(), event === 'game_start' - ? { ...common, event, players, receivedAt: timestamp(minute) } - : { ...common, event, round, players, receivedAt: timestamp(minute) }); + const report = ( + event: GameTelemetry['event'], + round: number, + players: TelemetryPlayerStats[], + minute: number + ) => { + recordLeaderboardTelemetry( + db(), + event === 'game_start' + ? { ...common, event, players, receivedAt: timestamp(minute) } + : { ...common, event, round, players, receivedAt: timestamp(minute) } + ); }; report('game_start', 0, [player('101')], 0); report('heartbeat', 0, [player('101'), player('303')], 1); @@ -140,12 +245,24 @@ it('records first observed rounds for start participants and late arrivals witho report('game_end', 52, [player('101', 200), player('202', 40), player('303')], 4); closeDb(); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); - expect(getPlayerProfile('101')!.records.highestRound).toMatchObject({ value: 52, startObserved: true, - firstObservedRound: 0, presentAtStart: true }); - expect(getPlayerProfile('202')!.records.highestRound).toMatchObject({ value: 52, startObserved: true, - firstObservedRound: 50, presentAtStart: false }); - expect(getPlayerProfile('303')!.records.highestRound).toMatchObject({ value: 52, startObserved: true, - firstObservedRound: 0, presentAtStart: false }); + expect(getPlayerProfile('101')!.records.highestRound).toMatchObject({ + value: 52, + startObserved: true, + firstObservedRound: 0, + presentAtStart: true + }); + expect(getPlayerProfile('202')!.records.highestRound).toMatchObject({ + value: 52, + startObserved: true, + firstObservedRound: 50, + presentAtStart: false + }); + expect(getPlayerProfile('303')!.records.highestRound).toMatchObject({ + value: 52, + startObserved: true, + firstObservedRound: 0, + presentAtStart: false + }); }); it('migrates legacy participation as unknown before snapshot backfill and preserves it across updates and reopens', async () => { @@ -154,25 +271,58 @@ it('migrates legacy participation as unknown before snapshot backfill and preser seed(handle, 1, { status: 'active' }); handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN first_observed_round'); handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN present_at_start'); - handle.prepare(`INSERT INTO servers (name, game, map) - VALUES ('t5_zombie_moon_1', 't5', 'zombie_moon')`).run(); - const saved: GameTelemetry = { event: 'game_end', server: 't5_zombie_moon_1', map: 'zombie_moon', round: 8, - players: [player('202', 80)], receivedAt: timestamp(2) }; - handle.prepare('INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)') + handle + .prepare( + `INSERT INTO servers (name, game, map) + VALUES ('t5_zombie_moon_1', 't5', 'zombie_moon')` + ) + .run(); + const saved: GameTelemetry = { + event: 'game_end', + server: 't5_zombie_moon_1', + map: 'zombie_moon', + round: 8, + players: [player('202', 80)], + receivedAt: timestamp(2) + }; + handle + .prepare( + 'INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)' + ) .run(saved.server, 8, 1, saved.receivedAt, JSON.stringify(saved)); handle.prepare("DELETE FROM settings WHERE key = 'leaderboard_v1'").run(); closeDb(); - expect(db().prepare('SELECT guid, first_observed_round, present_at_start FROM leaderboard_match_players ORDER BY guid').all()).toEqual([ + expect( + db() + .prepare( + 'SELECT guid, first_observed_round, present_at_start FROM leaderboard_match_players ORDER BY guid' + ) + .all() + ).toEqual([ { guid: '101', first_observed_round: null, present_at_start: null }, { guid: '202', first_observed_round: 8, present_at_start: 0 } ]); - const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js'); - recordLeaderboardTelemetry(db(), { event: 'game_end', server: 't5_zombie_theater_1', map: 'theater', round: 11, - players: [player('101', 150)], receivedAt: timestamp(3) }); + const { recordLeaderboardTelemetry } = + await import('../../src/lib/server/fleet/leaderboard-ingest.js'); + recordLeaderboardTelemetry(db(), { + event: 'game_end', + server: 't5_zombie_theater_1', + map: 'theater', + round: 11, + players: [player('101', 150)], + receivedAt: timestamp(3) + }); closeDb(); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); - expect(getPlayerProfile('101')!.records.bestKills).toMatchObject({ matchId: 1, value: 150, - firstObservedRound: null, presentAtStart: null }); - expect(getPlayerProfile('202')!.records.highestRound).toMatchObject({ firstObservedRound: 8, - startObserved: false, presentAtStart: false }); + expect(getPlayerProfile('101')!.records.bestKills).toMatchObject({ + matchId: 1, + value: 150, + firstObservedRound: null, + presentAtStart: null + }); + expect(getPlayerProfile('202')!.records.highestRound).toMatchObject({ + firstObservedRound: 8, + startObserved: false, + presentAtStart: false + }); }); diff --git a/tests/unit/library-metadata.test.ts b/tests/unit/library-metadata.test.ts index 815016d..9384576 100644 --- a/tests/unit/library-metadata.test.ts +++ b/tests/unit/library-metadata.test.ts @@ -4,16 +4,30 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { migrateVersionLayout } from '../../src/lib/server/db/version-layout.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); it('returns metadata without plugin bytes and lists many artifacts in two queries', async () => { - const { createScript, getVersion, getVersionContent, getLatestVersion, listVersions } = await import('../../src/lib/server/fleet/scripts.js'); - const { scriptCatalog, betaScriptMetadata } = await import('../../src/lib/server/fleet/library-metadata.js'); + const { createScript, getVersion, getVersionContent, getLatestVersion, listVersions } = + await import('../../src/lib/server/fleet/scripts.js'); + const { scriptCatalog, betaScriptMetadata } = + await import('../../src/lib/server/fleet/library-metadata.js'); const { db } = await import('../../src/lib/server/db/index.js'); const content = Buffer.alloc(1024 * 1024, 77); let id = 0; - for (let i = 0; i < 20; i++) id = createScript({ name: `plugin-${i}`, kind: 'plugin', targetRelPath: 'plugin.dll', content }).version.id; + for (let i = 0; i < 20; i++) + id = createScript({ + name: `plugin-${i}`, + kind: 'plugin', + targetRelPath: 'plugin.dll', + content + }).version.id; expect(getVersionContent(id)).toEqual(content); expect(getVersion(id)).not.toHaveProperty('content'); expect(getLatestVersion(getVersion(id)!.script_id)).not.toHaveProperty('content'); @@ -43,11 +57,23 @@ it('moves legacy content last without losing bytes, IDs, indexes, triggers or re migrateVersionLayout(handle); migrateVersionLayout(handle); expect(handle.prepare('SELECT * FROM script_versions').all()).toEqual(before); - expect((handle.pragma('table_info(script_versions)') as { name: string }[]).at(-1)?.name).toBe('content'); + expect( + (handle.pragma('table_info(script_versions)') as { name: string }[]).at(-1)?.name + ).toBe('content'); expect(handle.pragma('foreign_key_check')).toEqual([]); - expect(handle.prepare("INSERT INTO script_versions(script_id,version,content) VALUES(1,3,x'01')").run().lastInsertRowid).toBe(101); + expect( + handle + .prepare("INSERT INTO script_versions(script_id,version,content) VALUES(1,3,x'01')") + .run().lastInsertRowid + ).toBe(101); handle.prepare("UPDATE script_versions SET state='tested' WHERE id=101").run(); expect(handle.prepare('SELECT n FROM updates').get()).toEqual({ n: 1 }); - expect((handle.pragma('index_list(script_versions)') as { name: string }[]).some(index => index.name === 'version_state')).toBe(true); - } finally { handle.close(); } + expect( + (handle.pragma('index_list(script_versions)') as { name: string }[]).some( + (index) => index.name === 'version_state' + ) + ).toBe(true); + } finally { + handle.close(); + } }); diff --git a/tests/unit/library-scope-preview.test.ts b/tests/unit/library-scope-preview.test.ts index f53b4c2..0d00604 100644 --- a/tests/unit/library-scope-preview.test.ts +++ b/tests/unit/library-scope-preview.test.ts @@ -4,21 +4,48 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; const server = 't5_zombie_theater_1'; beforeEach(async () => { - vi.resetModules(); fixture = await newFixture([server]); + vi.resetModules(); + fixture = await newFixture([server]); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); describe('scope impact preview', () => { it('reports unscoping tracked script copies without changing scope or inventory', async () => { const { createScript, getScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { recordScript, listServerScripts } = await import('../../src/lib/server/fleet/inventory.js'); + const { recordScript, listServerScripts } = + await import('../../src/lib/server/fleet/inventory.js'); const { GET } = await import('../../src/routes/api/scripts/[id]/scope/+server.js'); - const script = createScript({ name: 'Example', scopeType: 'game', scopeValue: 't5', targetRelPath: 'example.gsc', content: Buffer.from('example') }); - recordScript(server, 'example.gsc', script.version.id, script.version.sha256, script.version.size, 'test'); - const response = await GET({ params: { id: String(script.script.id) }, url: new URL('http://localhost/?type=none') }); - expect(await response.json()).toMatchObject({ before: [server], after: [], removed: [server], detached: [server], affected: [server], composed: false }); + const script = createScript({ + name: 'Example', + scopeType: 'game', + scopeValue: 't5', + targetRelPath: 'example.gsc', + content: Buffer.from('example') + }); + recordScript( + server, + 'example.gsc', + script.version.id, + script.version.sha256, + script.version.size, + 'test' + ); + const response = await GET({ + params: { id: String(script.script.id) }, + url: new URL('http://localhost/?type=none') + }); + expect(await response.json()).toMatchObject({ + before: [server], + after: [], + removed: [server], + detached: [server], + affected: [server], + composed: false + }); expect(getScript(script.script.id)?.scope_type).toBe('game'); expect(listServerScripts(server)).toHaveLength(1); }); @@ -26,10 +53,27 @@ describe('scope impact preview', () => { it('reports config composition impact separately from detached inventory', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { GET } = await import('../../src/routes/api/scripts/[id]/scope/+server.js'); - const config = createScript({ name: 'Overlay', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set g_inactivity 0') }); - const response = await GET({ params: { id: String(config.script.id) }, url: new URL('http://localhost/?type=none') }); - expect(await response.json()).toMatchObject({ composed: true, detached: [], affected: [server], removed: [server] }); - const invalid = await GET({ params: { id: String(config.script.id) }, url: new URL('http://localhost/?type=invalid&value=t5') }); + const config = createScript({ + name: 'Overlay', + kind: 'config', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('set g_inactivity 0') + }); + const response = await GET({ + params: { id: String(config.script.id) }, + url: new URL('http://localhost/?type=none') + }); + expect(await response.json()).toMatchObject({ + composed: true, + detached: [], + affected: [server], + removed: [server] + }); + const invalid = await GET({ + params: { id: String(config.script.id) }, + url: new URL('http://localhost/?type=invalid&value=t5') + }); expect(invalid.status).toBe(400); }); }); diff --git a/tests/unit/log-refresh.test.ts b/tests/unit/log-refresh.test.ts index 19afe09..d46d886 100644 --- a/tests/unit/log-refresh.test.ts +++ b/tests/unit/log-refresh.test.ts @@ -6,7 +6,10 @@ const response = (line: string) => new Response(JSON.stringify({ ok: true, lines function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; - const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); return { promise, resolve, reject }; } let documentStub: EventTarget & { visibilityState: string }; @@ -16,12 +19,18 @@ beforeEach(() => { documentStub = Object.assign(new EventTarget(), { visibilityState: 'visible' }); vi.stubGlobal('document', documentStub); }); -afterEach(() => { controller?.dispose(); vi.useRealTimers(); vi.unstubAllGlobals(); }); +afterEach(() => { + controller?.dispose(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); function setup() { const fetcher = vi.fn(); const onReset = vi.fn(); - const onData = vi.fn(); const onError = vi.fn(); const onLoading = vi.fn(); + const onData = vi.fn(); + const onError = vi.fn(); + const onLoading = vi.fn(); controller = createLogRefresh({ fetcher, onReset, onData, onError, onLoading }); return { fetcher, onReset, onData, onError, onLoading }; } @@ -32,11 +41,22 @@ const visibility = (state: string) => { describe('log refresh controls', () => { it('retains a manually loaded log through same-server fleet updates and resets only on server navigation', async () => { - const fetcher = vi.fn().mockImplementation(async () => response('loaded log')); + const fetcher = vi + .fn() + .mockImplementation(async () => response('loaded log')); let displayed: string[] = []; - const reset = vi.fn(() => { displayed = []; }); - controller = createLogRefresh({ fetcher, onReset: reset, - onData: (_key, body) => { displayed = body.lines ?? []; }, onError: vi.fn(), onLoading: vi.fn() }); + const reset = vi.fn(() => { + displayed = []; + }); + controller = createLogRefresh({ + fetcher, + onReset: reset, + onData: (_key, body) => { + displayed = body.lines ?? []; + }, + onError: vi.fn(), + onLoading: vi.fn() + }); controller.configure(target, true, 5000); await controller.refresh(); controller.configure(target, false, 5000); @@ -68,7 +88,8 @@ describe('log refresh controls', () => { await controller.refresh(); controller.configure(target, false, 5000); await vi.advanceTimersByTimeAsync(30_000); - visibility('hidden'); visibility('visible'); + visibility('hidden'); + visibility('visible'); await vi.advanceTimersByTimeAsync(30_000); expect(fetcher).toHaveBeenCalledTimes(1); expect(onData).toHaveBeenCalledTimes(1); @@ -77,7 +98,8 @@ describe('log refresh controls', () => { it('aborts a pending refresh immediately and ignores its late data and loading updates after resume', async () => { const { fetcher, onData, onLoading, onError } = setup(); - const old = deferred(); const fresh = deferred(); + const old = deferred(); + const fresh = deferred(); fetcher.mockReturnValueOnce(old.promise).mockReturnValueOnce(fresh.promise); controller.configure(target, true, 5000); const pending = controller.refresh(); @@ -89,7 +111,9 @@ describe('log refresh controls', () => { controller.configure(target, true, 5000); const resumed = controller.refresh(); expect(fetcher).toHaveBeenCalledTimes(2); - onData.mockClear(); onError.mockClear(); onLoading.mockClear(); + onData.mockClear(); + onError.mockClear(); + onLoading.mockClear(); // Simulate a transport that delivers a response despite cancellation. old.resolve(response('obsolete')); await pending; @@ -98,7 +122,10 @@ describe('log refresh controls', () => { expect(onLoading).not.toHaveBeenCalled(); fresh.resolve(response('current')); await resumed; - expect(onData).toHaveBeenCalledWith('console', expect.objectContaining({ lines: ['current'] })); + expect(onData).toHaveBeenCalledWith( + 'console', + expect.objectContaining({ lines: ['current'] }) + ); expect(onLoading).toHaveBeenLastCalledWith(false); }); @@ -115,20 +142,28 @@ describe('log refresh controls', () => { next.reject(new Error('cancelled response')); await pending; expect(onData).toHaveBeenCalledTimes(1); - expect(onData).toHaveBeenLastCalledWith('console', expect.objectContaining({ lines: ['saved'] })); + expect(onData).toHaveBeenLastCalledWith( + 'console', + expect.objectContaining({ lines: ['saved'] }) + ); expect(onError).not.toHaveBeenCalled(); }); it('allows a manual refresh while paused without restarting automatic polling', async () => { const { fetcher, onData } = setup(); - fetcher.mockResolvedValueOnce(response('initial')).mockResolvedValueOnce(response('manual')); + fetcher + .mockResolvedValueOnce(response('initial')) + .mockResolvedValueOnce(response('manual')); controller.configure(target, true, 5000); await controller.refresh(); controller.configure(target, false, 5000); await controller.refresh(); await vi.advanceTimersByTimeAsync(30_000); expect(fetcher).toHaveBeenCalledTimes(2); - expect(onData).toHaveBeenLastCalledWith('console', expect.objectContaining({ lines: ['manual'] })); + expect(onData).toHaveBeenLastCalledWith( + 'console', + expect.objectContaining({ lines: ['manual'] }) + ); }); it('loads a selected log while paused and never accepts a reply from the prior server or tab', async () => { @@ -146,7 +181,10 @@ describe('log refresh controls', () => { expect(fetcher).toHaveBeenCalledTimes(2); expect(fetcher.mock.calls[1][0]).toContain('/another-server/logs/games_zm?lines=1000'); expect(onData).toHaveBeenCalledTimes(1); - expect(onData).toHaveBeenLastCalledWith('games_zm', expect.objectContaining({ lines: ['selected log'] })); + expect(onData).toHaveBeenLastCalledWith( + 'games_zm', + expect.objectContaining({ lines: ['selected log'] }) + ); }); it('resumes one polling loop and coalesces manual and scheduled reads', async () => { @@ -173,7 +211,9 @@ describe('log refresh controls', () => { await Promise.resolve(); controller.dispose(); expect(fetcher.mock.calls[0][1]!.signal!.aborted).toBe(true); - onData.mockClear(); onError.mockClear(); onLoading.mockClear(); + onData.mockClear(); + onError.mockClear(); + onLoading.mockClear(); body.resolve({ ok: true, lines: ['late body'] }); await pending; visibility('visible'); @@ -187,7 +227,8 @@ describe('log refresh controls', () => { it('keeps the last tail on a non-JSON gateway failure and can recover with a retry', async () => { const { fetcher, onData, onError } = setup(); - fetcher.mockResolvedValueOnce(response('saved')) + fetcher + .mockResolvedValueOnce(response('saved')) .mockResolvedValueOnce(new Response('Bad Gateway', { status: 502 })) .mockResolvedValueOnce(response('recovered')); controller.configure(target, false, 5000); @@ -195,10 +236,16 @@ it('keeps the last tail on a non-JSON gateway failure and can recover with a ret await controller.refresh(); expect(onError).toHaveBeenLastCalledWith('HTTP 502: Expected a JSON log response.'); expect(onData).toHaveBeenCalledTimes(1); - expect(onData).toHaveBeenLastCalledWith('console', expect.objectContaining({ lines: ['saved'] })); + expect(onData).toHaveBeenLastCalledWith( + 'console', + expect.objectContaining({ lines: ['saved'] }) + ); await controller.refresh(); expect(onError).toHaveBeenLastCalledWith(''); - expect(onData).toHaveBeenLastCalledWith('console', expect.objectContaining({ lines: ['recovered'] })); + expect(onData).toHaveBeenLastCalledWith( + 'console', + expect.objectContaining({ lines: ['recovered'] }) + ); }); it('rejects malformed successful log payloads before they reach the renderer', async () => { diff --git a/tests/unit/logs.test.ts b/tests/unit/logs.test.ts index 0df592d..f8c1778 100644 --- a/tests/unit/logs.test.ts +++ b/tests/unit/logs.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, beforeEach, afterEach } from 'vitest'; -import { appendFileSync, mkdirSync, renameSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs'; +import { + appendFileSync, + mkdirSync, + renameSync, + rmSync, + statSync, + utimesSync, + writeFileSync +} from 'node:fs'; import { resolve } from 'node:path'; import { tmpdir } from 'node:os'; @@ -12,9 +20,8 @@ process.env.REL_DEST = 'runtime/plutonium/storage/t5/raw/scripts/sp'; process.env.BETA_ARCHIVE_DIR = resolve(TMP_ROOT, 'beta-archive'); process.env.DB_PATH = resolve(TMP_ROOT, 'test.db'); -const { resolveLogPath, tailLog, readLogChunk, isLogKey } = await import( - '../../src/lib/server/fleet/logs.js' -); +const { resolveLogPath, tailLog, readLogChunk, isLogKey } = + await import('../../src/lib/server/fleet/logs.js'); const SERVER = 't5_zombie_theater_1'; const LOG_DIR = resolve(TMP_ROOT, SERVER, 'logs'); @@ -75,7 +82,10 @@ describe('tailLog', () => { const path = resolve(LOG_DIR, 'big.log'); // 500 KB of 60-char lines — comfortably larger than the 256 KB read window. const line = 'x'.repeat(59); - const body = Array.from({ length: 8500 }, (_, i) => `${i.toString().padStart(4, '0')}${line.slice(4)}`).join('\n'); + const body = Array.from( + { length: 8500 }, + (_, i) => `${i.toString().padStart(4, '0')}${line.slice(4)}` + ).join('\n'); writeFileSync(path, body); const tail = await tailLog(path, 100); diff --git a/tests/unit/maintenance-api.test.ts b/tests/unit/maintenance-api.test.ts index 8605003..d3d7a7d 100644 --- a/tests/unit/maintenance-api.test.ts +++ b/tests/unit/maintenance-api.test.ts @@ -1,17 +1,44 @@ import { beforeEach, afterEach, describe, it, expect, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn().mockResolvedValue(undefined) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn().mockResolvedValue(undefined) +})); let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); }); -afterEach(async () => { await fixture.cleanup(); }); -const request = (body: unknown) => new Request('http://localhost/api/maintenance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture(); +}); +afterEach(async () => { + await fixture.cleanup(); +}); +const request = (body: unknown) => + new Request('http://localhost/api/maintenance', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); describe('Maintenance API', () => { it('saves independent policies and the global pause without changing the restart schedule', async () => { const { POST, GET } = await import('../../src/routes/api/maintenance/+server.js'); - expect((await POST({ request: request({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'America/Indiana/Indianapolis' }) })).status).toBe(200); - const before = (await (GET()).json()).runtime.schedule; - for (const body of [{ action: 'policy', policy: 'health', enabled: false }, { action: 'policy', policy: 'updates', enabled: true }, { action: 'pause', paused: true }]) { + expect( + ( + await POST({ + request: request({ + action: 'schedule', + enabled: true, + time: '06:00', + timeZone: 'America/Indiana/Indianapolis' + }) + }) + ).status + ).toBe(200); + const before = (await GET().json()).runtime.schedule; + for (const body of [ + { action: 'policy', policy: 'health', enabled: false }, + { action: 'policy', policy: 'updates', enabled: true }, + { action: 'pause', paused: true } + ]) { expect((await POST({ request: request(body) })).status).toBe(200); } const data = await GET().json(); @@ -22,32 +49,69 @@ describe('Maintenance API', () => { it('rejects malformed actions without changing policies or queuing work', async () => { const { POST, GET } = await import('../../src/routes/api/maintenance/+server.js'); const before = await GET().json(); - for (const body of [null, [], {}, { action: 'pause', paused: 'yes' }, { action: 'policy', policy: 'unknown', enabled: true }, - { action: 'policy', policy: 'updates', enabled: 1 }, { action: 'schedule', enabled: true, time: '25:00', timeZone: 'UTC' }, - { action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' }, { action: 'update', servers: [] }, - { action: 'update', servers: [1] }, { action: 'retry', id: '1' }]) { + for (const body of [ + null, + [], + {}, + { action: 'pause', paused: 'yes' }, + { action: 'policy', policy: 'unknown', enabled: true }, + { action: 'policy', policy: 'updates', enabled: 1 }, + { action: 'schedule', enabled: true, time: '25:00', timeZone: 'UTC' }, + { action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' }, + { action: 'update', servers: [] }, + { action: 'update', servers: [1] }, + { action: 'retry', id: '1' } + ]) { expect((await POST({ request: request(body) })).status).toBe(400); } expect(await GET().json()).toEqual(before); }); it('opens one retained request outside the recent history window without changing normal history', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const insert = db().prepare("INSERT INTO maintenance_jobs(server,state,requested_at,kind,origin) VALUES(?,'done',?,'schedule','schedule')"); - db().transaction(() => { for (let i = 0; i < 105; i++) insert.run('t5_zombie_theater_1', new Date(Date.UTC(2026, 0, 1, 0, i)).toISOString()); })(); + const insert = db().prepare( + "INSERT INTO maintenance_jobs(server,state,requested_at,kind,origin) VALUES(?,'done',?,'schedule','schedule')" + ); + db().transaction(() => { + for (let i = 0; i < 105; i++) + insert.run( + 't5_zombie_theater_1', + new Date(Date.UTC(2026, 0, 1, 0, i)).toISOString() + ); + })(); const { GET } = await import('../../src/routes/api/maintenance/+server.js'); const ordinary = await GET().json(); expect(ordinary.runtime.jobs).toHaveLength(100); expect(ordinary.runtime.jobs.some((job: { id: number }) => job.id === 1)).toBe(false); - const selected = await GET({ url: new URL('http://localhost/api/maintenance?job=1') }).json(); + const selected = await GET({ + url: new URL('http://localhost/api/maintenance?job=1') + }).json(); expect(selected.runtime.jobs).toHaveLength(101); - expect(selected.runtime.jobs.find((job: { id: number }) => job.id === 1)).toMatchObject({ state: 'done', kind: 'schedule' }); - for (const job of ['0', '-1', '', '1.5', 'bad', '99999999999999999999']) expect(GET({ url: new URL(`http://localhost/api/maintenance?job=${job}`) }).status).toBe(400); + expect(selected.runtime.jobs.find((job: { id: number }) => job.id === 1)).toMatchObject({ + state: 'done', + kind: 'schedule' + }); + for (const job of ['0', '-1', '', '1.5', 'bad', '99999999999999999999']) + expect( + GET({ url: new URL(`http://localhost/api/maintenance?job=${job}`) }).status + ).toBe(400); expect(await GET().json()).toEqual(ordinary); }); it('keeps old Watchdog URLs working through query-preserving redirects', async () => { const index = await import('../../src/routes/watchdog/+page.server.js'); const detail = await import('../../src/routes/watchdog/[id]/+page.server.js'); - expect(() => index.load({ url: new URL('http://localhost/watchdog?kind=drift&server=test') })).toThrow(expect.objectContaining({ status: 308, location: '/maintenance?kind=drift&server=test' })); - expect(() => detail.load({ params: { id: 'abc' }, url: new URL('http://localhost/watchdog/abc?test=1') })).toThrow(expect.objectContaining({ status: 308, location: '/maintenance/abc?test=1' })); + expect(() => + index.load({ url: new URL('http://localhost/watchdog?kind=drift&server=test') }) + ).toThrow( + expect.objectContaining({ + status: 308, + location: '/maintenance?kind=drift&server=test' + }) + ); + expect(() => + detail.load({ + params: { id: 'abc' }, + url: new URL('http://localhost/watchdog/abc?test=1') + }) + ).toThrow(expect.objectContaining({ status: 308, location: '/maintenance/abc?test=1' })); }); }); diff --git a/tests/unit/maintenance-presentation.test.ts b/tests/unit/maintenance-presentation.test.ts index 879c287..d82228b 100644 --- a/tests/unit/maintenance-presentation.test.ts +++ b/tests/unit/maintenance-presentation.test.ts @@ -1,16 +1,33 @@ import { describe, expect, it } from 'vitest'; -import { groupMaintenanceScanErrors, groupMaintenanceVersions } from '../../src/lib/ui/maintenance-presentation.js'; +import { + groupMaintenanceScanErrors, + groupMaintenanceVersions +} from '../../src/lib/ui/maintenance-presentation.js'; import type { MaintenanceJob, PlutoniumVersion } from '../../src/lib/plutonium.js'; const theater = 't5_zombie_theater_1'; const moon = 't5_zombie_moon_1'; const version = (server: string, changes: Partial = {}): PlutoniumVersion => ({ - server, revision: 4878, checked_at: '2026-10-07T12:00:00Z', error: null, update_status: 'unknown', ...changes + server, + revision: 4878, + checked_at: '2026-10-07T12:00:00Z', + error: null, + update_status: 'unknown', + ...changes }); const job = (server: string, changes: Partial = {}): MaintenanceJob => ({ - id: 1, server, state: 'queued', target_revision: 4878, requested_at: '2026-10-07T12:00:00Z', - restarted_at: null, provisioning_id: 'restart-1', finished_at: null, detail: null, - kind: 'update', origin: 'manual', ...changes + id: 1, + server, + state: 'queued', + target_revision: 4878, + requested_at: '2026-10-07T12:00:00Z', + restarted_at: null, + provisioning_id: 'restart-1', + finished_at: null, + detail: null, + kind: 'update', + origin: 'manual', + ...changes }); describe('maintenance scan summaries', () => { @@ -18,7 +35,11 @@ describe('maintenance scan summaries', () => { const errors = `${theater}: Host agent is offline; /srv/private-one unavailable\n${moon}: Host agent is offline; /srv/private-two unavailable`; const groups = groupMaintenanceScanErrors(errors, [theater, moon]); expect(groups).toHaveLength(1); - expect(groups[0]).toMatchObject({ key: 'host', title: 'Host agent unavailable', servers: [theater, moon] }); + expect(groups[0]).toMatchObject({ + key: 'host', + title: 'Host agent unavailable', + servers: [theater, moon] + }); expect(groups[0].details).toHaveLength(2); expect(groups[0].description).not.toContain('/srv/'); expect(groups[0].description).not.toContain(theater); @@ -26,21 +47,34 @@ describe('maintenance scan summaries', () => { }); it('separates file, permission, timeout and unknown causes without exposing raw messages as headings', () => { - const groups = groupMaintenanceScanErrors([ - `${theater}: ENOENT: no such file /srv/missing`, - `${moon}: EACCES: permission denied /srv/private`, - `${theater}: timeout: no reply`, - `${moon}: Unrecognized diagnostic code 123` - ].join('\n'), [theater, moon]); - expect(groups.map(group => group.key)).toEqual(['files', 'permissions', 'timeout', 'other']); - expect(groups.every(group => !/ENOENT|EACCES|123/.test(group.title))).toBe(true); + const groups = groupMaintenanceScanErrors( + [ + `${theater}: ENOENT: no such file /srv/missing`, + `${moon}: EACCES: permission denied /srv/private`, + `${theater}: timeout: no reply`, + `${moon}: Unrecognized diagnostic code 123` + ].join('\n'), + [theater, moon] + ); + expect(groups.map((group) => group.key)).toEqual([ + 'files', + 'permissions', + 'timeout', + 'other' + ]); + expect(groups.every((group) => !/ENOENT|EACCES|123/.test(group.title))).toBe(true); }); it('supports custom names containing colons, avoids linking unknown identifiers and deduplicates identical failures', () => { const message = 'Weekend: west: timeout: No reply'; - const groups = groupMaintenanceScanErrors(`${message}\n${message}\nretired_server: Agent unavailable`, ['Weekend', 'Weekend: west']); + const groups = groupMaintenanceScanErrors( + `${message}\n${message}\nretired_server: Agent unavailable`, + ['Weekend', 'Weekend: west'] + ); expect(groups[0].servers).toEqual(['Weekend: west']); - expect(groups[0].details).toEqual([{ server: 'Weekend: west', message: 'timeout: No reply' }]); + expect(groups[0].details).toEqual([ + { server: 'Weekend: west', message: 'timeout: No reply' } + ]); expect(groups[1].servers).toEqual([]); expect(groups[1].details[0].server).toBeNull(); }); @@ -48,46 +82,78 @@ describe('maintenance scan summaries', () => { it('handles setup messages and empty scans without inventing an affected server', () => { expect(groupMaintenanceScanErrors(null, [])).toEqual([]); expect(groupMaintenanceScanErrors(' \n ', [])).toEqual([]); - expect(groupMaintenanceScanErrors('Register a native host instance before automatic recovery.', [])) - .toEqual([expect.objectContaining({ key: 'setup', servers: [] })]); + expect( + groupMaintenanceScanErrors( + 'Register a native host instance before automatic recovery.', + [] + ) + ).toEqual([expect.objectContaining({ key: 'setup', servers: [] })]); }); }); describe('maintenance revision groups', () => { it('collapses identical revisions and verification states without claiming that unknown versions are current', () => { - const versions = Array.from({ length: 12 }, (_, index) => version(`t5_zombie_theater_${index + 1}`)); + const versions = Array.from({ length: 12 }, (_, index) => + version(`t5_zombie_theater_${index + 1}`) + ); const groups = groupMaintenanceVersions(versions, []); expect(groups).toHaveLength(1); - expect(groups[0]).toMatchObject({ revision: 4878, label: 'Unverified', tone: 'warning', updatable: [] }); + expect(groups[0]).toMatchObject({ + revision: 4878, + label: 'Unverified', + tone: 'warning', + updatable: [] + }); expect(groups[0].versions).toHaveLength(12); }); it('retains separate verification states and puts failures and available updates first', () => { - const groups = groupMaintenanceVersions([ - version('current', { update_status: 'current' }), version('unknown'), - version('outdated', { revision: 4800, update_status: 'outdated' }), - version('ahead', { revision: 5000, update_status: 'ahead' }) - ], []); - expect(groups.map(group => group.status)).toEqual(['unknown', 'outdated', 'current', 'ahead']); + const groups = groupMaintenanceVersions( + [ + version('current', { update_status: 'current' }), + version('unknown'), + version('outdated', { revision: 4800, update_status: 'outdated' }), + version('ahead', { revision: 5000, update_status: 'ahead' }) + ], + [] + ); + expect(groups.map((group) => group.status)).toEqual([ + 'unknown', + 'outdated', + 'current', + 'ahead' + ]); expect(groups[0].revision).toBe(groups[2].revision); }); it('excludes pending restarts from group update actions while preserving the pending operation links', () => { const pending = job(theater, { state: 'restarting' }); - const [group] = groupMaintenanceVersions([ - version(theater, { update_status: 'outdated' }), version(moon, { update_status: 'outdated' }) - ], [pending, job(moon, { id: 2, state: 'done' })]); + const [group] = groupMaintenanceVersions( + [ + version(theater, { update_status: 'outdated' }), + version(moon, { update_status: 'outdated' }) + ], + [pending, job(moon, { id: 2, state: 'done' })] + ); expect(group.updatable).toEqual([moon]); expect(group.pending).toEqual([pending]); }); it('does not let deleted-server jobs suppress a current server update and preserves stopped legacy evidence', () => { - const groups = groupMaintenanceVersions([ - version(theater, { update_status: 'outdated' }), - version(moon, { update_status: undefined, stopped: true, revision: null }) - ], [job(theater, { server_deleted_at: '2026-10-06' })]); + const groups = groupMaintenanceVersions( + [ + version(theater, { update_status: 'outdated' }), + version(moon, { update_status: undefined, stopped: true, revision: null }) + ], + [job(theater, { server_deleted_at: '2026-10-06' })] + ); expect(groups[0].updatable).toEqual([theater]); expect(groups[0].pending).toEqual([]); - expect(groups[1]).toMatchObject({ status: 'stopped', label: 'Intentionally stopped', revision: null, updatable: [] }); + expect(groups[1]).toMatchObject({ + status: 'stopped', + label: 'Intentionally stopped', + revision: null, + updatable: [] + }); }); }); diff --git a/tests/unit/map-aliases.test.ts b/tests/unit/map-aliases.test.ts index 6701169..0c8191d 100644 --- a/tests/unit/map-aliases.test.ts +++ b/tests/unit/map-aliases.test.ts @@ -11,22 +11,41 @@ import { serverTitle } from '../../src/lib/ui/server-labels.js'; describe('T6 map identity', () => { it.each([ - ['towngrief', 'zm_transit'], ['farmgrief', 'zm_transit'], ['busdepotgrief', 'zm_transit'], - ['cellblock', 'zm_prison'], ['buriedgrief', 'zm_buried'], ['diner', 'zm_transit_dr'], ['buriedturned', 'zm_buried'] - ])('excludes %s without falling back to a supported variant of the same engine map', (slug, canonical) => { - expect(zombieMapBySlug('t6', slug)).toBeUndefined(); - expect(selectedZombieMap('t6', canonical, slug)).toBeUndefined(); - expect(parseServerName(`t6_zombie_${slug}_1`)?.mapCanonical).toBeNull(); - }); + ['towngrief', 'zm_transit'], + ['farmgrief', 'zm_transit'], + ['busdepotgrief', 'zm_transit'], + ['cellblock', 'zm_prison'], + ['buriedgrief', 'zm_buried'], + ['diner', 'zm_transit_dr'], + ['buriedturned', 'zm_buried'] + ])( + 'excludes %s without falling back to a supported variant of the same engine map', + (slug, canonical) => { + expect(zombieMapBySlug('t6', slug)).toBeUndefined(); + expect(selectedZombieMap('t6', canonical, slug)).toBeUndefined(); + expect(parseServerName(`t6_zombie_${slug}_1`)?.mapCanonical).toBeNull(); + } + ); it.each(T6_ZOMBIE_MAPS)('parses and labels $friendly without losing its variant', (map) => { const name = `t6_zombie_${map.slug}_2`; - expect(parseServerName(name)).toMatchObject({ game: 't6', gameMode: 'zombie', mapCanonical: map.canonical, mapSlug: map.slug, lobby: 2 }); + expect(parseServerName(name)).toMatchObject({ + game: 't6', + gameMode: 'zombie', + mapCanonical: map.canonical, + mapSlug: map.slug, + lobby: 2 + }); expect(serverTitle(name)).toBe(`${map.friendly} · Lobby 2`); }); it('does not interpret another game or mode through the T5 catalog', () => { - for (const name of ['t6_zombie_theater_1', 't5_zombie_tomb_1', 't5_mp_theater_1', 'iw5_theater_1']) { + for (const name of [ + 't6_zombie_theater_1', + 't5_zombie_tomb_1', + 't5_mp_theater_1', + 'iw5_theater_1' + ]) { expect(parseServerName(name)?.mapCanonical).toBeNull(); } }); diff --git a/tests/unit/map-config-flow.test.ts b/tests/unit/map-config-flow.test.ts index 0afc841..09437ac 100644 --- a/tests/unit/map-config-flow.test.ts +++ b/tests/unit/map-config-flow.test.ts @@ -15,7 +15,8 @@ beforeEach(async () => { vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - for (const server of servers) await mkdir(join(fixture.t5, server, 'configs'), { recursive: true }); + for (const server of servers) + await mkdir(join(fixture.t5, server, 'configs'), { recursive: true }); }); afterEach(async () => { @@ -25,22 +26,32 @@ afterEach(async () => { describe('map selection through config lifecycle', () => { it('uses each server map on beta, promotion, and drift repair without map overlays', async () => { - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); + const { scanServerDrift, runDriftDeployment } = + await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script } = createScript({ - name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', + name: 'baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't5', content: Buffer.from('set sv_voice "1"\nset sv_maprotation "map zombie_moon"\n') }); const version = getLatestVersion(script.id)!; const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: [servers[0]], waitForHealth: false, waitForMatch: false + servers: [servers[0]], + waitForHealth: false, + waitForMatch: false }); await activateFiles('beta', betaId); - expect(await readFile(cfgPath(servers[0]), 'utf8')).toContain('set sv_maprotation "map zombie_theater"'); - db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?") + expect(await readFile(cfgPath(servers[0]), 'utf8')).toContain( + 'set sv_maprotation "map zombie_theater"' + ); + db() + .prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?") .run(new Date().toISOString(), betaId); db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId); const promoted = await promoteBeta(betaId); @@ -60,24 +71,35 @@ describe('map selection through config lifecycle', () => { }); it('preserves unrelated legacy overlay settings and restores exact prior config on rollback', async () => { - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); const baseline = createScript({ - name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', + name: 'baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }).script; const baselineVersion = getLatestVersion(baseline.id)!; - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baselineVersion.id, baseline.id); + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(baselineVersion.id, baseline.id); const overlay = createScript({ - name: 'legacy map choice', kind: 'config', scopeType: 'map', scopeValue: 'zombie_cosmodrome', + name: 'legacy map choice', + kind: 'config', + scopeType: 'map', + scopeValue: 'zombie_cosmodrome', content: Buffer.from('set sv_maprotation "map zombie_moon"\nset g_inactivity "0"\n') }).script; const oldConfig = '// operator config\r\nset sv_maprotation "map zombie_cosmodrome"\r\n'; await writeFile(cfgPath(servers[1]), oldConfig); const { betaId } = await openBeta({ scripts: [{ scriptId: overlay.id, scriptVersionId: getLatestVersion(overlay.id)!.id }], - servers: [servers[1]], waitForHealth: false, waitForMatch: false + servers: [servers[1]], + waitForHealth: false, + waitForMatch: false }); await activateFiles('beta', betaId); const preview = await readFile(cfgPath(servers[1]), 'utf8'); @@ -91,7 +113,8 @@ describe('map selection through config lifecycle', () => { }); it('keeps Plutainer default config during reconciliation until a baseline is promoted', async () => { - const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); + const { scanServerDrift, runDriftDeployment } = + await import('../../src/lib/server/fleet/drift.js'); const defaults = 'set sv_voice "1"\nset sv_maprotation "map zombie_theater"\n'; await writeFile(cfgPath(servers[1]), defaults); await writeFile(join(fixture.t5, servers[1], 'configs', 'stray.cfg'), 'set stray "1"'); diff --git a/tests/unit/match-detail.test.ts b/tests/unit/match-detail.test.ts index 926fc0c..2587f5c 100644 --- a/tests/unit/match-detail.test.ts +++ b/tests/unit/match-detail.test.ts @@ -7,7 +7,14 @@ let received = 0; const server = 't5_zombie_theater_1'; const timestamp = (minute: number) => new Date(Date.UTC(2026, 8, 18, 20, minute)).toISOString(); const player = (guid: string, stats: Partial = {}): TelemetryPlayerStats => ({ - guid, name: `Player ${guid}`, kills: 0, headshots: 0, downs: 0, revives: 0, score: 0, ...stats + guid, + name: `Player ${guid}`, + kills: 0, + headshots: 0, + downs: 0, + revives: 0, + score: 0, + ...stats }); beforeEach(async () => { @@ -15,49 +22,118 @@ beforeEach(async () => { received = 0; fixture = await newFixture([]); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't5', 'zombie_theater')") + db() + .prepare("INSERT INTO servers (name, game, map) VALUES (?, 't5', 'zombie_theater')") .run(server); }); afterEach(async () => fixture.cleanup()); async function report(event: GameEvent['event'], round = 0, players = [player('101')]) { const { db } = await import('../../src/lib/server/db/index.js'); - const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js'); + const { recordLeaderboardTelemetry } = + await import('../../src/lib/server/fleet/leaderboard-ingest.js'); const common = { server, map: 'zombie_theater', receivedAt: timestamp(received++) }; - recordLeaderboardTelemetry(db(), event === 'game_start' - ? { ...common, event, players: players.map(({ guid, name }) => ({ guid, name })) } - : { ...common, event, round, players }); + recordLeaderboardTelemetry( + db(), + event === 'game_start' + ? { ...common, event, players: players.map(({ guid, name }) => ({ guid, name })) } + : { ...common, event, round, players } + ); } describe('completed match details', () => { it('returns the history summary and every participant with recorded counters, including departed and late players', async () => { await report('game_start', 0, [player('101'), player('202'), player('303')]); - await report('round_change', 2, [player('101', { kills: 30, headshots: 10, downs: 1, revives: 2, score: 2000 }), - player('202', { kills: 15, score: 1500 })]); - await report('heartbeat', 2, [player('101', { kills: 40, headshots: 12, downs: 1, revives: 2, score: 1000 })]); - await report('game_end', 4, [player('101', { kills: 50, headshots: 20, downs: 2, revives: 3, score: 500 }), player('404')]); - const { getMatchDetail, serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); + await report('round_change', 2, [ + player('101', { kills: 30, headshots: 10, downs: 1, revives: 2, score: 2000 }), + player('202', { kills: 15, score: 1500 }) + ]); + await report('heartbeat', 2, [ + player('101', { kills: 40, headshots: 12, downs: 1, revives: 2, score: 1000 }) + ]); + await report('game_end', 4, [ + player('101', { kills: 50, headshots: 20, downs: 2, revives: 3, score: 500 }), + player('404') + ]); + const { getMatchDetail, serverMatchHistory } = + await import('../../src/lib/server/fleet/match-history.js'); const match = getMatchDetail(1)!; const { game, startObserved, players, chat, ...summary } = match; expect(chat).toEqual([]); expect({ ...summary, game }).toEqual(serverMatchHistory(server).matches[0]); expect({ ...summary, game, startObserved }).toMatchObject({ - id: 1, game: 't5', startObserved: true, server, status: 'completed', - startedAt: timestamp(0), endedAt: timestamp(3), durationMs: 180_000, - startPlayers: 3, uniquePlayers: 4, endPlayers: 2, finalRound: 4 + id: 1, + game: 't5', + startObserved: true, + server, + status: 'completed', + startedAt: timestamp(0), + endedAt: timestamp(3), + durationMs: 180_000, + startPlayers: 3, + uniquePlayers: 4, + endPlayers: 2, + finalRound: 4 + }); + expect(players.map((entry) => entry.guid)).toEqual(['101', '202', '404', '303']); + expect(players[0]).toEqual({ + guid: '101', + name: 'Player 101', + hasStats: true, + presentAtEnd: true, + canViewProfile: true, + firstSeenAt: timestamp(0), + lastSeenAt: timestamp(3), + highestRound: 4, + roundsSurvived: 3, + kills: 50, + headshots: 20, + downs: 2, + revives: 3, + bestScore: 2000 + }); + expect(players[1]).toEqual({ + guid: '202', + name: 'Player 202', + hasStats: true, + presentAtEnd: false, + canViewProfile: true, + firstSeenAt: timestamp(0), + lastSeenAt: timestamp(1), + highestRound: 2, + roundsSurvived: 1, + kills: 15, + headshots: 0, + downs: 0, + revives: 0, + bestScore: 1500 + }); + expect(players[2]).toMatchObject({ + guid: '404', + presentAtEnd: true, + highestRound: 4, + roundsSurvived: 0, + firstSeenAt: timestamp(3), + lastSeenAt: timestamp(3), + kills: 0, + bestScore: 0 + }); + expect(players[3]).toEqual({ + guid: '303', + name: 'Player 303', + hasStats: false, + presentAtEnd: false, + canViewProfile: false, + firstSeenAt: timestamp(0), + lastSeenAt: timestamp(0), + highestRound: null, + roundsSurvived: null, + kills: null, + headshots: null, + downs: null, + revives: null, + bestScore: null }); - expect(players.map(entry => entry.guid)).toEqual(['101', '202', '404', '303']); - expect(players[0]).toEqual({ guid: '101', name: 'Player 101', hasStats: true, presentAtEnd: true, - canViewProfile: true, firstSeenAt: timestamp(0), lastSeenAt: timestamp(3), - highestRound: 4, roundsSurvived: 3, kills: 50, headshots: 20, downs: 2, revives: 3, bestScore: 2000 }); - expect(players[1]).toEqual({ guid: '202', name: 'Player 202', hasStats: true, presentAtEnd: false, - canViewProfile: true, firstSeenAt: timestamp(0), lastSeenAt: timestamp(1), - highestRound: 2, roundsSurvived: 1, kills: 15, headshots: 0, downs: 0, revives: 0, bestScore: 1500 }); - expect(players[2]).toMatchObject({ guid: '404', presentAtEnd: true, highestRound: 4, roundsSurvived: 0, - firstSeenAt: timestamp(3), lastSeenAt: timestamp(3), kills: 0, bestScore: 0 }); - expect(players[3]).toEqual({ guid: '303', name: 'Player 303', hasStats: false, presentAtEnd: false, - canViewProfile: false, firstSeenAt: timestamp(0), lastSeenAt: timestamp(0), - highestRound: null, roundsSurvived: null, kills: null, headshots: null, downs: null, revives: null, bestScore: null }); }); it('preserves an unknown start and an observed empty ending roster across database reopen', async () => { @@ -66,19 +142,40 @@ describe('completed match details', () => { const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { getMatchDetail } = await import('../../src/lib/server/fleet/match-history.js'); - expect(getMatchDetail(1)).toMatchObject({ startObserved: false, startedAt: timestamp(0), - startPlayers: null, durationMs: null, endedAt: timestamp(1), endPlayers: 0, - uniquePlayers: 1, finalRound: 6, players: [expect.objectContaining({ - guid: '101', presentAtEnd: false, highestRound: 5, kills: 40, roundsSurvived: 0 - })] }); + expect(getMatchDetail(1)).toMatchObject({ + startObserved: false, + startedAt: timestamp(0), + startPlayers: null, + durationMs: null, + endedAt: timestamp(1), + endPlayers: 0, + uniquePlayers: 1, + finalRound: 6, + players: [ + expect.objectContaining({ + guid: '101', + presentAtEnd: false, + highestRound: 5, + kills: 40, + roundsSurvived: 0 + }) + ] + }); }); it('loads an empty completed record without inventing player rows', async () => { await report('game_start', 0, []); await report('game_end', 2, []); const { getMatchDetail } = await import('../../src/lib/server/fleet/match-history.js'); - expect(getMatchDetail(1)).toMatchObject({ startPlayers: 0, uniquePlayers: 0, endPlayers: 0, - startObserved: true, durationMs: 60_000, finalRound: 2, players: [] }); + expect(getMatchDetail(1)).toMatchObject({ + startPlayers: 0, + uniquePlayers: 0, + endPlayers: 0, + startObserved: true, + durationMs: 60_000, + finalRound: 2, + players: [] + }); }); it('preserves original match and server identity after deletion and reuse of the server name', async () => { @@ -90,13 +187,27 @@ describe('completed match details', () => { db().prepare('DELETE FROM servers WHERE name = ?').run(server); const deleted = getMatchDetail(1)!; expect(deleted).toEqual({ ...before, server_deleted_at: expect.any(String) }); - db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't6', 'zm_transit')").run(server); - const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js'); - recordLeaderboardTelemetry(db(), { event: 'game_end', server, map: 'zm_transit', round: 20, - players: [player('999', { kills: 999 })], receivedAt: timestamp(4) }); + db() + .prepare("INSERT INTO servers (name, game, map) VALUES (?, 't6', 'zm_transit')") + .run(server); + const { recordLeaderboardTelemetry } = + await import('../../src/lib/server/fleet/leaderboard-ingest.js'); + recordLeaderboardTelemetry(db(), { + event: 'game_end', + server, + map: 'zm_transit', + round: 20, + players: [player('999', { kills: 999 })], + receivedAt: timestamp(4) + }); expect(getMatchDetail(1)).toEqual(deleted); - expect(getMatchDetail(2)).toMatchObject({ id: 2, game: 't6', server_deleted_at: null, - finalRound: 20, players: [expect.objectContaining({ guid: '999', kills: 999 })] }); + expect(getMatchDetail(2)).toMatchObject({ + id: 2, + game: 't6', + server_deleted_at: null, + finalRound: 20, + players: [expect.objectContaining({ guid: '999', kills: 999 })] + }); }); it('uses the latest shared player name and only offers profiles that actually exist', async () => { @@ -105,32 +216,58 @@ describe('completed match details', () => { await report('game_end', 2, [player('101')]); const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES ('t6', '101', 'Latest name', ?, ?), ('t7', '101', 'Unsupported name', ?, ?)`) + handle + .prepare( + `INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES ('t6', '101', 'Latest name', ?, ?), ('t7', '101', 'Unsupported name', ?, ?)` + ) .run(timestamp(0), timestamp(4), timestamp(0), timestamp(5)); const { getMatchDetail } = await import('../../src/lib/server/fleet/match-history.js'); const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js'); let participants = getMatchDetail(1)!.players; - expect(participants.find(entry => entry.guid === '101')).toMatchObject({ name: 'Latest name', canViewProfile: true }); - expect(participants.find(entry => entry.guid === '202')).toMatchObject({ hasStats: true, highestRound: 1, canViewProfile: false }); - expect(participants.find(entry => entry.guid === '303')).toMatchObject({ hasStats: false, canViewProfile: false }); - for (const participant of participants) expect(participant.canViewProfile).toBe(getPlayerProfile(participant.guid) !== null); + expect(participants.find((entry) => entry.guid === '101')).toMatchObject({ + name: 'Latest name', + canViewProfile: true + }); + expect(participants.find((entry) => entry.guid === '202')).toMatchObject({ + hasStats: true, + highestRound: 1, + canViewProfile: false + }); + expect(participants.find((entry) => entry.guid === '303')).toMatchObject({ + hasStats: false, + canViewProfile: false + }); + for (const participant of participants) + expect(participant.canViewProfile).toBe(getPlayerProfile(participant.guid) !== null); // A start-only participant can still have a valid profile from another match. await report('game_start', 0, [player('303')]); await report('heartbeat', 3, [player('303')]); - expect(getMatchDetail(1)!.players.find(entry => entry.guid === '303')).toMatchObject({ canViewProfile: false }); + expect(getMatchDetail(1)!.players.find((entry) => entry.guid === '303')).toMatchObject({ + canViewProfile: false + }); expect(getPlayerProfile('303')).toBeNull(); await report('game_end', 3, [player('303')]); participants = getMatchDetail(1)!.players; - expect(participants.find(entry => entry.guid === '303')).toMatchObject({ hasStats: false, canViewProfile: true }); + expect(participants.find((entry) => entry.guid === '303')).toMatchObject({ + hasStats: false, + canViewProfile: true + }); // Missing identities must not discard a saved roster row or produce a dead link. handle.prepare("DELETE FROM leaderboard_players WHERE guid = '101'").run(); - expect(getMatchDetail(1)!.players.find(entry => entry.guid === '101')).toMatchObject({ name: '101', canViewProfile: false }); + expect(getMatchDetail(1)!.players.find((entry) => entry.guid === '101')).toMatchObject({ + name: '101', + canViewProfile: false + }); handle.prepare("UPDATE leaderboard_matches SET game = 'custom' WHERE id = 2").run(); handle.prepare("UPDATE leaderboard_players SET game = 'custom' WHERE guid = '303'").run(); - expect(getMatchDetail(2)!.players[0]).toMatchObject({ name: 'Player 303', hasStats: true, canViewProfile: false }); + expect(getMatchDetail(2)!.players[0]).toMatchObject({ + name: 'Player 303', + hasStats: true, + canViewProfile: false + }); }); it('only exposes completed records at round 2 or higher and rejects invalid IDs', async () => { @@ -164,9 +301,26 @@ describe('match detail page load', () => { await report('game_start'); await report('round_change', 2); const { load } = await import('../../src/routes/matches/[id]/+page.server.js'); - for (const id of ['1', '2', '3', '999', '', '0', '-1', '1.0', '01', '1e0', '1x', ' 1', 'Infinity', - '9007199254740992', "1' OR 1=1 --"]) { - expect(() => load({ params: { id } }), id).toThrow(expect.objectContaining({ status: 404 })); + for (const id of [ + '1', + '2', + '3', + '999', + '', + '0', + '-1', + '1.0', + '01', + '1e0', + '1x', + ' 1', + 'Infinity', + '9007199254740992', + "1' OR 1=1 --" + ]) { + expect(() => load({ params: { id } }), id).toThrow( + expect.objectContaining({ status: 404 }) + ); } }); }); diff --git a/tests/unit/match-history-query.test.ts b/tests/unit/match-history-query.test.ts index 739f244..d088132 100644 --- a/tests/unit/match-history-query.test.ts +++ b/tests/unit/match-history-query.test.ts @@ -1,26 +1,48 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { readMatchQuery, matchHistoryHref, matchApiParams } from '../../src/lib/match-history-query.js'; +import { + readMatchQuery, + matchHistoryHref, + matchApiParams +} from '../../src/lib/match-history-query.js'; let fixture: Fixture; const now = Date.parse('2026-10-06T20:00:00Z'); beforeEach(async () => { - vi.resetModules(); fixture = await newFixture([]); + vi.resetModules(); + fixture = await newFixture([]); const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - const match = handle.prepare(`INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, start_observed, highest_round, last_event, last_fingerprint) + const match = + handle.prepare(`INSERT INTO leaderboard_matches (id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, start_observed, highest_round, last_event, last_fingerprint) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'game_end', 'test')`); - const player = handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?)`); + const player = handle.prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?)` + ); for (let id = 1; id <= 40; id++) { const start = new Date(now - id * 86400000).toISOString(); const end = new Date(now - id * 86400000 + id * 60000).toISOString(); const game = id % 2 ? 't5' : 't6'; const name = game === 't5' ? 'Kino Der Toten' : 'Town Survival'; const map = game === 't5' ? 'theater' : 'town'; - match.run(id, `${game}_zombie_${map}_1`, game, map, `${game}:${map}`, name, start, end, end, - id === 2 ? 'active' : id === 4 ? 'interrupted' : 'completed', id === 3 ? 0 : 1, id + 1); + match.run( + id, + `${game}_zombie_${map}_1`, + game, + map, + `${game}:${map}`, + name, + start, + end, + end, + id === 2 ? 'active' : id === 4 ? 'interrupted' : 'completed', + id === 3 ? 0 : 1, + id + 1 + ); for (let n = 0; n < id % 5; n++) player.run(id, String(n), start, end); } - handle.prepare("UPDATE leaderboard_matches SET server_deleted_at = ? WHERE id = 1").run(new Date(now).toISOString()); + handle + .prepare('UPDATE leaderboard_matches SET server_deleted_at = ? WHERE id = 1') + .run(new Date(now).toISOString()); }); afterEach(async () => fixture.cleanup()); const query = (value = '') => readMatchQuery(new URLSearchParams(value)); @@ -28,16 +50,27 @@ it('sorts the whole history before paging, keeps ties deterministic, and clamps const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); const first = serverMatchHistory(undefined, 25, query()); const second = serverMatchHistory(undefined, 25, query('page=2')); - expect(first.total).toBe(40); expect(first.matches.map(m => m.id)).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)); - expect(second.matches.map(m => m.id)).toEqual(Array.from({ length: 15 }, (_, i) => i + 26)); + expect(first.total).toBe(40); + expect(first.matches.map((m) => m.id)).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)); + expect(second.matches.map((m) => m.id)).toEqual(Array.from({ length: 15 }, (_, i) => i + 26)); expect(serverMatchHistory(undefined, 25, query('page=999')).page).toBe(2); expect(serverMatchHistory(undefined, 25, query('sort=round')).matches[0].id).toBe(40); - expect(serverMatchHistory(undefined, 25, query('sort=players')).matches.slice(0, 3).map(m => m.id)).toEqual([39, 34, 29]); + expect( + serverMatchHistory(undefined, 25, query('sort=players')) + .matches.slice(0, 3) + .map((m) => m.id) + ).toEqual([39, 34, 29]); }); it('combines game, map, readable instance search, status and date filters before counting', async () => { const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); - const data = serverMatchHistory(undefined, 25, query('game=t6&map=t6%3Atown&q=Town+Lobby+1&status=completed&days=7'), now); - expect(data.total).toBe(1); expect(data.matches.map(m => m.id)).toEqual([6]); + const data = serverMatchHistory( + undefined, + 25, + query('game=t6&map=t6%3Atown&q=Town+Lobby+1&status=completed&days=7'), + now + ); + expect(data.total).toBe(1); + expect(data.matches.map((m) => m.id)).toEqual([6]); expect(data.maps).toEqual([{ key: 't6:town', name: 'Town Survival', game: 't6' }]); expect(serverMatchHistory(undefined, 25, query('q=%25')).total).toBe(0); expect(serverMatchHistory('t5_zombie_theater_1', 25, query()).total).toBe(20); @@ -47,19 +80,43 @@ it('combines game, map, readable instance search, status and date filters before it('keeps unknown durations last in either direction and retains historical and unfinished semantics', async () => { const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js'); for (const direction of ['asc', 'desc']) { - const data = serverMatchHistory(undefined, 100, query(`sort=duration&direction=${direction}`)); - expect(data.matches.slice(-3).map(m => m.durationMs)).toEqual([null, null, null]); + const data = serverMatchHistory( + undefined, + 100, + query(`sort=duration&direction=${direction}`) + ); + expect(data.matches.slice(-3).map((m) => m.durationMs)).toEqual([null, null, null]); } const data = serverMatchHistory(); expect(data.matches[0].server_deleted_at).toBeTruthy(); - expect(data.matches[1]).toMatchObject({ status: 'active', observedRound: 3, finalRound: null, durationMs: null }); - expect(data.matches[3]).toMatchObject({ status: 'interrupted', observedRound: 5, finalRound: null, durationMs: null }); - expect(data.matches.find(m => m.id === 5)?.uniquePlayers).toBe(0); + expect(data.matches[1]).toMatchObject({ + status: 'active', + observedRound: 3, + finalRound: null, + durationMs: null + }); + expect(data.matches[3]).toMatchObject({ + status: 'interrupted', + observedRound: 5, + finalRound: null, + durationMs: null + }); + expect(data.matches.find((m) => m.id === 5)?.uniquePlayers).toBe(0); }); it('validates both API routes and loads the same filtered page on a direct page visit', async () => { const { GET } = await import('../../src/routes/api/matches/+server.js'); - const { GET: instanceGET } = await import('../../src/routes/api/servers/[name]/matches/+server.js'); - for (const value of ['page=-1', 'page=1.5', 'sort=sql', 'direction=sideways', 'game=t7', 'days=3', 'status=deleted', 'limit=0']) { + const { GET: instanceGET } = + await import('../../src/routes/api/servers/[name]/matches/+server.js'); + for (const value of [ + 'page=-1', + 'page=1.5', + 'sort=sql', + 'direction=sideways', + 'game=t7', + 'days=3', + 'status=deleted', + 'limit=0' + ]) { const url = new URL(`http://localhost/api/matches?${value}`); expect(GET({ url }).status).toBe(400); expect(instanceGET({ url, params: { name: 'anything' } }).status).toBe(400); @@ -67,14 +124,22 @@ it('validates both API routes and loads the same filtered page on a direct page const { load } = await import('../../src/routes/matches/+page.server.js'); const url = new URL('http://localhost/matches?matchPage=2&matchSort=round'); const loaded = load({ url }).history; - const body = await GET({ url: new URL(`http://localhost/api/matches?${matchApiParams(url)}`) }).json(); - expect(loaded).toEqual(body); expect(body.matches[0].id).toBe(15); + const body = await GET({ + url: new URL(`http://localhost/api/matches?${matchApiParams(url)}`) + }).json(); + expect(loaded).toEqual(body); + expect(body.matches[0].id).toBe(15); }); it('keeps return context and sorting while resetting page for a filter change', () => { - const url = new URL('http://localhost/servers/t5_zombie_theater_1?tab=matches&returnTo=%2Fhosts&matchSort=duration&matchDirection=asc&matchPage=3&matches=100'); + const url = new URL( + 'http://localhost/servers/t5_zombie_theater_1?tab=matches&returnTo=%2Fhosts&matchSort=duration&matchDirection=asc&matchPage=3&matches=100' + ); const href = matchHistoryHref(url, { status: 'completed' }); - expect(href).toContain('tab=matches'); expect(href).toContain('returnTo=%2Fhosts'); - expect(href).toContain('matchSort=duration'); expect(href).toContain('matchDirection=asc'); - expect(href).not.toContain('matchPage'); expect(href).not.toContain('matches=100'); + expect(href).toContain('tab=matches'); + expect(href).toContain('returnTo=%2Fhosts'); + expect(href).toContain('matchSort=duration'); + expect(href).toContain('matchDirection=asc'); + expect(href).not.toContain('matchPage'); + expect(href).not.toContain('matches=100'); expect(matchHistoryHref(url, { page: '4' })).toContain('matchPage=4'); }); diff --git a/tests/unit/metrics-insights.test.ts b/tests/unit/metrics-insights.test.ts index dbe2650..679e513 100644 --- a/tests/unit/metrics-insights.test.ts +++ b/tests/unit/metrics-insights.test.ts @@ -3,20 +3,38 @@ import { metricRecommendations } from '../../src/lib/metrics-insights.js'; import type { MapMetrics } from '../../src/lib/metrics.js'; const map = (values: Partial = {}): MapMetrics => ({ - game: 't5', map: 'Moon', observed: 10, active: 5, idle: 5, unknown: 0, - playerHours: 10, full: 0, capacityObserved: 5, ...values + game: 't5', + map: 'Moon', + observed: 10, + active: 5, + idle: 5, + unknown: 0, + playerHours: 10, + full: 0, + capacityObserved: 5, + ...values }); describe('advisory hosting experiments', () => { it('uses map capacity time rather than summed server time for expansion', () => { - const [result] = metricRecommendations([map({ observed: 100, active: 50, idle: 50, full: 1, capacityObserved: 4 })]); + const [result] = metricRecommendations([ + map({ observed: 100, active: 50, idle: 50, full: 1, capacityObserved: 4 }) + ]); expect(result.kind).toBe('expand'); expect(result.evidence).toContain('25.0%'); expect(result.tentative).toBe(false); }); it('offers a tentative trial even with a tiny recorded sample', () => { - const [result] = metricRecommendations([map({ observed: 1 / 120, active: 1 / 120, idle: 0, full: 1 / 120, capacityObserved: 1 / 120 })]); + const [result] = metricRecommendations([ + map({ + observed: 1 / 120, + active: 1 / 120, + idle: 0, + full: 1 / 120, + capacityObserved: 1 / 120 + }) + ]); expect(result.kind).toBe('expand'); expect(result.tentative).toBe(true); expect(result.evidence).toContain('30 s'); @@ -30,21 +48,39 @@ describe('advisory hosting experiments', () => { }); it('does not confuse unknown occupancy with an idle fleet', () => { - const [result] = metricRecommendations([map({ observed: 0, active: 0, idle: 0, unknown: 10, playerHours: 0, capacityObserved: 0 })]); + const [result] = metricRecommendations([ + map({ + observed: 0, + active: 0, + idle: 0, + unknown: 10, + playerHours: 0, + capacityObserved: 0 + }) + ]); expect(result.kind).toBe('capacity'); expect(result.title).toBe('Check occupancy reporting'); expect(result.tentative).toBe(true); }); it('keeps advice scoped to input maps and considers idle trials with incomplete capacity', () => { - const [result] = metricRecommendations([map({ game: 't6', map: 'Town', active: 1, idle: 9, capacityObserved: 0 })]); - expect(result).toMatchObject({ game: 't6', map: 'Town', kind: 'consolidate', tentative: true }); + const [result] = metricRecommendations([ + map({ game: 't6', map: 'Town', active: 1, idle: 9, capacityObserved: 0 }) + ]); + expect(result).toMatchObject({ + game: 't6', + map: 'Town', + kind: 'consolidate', + tentative: true + }); expect(result.reason).toContain('preserve access'); expect(metricRecommendations([])).toEqual([]); }); it('distinguishes a zero-duration sample from failed occupancy reporting', () => { - const [result] = metricRecommendations([map({ observed: 0, active: 0, idle: 0, playerHours: 0, capacityObserved: 0 })]); + const [result] = metricRecommendations([ + map({ observed: 0, active: 0, idle: 0, playerHours: 0, capacityObserved: 0 }) + ]); expect(result.title).toBe('Let observations accumulate'); expect(result.reason).toContain('no elapsed observations'); }); diff --git a/tests/unit/metrics-refresh.test.ts b/tests/unit/metrics-refresh.test.ts index f43b37f..997b69f 100644 --- a/tests/unit/metrics-refresh.test.ts +++ b/tests/unit/metrics-refresh.test.ts @@ -3,23 +3,60 @@ import { createMetricsRefresh } from '../../src/lib/ui/metrics-refresh.js'; afterEach(() => vi.useRealTimers()); const totals = { observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 }; -const report = (days: number) => ({ days, now: 1000, since: 0, summary: totals, points: [], maps: [], servers: [], - scope: { game: null, map: null, timeZone: 'UTC' }, scopeOptions: { games: [], maps: [] }, - registeredServerCount: 0, firstRetainedSampleAt: null, latestSampleAt: null, scopeFirstSampleAt: null, scopeLatestSampleAt: null, - previous: { since: -1000, until: 0, summary: totals, firstSampleAt: null, lastSampleAt: null, partialHistory: true }, +const report = (days: number) => ({ + days, + now: 1000, + since: 0, + summary: totals, + points: [], + maps: [], + servers: [], + scope: { game: null, map: null, timeZone: 'UTC' }, + scopeOptions: { games: [], maps: [] }, + registeredServerCount: 0, + firstRetainedSampleAt: null, + latestSampleAt: null, + scopeFirstSampleAt: null, + scopeLatestSampleAt: null, + previous: { + since: -1000, + until: 0, + summary: totals, + firstSampleAt: null, + lastSampleAt: null, + partialHistory: true + }, heatmap: { timeZone: 'UTC', cells: [] }, pooledPressure: { value: null, fullMapHours: 0, knownMapHours: 0, coveredMaps: 0, totalMaps: 0 } }); function setup(fetcher: typeof fetch) { let query = 'days=7'; - const onData = vi.fn(), onError = vi.fn(), onLoading = vi.fn(); - return { ...createMetricsRefresh({ getQuery: () => query, onData, onError, onLoading, fetcher, timeoutMs: 100 }), - onData, onError, onLoading, setQuery: (value: string) => { query = value; } }; + const onData = vi.fn(), + onError = vi.fn(), + onLoading = vi.fn(); + return { + ...createMetricsRefresh({ + getQuery: () => query, + onData, + onError, + onLoading, + fetcher, + timeoutMs: 100 + }), + onData, + onError, + onLoading, + setQuery: (value: string) => { + query = value; + } + }; } describe('metrics background refresh', () => { it('keeps the previous report after failure and accepts a successful retry', async () => { - const fetcher = vi.fn().mockResolvedValueOnce(new Response('', { status: 503 })) + const fetcher = vi + .fn() + .mockResolvedValueOnce(new Response('', { status: 503 })) .mockResolvedValueOnce(Response.json(report(7))); const refresh = setup(fetcher); await refresh.refresh(); @@ -33,7 +70,7 @@ describe('metrics background refresh', () => { it('coalesces duplicate requests and ignores old replies even when transport ignores abort', async () => { const resolve: ((r: Response) => void)[] = []; - const fetcher = vi.fn(() => new Promise(done => resolve.push(done))); + const fetcher = vi.fn(() => new Promise((done) => resolve.push(done))); const refresh = setup(fetcher); const old = refresh.refresh(); expect(refresh.refresh()).toBe(old); @@ -49,8 +86,15 @@ describe('metrics background refresh', () => { }); it('preserves the report when an HTTP success lacks the expanded report contract', async () => { - const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ...report(7), heatmap: null })) - .mockResolvedValueOnce(Response.json({ ...report(7), scope: { game: null, map: null, timeZone: 'Invalid/Zone' } })) + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json({ ...report(7), heatmap: null })) + .mockResolvedValueOnce( + Response.json({ + ...report(7), + scope: { game: null, map: null, timeZone: 'Invalid/Zone' } + }) + ) .mockResolvedValueOnce(Response.json(report(7))); const refresh = setup(fetcher); await refresh.refresh(); @@ -63,7 +107,14 @@ describe('metrics background refresh', () => { it('suppresses errors from a cancelled request', async () => { let reject!: (error: Error) => void; - const refresh = setup(vi.fn(() => new Promise((_, fail) => { reject = fail; }))); + const refresh = setup( + vi.fn( + () => + new Promise((_, fail) => { + reject = fail; + }) + ) + ); const pending = refresh.refresh(); refresh.cancel(); reject(new Error('Obsolete response')); @@ -75,9 +126,12 @@ describe('metrics background refresh', () => { it('times out with retry feedback and clears loading', async () => { vi.useFakeTimers(); - const fetcher = vi.fn((_input, init) => new Promise((_, reject) => { - init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); - })) as typeof fetch; + const fetcher = vi.fn( + (_input, init) => + new Promise((_, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }) + ) as typeof fetch; const refresh = setup(fetcher); const pending = refresh.refresh(); await vi.advanceTimersByTimeAsync(100); diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts index 96f510b..c488601 100644 --- a/tests/unit/metrics.test.ts +++ b/tests/unit/metrics.test.ts @@ -3,12 +3,19 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { parseMetricsQuery } from '../../src/lib/metrics.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); describe('historical metrics', () => { it('weights occupancy by time and excludes failed checks and collection gaps', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }; const at = Date.UTC(2026, 0, 2, 12); recordMetrics([sample], at); @@ -19,11 +26,14 @@ describe('historical metrics', () => { expect(result.summary.idle * 3600).toBeCloseTo(30); expect(result.summary.unknown * 3600).toBeCloseTo(90); expect(result.summary.playerHours * 3600).toBeCloseTo(60); - expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed); + expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo( + result.summary.observed + ); }); it('only counts map saturation when every lobby has known capacity and is full', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 2, 12); const sample = { server: 'one', game: 't5', map: 'Moon', players: 4, capacity: 4 }; recordMetrics([sample, { ...sample, server: 'two' }], at); @@ -36,19 +46,25 @@ describe('historical metrics', () => { }); it('clips the reporting window and splits observations across UTC chart boundaries', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const midnight = Date.UTC(2026, 0, 2); const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 }; recordMetrics([sample], midnight - 15_000); let result = readMetrics(1, midnight + 15_000); - expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]); + expect(result.points.filter((p) => p.active > 0).map((p) => p.active * 3600)).toEqual([ + 15, 15 + ]); result = readMetrics(1, midnight + 86400_000); expect(result.summary.active * 3600).toBeCloseTo(75); - expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active); + expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo( + result.summary.active + ); }); it('retains history without a live server and prunes observations older than 31 days', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const { db } = await import('../../src/lib/server/db/index.js'); const at = Date.UTC(2026, 0, 2); const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null }; @@ -59,29 +75,55 @@ describe('historical metrics', () => { }); it('distinguishes retained history, an empty range, and current collection targets', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const { db } = await import('../../src/lib/server/db/index.js'); const at = Date.UTC(2026, 0, 2); - expect(readMetrics(1, at)).toMatchObject({ firstRetainedSampleAt: null, latestSampleAt: null, - registeredServerCount: 0, servers: [], maps: [] }); + expect(readMetrics(1, at)).toMatchObject({ + firstRetainedSampleAt: null, + latestSampleAt: null, + registeredServerCount: 0, + servers: [], + maps: [] + }); db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('current', null); - db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('removed', new Date(at).toISOString()); + db() + .prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)') + .run('removed', new Date(at).toISOString()); const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null }; recordMetrics([sample], at); const emptyRange = readMetrics(1, at + 2 * 86400_000); - expect(emptyRange).toMatchObject({ firstRetainedSampleAt: at, latestSampleAt: at, - registeredServerCount: 1, servers: [], maps: [], summary: { observed: 0, unknown: 0 } }); - expect(emptyRange.points.every(point => point.observed === 0 && point.unknown === 0)).toBe(true); + expect(emptyRange).toMatchObject({ + firstRetainedSampleAt: at, + latestSampleAt: at, + registeredServerCount: 1, + servers: [], + maps: [], + summary: { observed: 0, unknown: 0 } + }); + expect( + emptyRange.points.every((point) => point.observed === 0 && point.unknown === 0) + ).toBe(true); }); it('keeps unknown status separate from no samples and preserves zero elapsed observations', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 2, 12); - recordMetrics([{ server: 'one', game: 't5', map: 'Moon', players: null, capacity: null }], at); + recordMetrics( + [{ server: 'one', game: 't5', map: 'Moon', players: null, capacity: null }], + at + ); const initial = readMetrics(1, at); expect(initial.latestSampleAt).toBe(at); expect(initial.servers).toHaveLength(1); - expect(initial.summary).toEqual({ observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 }); + expect(initial.summary).toEqual({ + observed: 0, + active: 0, + idle: 0, + unknown: 0, + playerHours: 0 + }); const later = readMetrics(1, at + 120_000); expect(later.summary.observed).toBe(0); expect(later.summary.unknown * 3600).toBeCloseTo(90); @@ -97,14 +139,19 @@ describe('historical metrics', () => { expect(result.points.at(-1)?.endAt).toBe(now); expect(result.points[0].partial).toBe(true); expect(result.points.at(-1)?.partial).toBe(true); - expect(result.points.slice(1, -1).every(point => !point.partial)).toBe(true); - expect(result.points.reduce((ms, point) => ms + point.endAt - point.startAt, 0)).toBe(days * 86400_000); + expect(result.points.slice(1, -1).every((point) => !point.partial)).toBe(true); + expect(result.points.reduce((ms, point) => ms + point.endAt - point.startAt, 0)).toBe( + days * 86400_000 + ); } - expect(readMetrics(1, Date.UTC(2026, 0, 8, 12)).points.every(point => !point.partial)).toBe(true); + expect( + readMetrics(1, Date.UTC(2026, 0, 8, 12)).points.every((point) => !point.partial) + ).toBe(true); }); it('uses complete collection boundaries across fleet changes and keeps map pressure in wall time', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 2, 23, 59, 40); const one = { server: 'one', game: 't6', map: 'Town', players: 4, capacity: 4 }; const two = { ...one, server: 'two' }; @@ -117,21 +164,30 @@ describe('historical metrics', () => { expect(result.summary.active * 3600).toBeCloseTo(90); expect(result.summary.idle * 3600).toBeCloseTo(90); expect(result.summary.playerHours * 3600).toBeCloseTo(300); - expect(result.servers.find(server => server.server === 'two')!.observed * 3600).toBeCloseTo(30); - const town = result.maps.find(map => map.game === 't6' && map.map === 'Town')!; + expect( + result.servers.find((server) => server.server === 'two')!.observed * 3600 + ).toBeCloseTo(30); + const town = result.maps.find((map) => map.game === 't6' && map.map === 'Town')!; expect(town.observed * 3600).toBeCloseTo(60); expect(town.capacityObserved * 3600).toBeCloseTo(30); expect(town.full * 3600).toBeCloseTo(30); - expect(result.maps.find(map => map.game === 't5')!.capacityObserved).toBe(0); + expect(result.maps.find((map) => map.game === 't5')!.capacityObserved).toBe(0); for (const field of ['observed', 'active', 'idle', 'unknown', 'playerHours'] as const) { - expect(result.points.reduce((sum, point) => sum + point[field], 0)).toBeCloseTo(result.summary[field]); - expect(result.servers.reduce((sum, server) => sum + server[field], 0)).toBeCloseTo(result.summary[field]); - expect(result.maps.reduce((sum, map) => sum + map[field], 0)).toBeCloseTo(result.summary[field]); + expect(result.points.reduce((sum, point) => sum + point[field], 0)).toBeCloseTo( + result.summary[field] + ); + expect(result.servers.reduce((sum, server) => sum + server[field], 0)).toBeCloseTo( + result.summary[field] + ); + expect(result.maps.reduce((sum, map) => sum + map[field], 0)).toBeCloseTo( + result.summary[field] + ); } }); it('does not include future samples in collection metadata or aggregates', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 2, 12); const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 }; recordMetrics([sample], at); @@ -156,8 +212,14 @@ describe('historical metrics', () => { expect(payload).toEqual(await load({ url } as Parameters[0])); expect(payload.days).toBe(['1', '7', '30'].includes(range) ? Number(range) : 7); } - expect((await (await GET({ url: new URL('http://localhost/api/metrics') })).json()).days).toBe(7); - for (const search of ['days=1&game=t5&map=Moon&timeZone=Asia%2FKathmandu', 'days=7&map=Moon', 'game=t6&timeZone=Invalid']) { + expect( + (await (await GET({ url: new URL('http://localhost/api/metrics') })).json()).days + ).toBe(7); + for (const search of [ + 'days=1&game=t5&map=Moon&timeZone=Asia%2FKathmandu', + 'days=7&map=Moon', + 'game=t6&timeZone=Invalid' + ]) { const url = new URL(`http://localhost/metrics?${search}`); const payload = await (await GET({ url })).json(); const { days, ...scope } = parseMetricsQuery(url.searchParams); @@ -167,29 +229,48 @@ describe('historical metrics', () => { }); it('normalizes malformed scopes consistently and requires a game for a map selection', () => { - expect(parseMetricsQuery(new URLSearchParams('days=365&map=Moon&timeZone=Invalid/Zone'))) - .toEqual({ days: 7, game: null, map: null, timeZone: 'America/New_York' }); - expect(parseMetricsQuery(new URLSearchParams({ game: 't5; DROP TABLE', map: 'Moon' }))).toMatchObject({ game: null, map: null }); - expect(parseMetricsQuery(new URLSearchParams({ game: ' t5 ', map: 'Moon', timeZone: 'Asia/Kathmandu', days: '30' }))) - .toEqual({ days: 30, game: 't5', map: 'Moon', timeZone: 'Asia/Kathmandu' }); + expect( + parseMetricsQuery(new URLSearchParams('days=365&map=Moon&timeZone=Invalid/Zone')) + ).toEqual({ days: 7, game: null, map: null, timeZone: 'America/New_York' }); + expect( + parseMetricsQuery(new URLSearchParams({ game: 't5; DROP TABLE', map: 'Moon' })) + ).toMatchObject({ game: null, map: null }); + expect( + parseMetricsQuery( + new URLSearchParams({ + game: ' t5 ', + map: 'Moon', + timeZone: 'Asia/Kathmandu', + days: '30' + }) + ) + ).toEqual({ days: 30, game: 't5', map: 'Moon', timeZone: 'Asia/Kathmandu' }); for (const map of ['', ' ', 'a'.repeat(201), 'Moon\nTown']) { expect(parseMetricsQuery(new URLSearchParams({ game: 't5', map })).map).toBeNull(); } - expect(parseMetricsQuery(new URLSearchParams({ game: 't5', map: 'Not recorded yet' })).map).toBe('Not recorded yet'); + expect( + parseMetricsQuery(new URLSearchParams({ game: 't5', map: 'Not recorded yet' })).map + ).toBe('Not recorded yet'); }); it('filters every derived value by exact game and map while preserving unfiltered collection boundaries', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const now = Date.UTC(2026, 0, 3, 12); const since = now - 86400_000; const moon = { server: 'one', game: 't5', map: 'Moon', players: 4, capacity: 4 }; const other = { ...moon, server: 'other', game: 't6', players: 2 }; recordMetrics([moon, other], since - 30_000); recordMetrics([other], since); - recordMetrics([moon, other, { ...moon, server: 'town', map: 'Town', players: 0 }], now - 60_000); + recordMetrics( + [moon, other, { ...moon, server: 'town', map: 'Town', players: 0 }], + now - 60_000 + ); recordMetrics([other], now - 45_000); recordMetrics([{ ...moon, players: 0 }, other], now - 30_000); - const query = parseMetricsQuery(new URLSearchParams({ days: '1', game: 't5', map: 'Moon', timeZone: 'UTC' })); + const query = parseMetricsQuery( + new URLSearchParams({ days: '1', game: 't5', map: 'Moon', timeZone: 'UTC' }) + ); const result = readMetrics(query, now); expect(result.scope).toEqual({ game: 't5', map: 'Moon', timeZone: 'UTC' }); expect(result.summary.active * 3600).toBeCloseTo(15); @@ -197,35 +278,54 @@ describe('historical metrics', () => { expect(result.summary.playerHours * 3600).toBeCloseTo(60); expect(result.previous.summary.active * 3600).toBeCloseTo(30); expect(result.previous.summary.playerHours * 3600).toBeCloseTo(120); - expect(result.servers.map(server => server.server)).toEqual(['one']); - expect(result.maps.map(map => [map.game, map.map])).toEqual([['t5', 'Moon']]); - expect(result.scopeOptions).toEqual({ games: ['t5', 't6'], maps: [ - { game: 't5', map: 'Moon' }, { game: 't5', map: 'Town' }, { game: 't6', map: 'Moon' } - ] }); + expect(result.servers.map((server) => server.server)).toEqual(['one']); + expect(result.maps.map((map) => [map.game, map.map])).toEqual([['t5', 'Moon']]); + expect(result.scopeOptions).toEqual({ + games: ['t5', 't6'], + maps: [ + { game: 't5', map: 'Moon' }, + { game: 't5', map: 'Town' }, + { game: 't6', map: 'Moon' } + ] + }); expect(result.scopeFirstSampleAt).toBe(since - 30_000); expect(result.scopeLatestSampleAt).toBe(now - 30_000); expect(result.pooledPressure.value).toBeCloseTo(100 / 3); for (const field of ['observed', 'active', 'idle', 'unknown', 'playerHours'] as const) { - expect(result.points.reduce((sum, point) => sum + point[field], 0)).toBeCloseTo(result.summary[field]); - expect(result.heatmap.cells.reduce((sum, cell) => sum + cell[field], 0)).toBeCloseTo(result.summary[field]); + expect(result.points.reduce((sum, point) => sum + point[field], 0)).toBeCloseTo( + result.summary[field] + ); + expect(result.heatmap.cells.reduce((sum, cell) => sum + cell[field], 0)).toBeCloseTo( + result.summary[field] + ); } const gameOnly = readMetrics({ ...query, map: null }, now); - expect(gameOnly.maps.map(map => map.map).sort()).toEqual(['Moon', 'Town']); + expect(gameOnly.maps.map((map) => map.map).sort()).toEqual(['Moon', 'Town']); const missing = readMetrics({ ...query, map: 'Unrecorded' }, now); - expect(missing).toMatchObject({ scopeFirstSampleAt: null, scopeLatestSampleAt: null, - summary: { observed: 0, unknown: 0 }, maps: [], servers: [], pooledPressure: { value: null }, - previous: { firstSampleAt: null, lastSampleAt: null, partialHistory: true } }); + expect(missing).toMatchObject({ + scopeFirstSampleAt: null, + scopeLatestSampleAt: null, + summary: { observed: 0, unknown: 0 }, + maps: [], + servers: [], + pooledPressure: { value: null }, + previous: { firstSampleAt: null, lastSampleAt: null, partialHistory: true } + }); expect(missing.scopeOptions).toEqual(result.scopeOptions); }); it('separates scoped evidence timestamps from global collection timestamps', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 3, 12); const sample = { server: 'one', game: 't5', map: 'Moon', players: 1, capacity: 4 }; recordMetrics([{ ...sample, game: 't6' }], at - 60_000); recordMetrics([sample], at - 30_000); recordMetrics([{ ...sample, game: 't6' }], at); - const result = readMetrics(parseMetricsQuery(new URLSearchParams({ game: 't5' })), at + 30_000); + const result = readMetrics( + parseMetricsQuery(new URLSearchParams({ game: 't5' })), + at + 30_000 + ); expect(result.firstRetainedSampleAt).toBe(at - 60_000); expect(result.latestSampleAt).toBe(at); expect(result.scopeFirstSampleAt).toBe(at - 30_000); @@ -233,7 +333,8 @@ describe('historical metrics', () => { }); it('compares adjacent equal windows and splits an observation at their shared boundary', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const now = Date.UTC(2026, 0, 3, 12); const since = now - 86400_000; const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }; @@ -245,11 +346,16 @@ describe('historical metrics', () => { expect(result.previous.summary.active * 3600).toBeCloseTo(15); expect(result.summary.active * 3600).toBeCloseTo(15); expect(result.summary.idle * 3600).toBeCloseTo(90); - expect(result.previous).toMatchObject({ firstSampleAt: since - 15_000, lastSampleAt: since - 15_000, partialHistory: true }); + expect(result.previous).toMatchObject({ + firstSampleAt: since - 15_000, + lastSampleAt: since - 15_000, + partialHistory: true + }); }); it('returns the available prior evidence when a thirty-day comparison exceeds retained history', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const now = Date.UTC(2026, 2, 3, 12); const day = 86400_000; const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }; @@ -257,7 +363,11 @@ describe('historical metrics', () => { recordMetrics([{ ...sample, players: 0 }], now - 30 * day + 30_000); recordMetrics([sample], now - 30_000); const result = readMetrics(30, now); - expect(result.previous).toMatchObject({ since: now - 60 * day, until: now - 30 * day, partialHistory: true }); + expect(result.previous).toMatchObject({ + since: now - 60 * day, + until: now - 30 * day, + partialHistory: true + }); expect(result.previous.summary.active * 3600).toBeCloseTo(90); expect(result.previous.firstSampleAt).toBe(now - 31 * day + 30_000); expect(result.summary.active * 3600).toBeCloseTo(30); @@ -265,7 +375,8 @@ describe('historical metrics', () => { }); it('marks the retained start bound separately from gaps inside the previous window', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const now = Date.UTC(2026, 0, 3, 12); const priorStart = now - 2 * 86400_000; const sample = { server: 'one', game: 't5', map: 'Moon', players: null, capacity: null }; @@ -281,19 +392,33 @@ describe('historical metrics', () => { }); it('pools pressure by known map-hours rather than server-hours or fleet elapsed time', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.UTC(2026, 0, 3, 12); const full = { server: 'one', game: 't5', map: 'Moon', players: 4, capacity: 4 }; - recordMetrics([full, { ...full, server: 'two' }, { ...full, server: 'three', map: 'Town', players: 0 }, - { ...full, server: 'four', map: 'Farm', players: 2, capacity: null }], at); + recordMetrics( + [ + full, + { ...full, server: 'two' }, + { ...full, server: 'three', map: 'Town', players: 0 }, + { ...full, server: 'four', map: 'Farm', players: 2, capacity: null } + ], + at + ); const result = readMetrics(1, at + 30_000); expect(result.pooledPressure).toMatchObject({ value: 50, coveredMaps: 2, totalMaps: 3 }); expect(result.pooledPressure.fullMapHours * 3600).toBeCloseTo(30); expect(result.pooledPressure.knownMapHours * 3600).toBeCloseTo(60); expect(result.summary.observed * 3600).toBeCloseTo(120); - const empty = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', game: 't5', map: 'Town' })), at + 30_000); + const empty = readMetrics( + parseMetricsQuery(new URLSearchParams({ days: '1', game: 't5', map: 'Town' })), + at + 30_000 + ); expect(empty.pooledPressure.value).toBe(0); - const missing = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', game: 't5', map: 'Farm' })), at + 30_000); + const missing = readMetrics( + parseMetricsQuery(new URLSearchParams({ days: '1', game: 't5', map: 'Farm' })), + at + 30_000 + ); expect(missing.pooledPressure.value).toBeNull(); }); @@ -302,52 +427,86 @@ describe('historical metrics', () => { { zone: 'Asia/Kolkata', at: '2026-01-01T18:29:45Z', first: [3, 23], second: [4, 0] }, { zone: 'Asia/Kathmandu', at: '2026-01-01T18:14:45Z', first: [3, 23], second: [4, 0] }, { zone: 'Australia/Lord_Howe', at: '2026-10-03T15:29:45Z', first: [6, 1], second: [6, 2] } - ])('splits heatmap time across local boundaries in $zone', async ({ zone, at: timestamp, first, second }) => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); - const at = Date.parse(timestamp); - recordMetrics([{ server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }], at); - const result = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: zone })), at + 30_000); - expect(result.heatmap.timeZone).toBe(zone); - expect(result.heatmap.cells).toHaveLength(168); - for (const [weekday, hour] of [first, second]) { - const cell = result.heatmap.cells.find(cell => cell.weekday === weekday && cell.hour === hour)!; - expect(cell.observed * 3600).toBeCloseTo(15); - expect(cell.playerHours * 3600).toBeCloseTo(30); - expect(cell.distinctDates).toBe(1); + ])( + 'splits heatmap time across local boundaries in $zone', + async ({ zone, at: timestamp, first, second }) => { + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); + const at = Date.parse(timestamp); + recordMetrics( + [{ server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }], + at + ); + const result = readMetrics( + parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: zone })), + at + 30_000 + ); + expect(result.heatmap.timeZone).toBe(zone); + expect(result.heatmap.cells).toHaveLength(168); + for (const [weekday, hour] of [first, second]) { + const cell = result.heatmap.cells.find( + (cell) => cell.weekday === weekday && cell.hour === hour + )!; + expect(cell.observed * 3600).toBeCloseTo(15); + expect(cell.playerHours * 3600).toBeCloseTo(30); + expect(cell.distinctDates).toBe(1); + } + expect(result.heatmap.cells.reduce((sum, cell) => sum + cell.observed, 0)).toBeCloseTo( + result.summary.observed + ); } - expect(result.heatmap.cells.reduce((sum, cell) => sum + cell.observed, 0)).toBeCloseTo(result.summary.observed); - }); + ); it('weights repeated fall-back hours and counts their local date once', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const at = Date.parse('2026-11-01T05:30:00Z'); const sample = { server: 'one', game: 't5', map: 'Moon', players: 2, capacity: 4 }; recordMetrics([sample], at); recordMetrics([{ ...sample, players: 0 }], at + 3600_000); - const result = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: 'America/New_York' })), at + 3630_000); - const cell = result.heatmap.cells.find(cell => cell.weekday === 6 && cell.hour === 1)!; + const result = readMetrics( + parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: 'America/New_York' })), + at + 3630_000 + ); + const cell = result.heatmap.cells.find((cell) => cell.weekday === 6 && cell.hour === 1)!; expect(cell.observed * 3600).toBeCloseTo(120); expect(cell.active * 3600).toBeCloseTo(90); expect(cell.idle * 3600).toBeCloseTo(30); expect(cell.distinctDates).toBe(1); - expect(result.heatmap.cells.filter(cell => cell.observed > 0)).toHaveLength(1); + expect(result.heatmap.cells.filter((cell) => cell.observed > 0)).toHaveLength(1); }); it('counts unknown heatmap evidence and distinguishes retained names from current registrations', async () => { - const { recordMetrics, readMetrics } = await import('../../src/lib/server/fleet/metrics.js'); + const { recordMetrics, readMetrics } = + await import('../../src/lib/server/fleet/metrics.js'); const { db } = await import('../../src/lib/server/db/index.js'); const at = Date.UTC(2026, 0, 3, 12); const sample = { server: 'reused', game: 't5', map: 'Moon', players: null, capacity: null }; - recordMetrics([sample, { ...sample, server: 'removed' }, { ...sample, server: 'tombstone' }], at); + recordMetrics( + [sample, { ...sample, server: 'removed' }, { ...sample, server: 'tombstone' }], + at + ); // A newly registered name can link to the current server without asserting // that its earlier historical observations belong to this registration. - db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run('reused', new Date(at + 15_000).toISOString()); - db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('tombstone', new Date(at).toISOString()); + db() + .prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)') + .run('reused', new Date(at + 15_000).toISOString()); + db() + .prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)') + .run('tombstone', new Date(at).toISOString()); const result = readMetrics(1, at + 30_000); - expect(result.servers.find(server => server.server === 'reused')!.currentRegistered).toBe(true); - expect(result.servers.filter(server => server.server !== 'reused').every(server => !server.currentRegistered)).toBe(true); - expect(result.heatmap.cells.reduce((sum, cell) => sum + cell.unknown, 0) * 3600).toBeCloseTo(90); + expect(result.servers.find((server) => server.server === 'reused')!.currentRegistered).toBe( + true + ); + expect( + result.servers + .filter((server) => server.server !== 'reused') + .every((server) => !server.currentRegistered) + ).toBe(true); + expect( + result.heatmap.cells.reduce((sum, cell) => sum + cell.unknown, 0) * 3600 + ).toBeCloseTo(90); expect(result.heatmap.cells.reduce((sum, cell) => sum + cell.observed, 0)).toBe(0); - expect(result.heatmap.cells.find(cell => cell.unknown > 0)!.distinctDates).toBe(1); + expect(result.heatmap.cells.find((cell) => cell.unknown > 0)!.distinctDates).toBe(1); }); }); diff --git a/tests/unit/migration-memory.test.ts b/tests/unit/migration-memory.test.ts index 6ad3921..ad9cd37 100644 --- a/tests/unit/migration-memory.test.ts +++ b/tests/unit/migration-memory.test.ts @@ -7,67 +7,103 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { enableIncrementalVacuum } from '../../src/lib/server/db/retention-clocks.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await fixture.cleanup(); +}); it('keeps a 256 MiB database rebuild below a 160 MiB increase in resident memory', async () => { const helper = fileURLToPath(new URL('../helpers/vacuum-memory.ts', import.meta.url)); - const { stdout } = await promisify(execFile)(process.execPath, ['--import', 'tsx', helper, fixture.dbPath], { timeout: 60000 }); + const { stdout } = await promisify(execFile)( + process.execPath, + ['--import', 'tsx', helper, fixture.dbPath], + { timeout: 60000 } + ); const result = JSON.parse(stdout.trim().split('\n').at(-1)!); expect(result).toMatchObject({ vacuum: 2, integrity: 'ok', rows: { n: 256 } }); expect(result.peakKiB - result.beforeKiB).toBeLessThan(160 * 1024); }, 65000); -it.each(['before', 'after'])('recovers a rebuild interrupted %s copying without losing host preparation boundaries', mode => { - let handle = new Database(fixture.dbPath); - try { - handle.exec(`CREATE TABLE ec_host_operations(id TEXT,host_id TEXT); +it.each(['before', 'after'])( + 'recovers a rebuild interrupted %s copying without losing host preparation boundaries', + (mode) => { + let handle = new Database(fixture.dbPath); + try { + handle.exec(`CREATE TABLE ec_host_operations(id TEXT,host_id TEXT); CREATE TABLE ec_host_enable_requests(host_id TEXT,operation_floor INTEGER); INSERT INTO ec_host_operations(rowid,id,host_id) VALUES(10,'before','host'),(30,'after','host'); INSERT INTO ec_host_enable_requests VALUES('host',20)`); - handle.pragma('temp_store = MEMORY'); handle.pragma('cache_size = -131072'); - const exec = handle.exec.bind(handle); - const mock = vi.spyOn(handle, 'exec').mockImplementation(sql => { - if (sql === 'VACUUM') { - expect(handle.pragma('temp_store', { simple: true })).toBe(1); - expect(handle.pragma('cache_size', { simple: true })).toBe(-16384); - if (mode === 'after') exec(sql); - throw new Error('simulated interruption'); - } - return exec(sql); - }); - expect(() => enableIncrementalVacuum(handle)).toThrow('simulated interruption'); - mock.mockRestore(); - expect(handle.pragma('temp_store', { simple: true })).toBe(2); - expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); - handle.close(); handle = new Database(fixture.dbPath); - enableIncrementalVacuum(handle); - expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2); - expect(handle.prepare(`SELECT o.id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id - WHERE o.rowid>e.operation_floor`).all()).toEqual([{ id: 'after' }]); - expect(handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()).toBeUndefined(); - expect(handle.pragma('quick_check', { simple: true })).toBe('ok'); - } finally { handle.close(); } -}); + handle.pragma('temp_store = MEMORY'); + handle.pragma('cache_size = -131072'); + const exec = handle.exec.bind(handle); + const mock = vi.spyOn(handle, 'exec').mockImplementation((sql) => { + if (sql === 'VACUUM') { + expect(handle.pragma('temp_store', { simple: true })).toBe(1); + expect(handle.pragma('cache_size', { simple: true })).toBe(-16384); + if (mode === 'after') exec(sql); + throw new Error('simulated interruption'); + } + return exec(sql); + }); + expect(() => enableIncrementalVacuum(handle)).toThrow('simulated interruption'); + mock.mockRestore(); + expect(handle.pragma('temp_store', { simple: true })).toBe(2); + expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); + handle.close(); + handle = new Database(fixture.dbPath); + enableIncrementalVacuum(handle); + expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2); + expect( + handle + .prepare( + `SELECT o.id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id + WHERE o.rowid>e.operation_floor` + ) + .all() + ).toEqual([{ id: 'after' }]); + expect( + handle + .prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'") + .get() + ).toBeUndefined(); + expect(handle.pragma('quick_check', { simple: true })).toBe('ok'); + } finally { + handle.close(); + } + } +); it('uses migration settings before startup writes and restores runtime tuning only after the rebuild', async () => { const legacy = new Database(fixture.dbPath); - legacy.exec('CREATE TABLE legacy(id INTEGER PRIMARY KEY, content BLOB); INSERT INTO legacy(content) VALUES(zeroblob(1024))'); + legacy.exec( + 'CREATE TABLE legacy(id INTEGER PRIMARY KEY, content BLOB); INSERT INTO legacy(content) VALUES(zeroblob(1024))' + ); legacy.close(); vi.stubEnv('EC_SQLITE_MMAP_MB', '256'); const exec = Database.prototype.exec; - let rebuilt = false, migrated = false; - const mock = vi.spyOn(Database.prototype, 'exec').mockImplementation(function(this: Database.Database, sql) { + let rebuilt = false, + migrated = false; + const mock = vi.spyOn(Database.prototype, 'exec').mockImplementation(function ( + this: Database.Database, + sql + ) { if (sql === 'VACUUM' || sql.includes('CREATE TABLE IF NOT EXISTS file_blobs')) { expect(this.pragma('temp_store', { simple: true })).toBe(1); expect(this.pragma('cache_size', { simple: true })).toBe(-16384); expect(this.pragma('mmap_size', { simple: true })).toBe(0); - if (sql === 'VACUUM') rebuilt = true; else migrated = true; + if (sql === 'VACUUM') rebuilt = true; + else migrated = true; } return exec.call(this, sql); }); const { db } = await import('../../src/lib/server/db/index.js'); - const handle = db(); mock.mockRestore(); + const handle = db(); + mock.mockRestore(); expect(rebuilt && migrated).toBe(true); expect(handle.pragma('temp_store', { simple: true })).toBe(1); expect(handle.pragma('cache_size', { simple: true })).toBe(-131072); diff --git a/tests/unit/native-creation-intents.test.ts b/tests/unit/native-creation-intents.test.ts index c2a40d3..9a18e04 100644 --- a/tests/unit/native-creation-intents.test.ts +++ b/tests/unit/native-creation-intents.test.ts @@ -13,7 +13,11 @@ beforeEach(async () => { vi.stubEnv('RCON_PASSWORD', ''); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); async function intent() { const { db } = await import('../../src/lib/server/db/index.js'); @@ -22,10 +26,18 @@ async function intent() { const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); const id = beginDeployment(server, 'create'); db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id); - db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)') - .run(id, JSON.stringify({ automatic: true, libraryId: 'library', image: 'plutainer:test' })); - db().prepare(`INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) - VALUES(?,?,NULL,'zombie','theater','zombie_theater')`).run(id, server); + db() + .prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)') + .run( + id, + JSON.stringify({ automatic: true, libraryId: 'library', image: 'plutainer:test' }) + ); + db() + .prepare( + `INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) + VALUES(?,?,NULL,'zombie','theater','zombie_theater')` + ) + .run(id, server); return id; } @@ -33,8 +45,10 @@ describe('native creation intents before placement', () => { it('retains its identity and native history across restart without changing ownership', async () => { const id = await intent(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { getDeployment, interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); - const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js'); + const { getDeployment, interruptDeployments } = + await import('../../src/lib/server/provisioning/history.js'); + const { nativeDeployment, unplacedNativeCreation } = + await import('../../src/lib/server/backends/native.js'); const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); expect(nativeDeployment(id)).toBe(true); @@ -45,9 +59,15 @@ describe('native creation intents before placement', () => { interruptDeployments(); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning(); - expect(getDeployment(id)).toMatchObject({ backend: 'native', canCancelCreation: true, agentOperation: null, - deployment: { state: 'queued', server_exists: 0 } }); - expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null }); + expect(getDeployment(id)).toMatchObject({ + backend: 'native', + canCancelCreation: true, + agentOperation: null, + deployment: { state: 'queued', server_exists: 0 } + }); + expect( + db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id) + ).toEqual({ port: null }); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); expect(db().prepare('SELECT deployment_id FROM native_provisioning').all()).toEqual([]); }); @@ -55,10 +75,13 @@ describe('native creation intents before placement', () => { it('cancels unplaced work locally once, retains keys, and frees its logical server name', async () => { const id = await intent(); const { db } = await import('../../src/lib/server/db/index.js'); - const { reserveServerKey, getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js'); - const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { reserveServerKey, getServerKey, serverKeyPool } = + await import('../../src/lib/server/fleet/server-keys.js'); + const { cancelCreation, canCancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js'); + const { nativeDeployment, unplacedNativeCreation } = + await import('../../src/lib/server/backends/native.js'); const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js'); reserveServerKey(server, 't5', 'saved-key'); expect(canCancelCreation(id)).toBe(true); @@ -67,8 +90,15 @@ describe('native creation intents before placement', () => { expect(cancelCreation(id)).toBe(cleanup); expect(canCancelCreation(id)).toBe(false); expect(nativeDeployment(cleanup)).toBe(true); - expect(getDeployment(id)).toMatchObject({ backend: 'native', cancellation: { state: 'done' }, deployment: { state: 'cancelled' } }); - expect(getDeployment(cleanup)).toMatchObject({ backend: 'native', deployment: { state: 'done', server_deleted_at: expect.any(String) } }); + expect(getDeployment(id)).toMatchObject({ + backend: 'native', + cancellation: { state: 'done' }, + deployment: { state: 'cancelled' } + }); + expect(getDeployment(cleanup)).toMatchObject({ + backend: 'native', + deployment: { state: 'done', server_deleted_at: expect.any(String) } + }); expect(unplacedNativeCreation(server)).toBeNull(); expect(nextLobbyNumber('t5_zombie_theater')).toBe(1); expect(getServerKey(server)).toBeUndefined(); @@ -80,43 +110,72 @@ describe('native creation intents before placement', () => { it('will not cancel an unplaced historical intent against a replacement server', async () => { const id = await intent(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers(name) VALUES(?)").run(server); - const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + db().prepare('INSERT INTO servers(name) VALUES(?)').run(server); + const { cancelCreation, canCancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(canCancelCreation(id)).toBe(false); expect(cancelCreation(id)).toBeNull(); - expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server }); - expect(db().prepare('SELECT cleanup_id FROM native_creation_cancellations').all()).toEqual([]); + expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ + name: server + }); + expect(db().prepare('SELECT cleanup_id FROM native_creation_cancellations').all()).toEqual( + [] + ); }); - it.each([false, true])('does not adopt a replacement instance during placement (previously bound: %s)', async previouslyBound => { - const id = await intent(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { createHost } = await import('../../src/lib/server/hosts/index.js'); - const { captureRecipePlan, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const hostId = createHost({ name: 'Replacement host' }).id; - const insert = db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at) + it.each([false, true])( + 'does not adopt a replacement instance during placement (previously bound: %s)', + async (previouslyBound) => { + const id = await intent(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { createHost } = await import('../../src/lib/server/hosts/index.js'); + const { captureRecipePlan, recipeStorage } = + await import('../../src/lib/server/fleet/launch-recipes.js'); + const hostId = createHost({ name: 'Replacement host' }).id; + const insert = db() + .prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at) VALUES(?,?,?,'library','plutainer:test',28960,?,?)`); - if (previouslyBound) { - insert.run('original-instance', server, hostId, '2025-01-01', '2025-01-02'); - db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(id, 'original-instance'); + if (previouslyBound) { + insert.run('original-instance', server, hostId, '2025-01-01', '2025-01-02'); + db() + .prepare( + 'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)' + ) + .run(id, 'original-instance'); + } + insert.run('replacement-instance', server, hostId, '2025-01-02', null); + const before = db().prepare('SELECT * FROM native_instances ORDER BY id').all(); + const stored = recipeStorage(captureRecipePlan('t5', 'theater')); + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)' + ) + .run(id, stored.recipe, stored.recipeSummary); + const preparation = + await import('../../src/lib/server/provisioning/launch-preparation.js'); + const placement = vi + .spyOn(preparation, 'evaluateNativePlacement') + .mockRejectedValue(new Error('Unexpected placement')); + const { processNativeProvisioning } = + await import('../../src/lib/server/provisioning/native.js'); + await processNativeProvisioning(() => null); + expect(placement).not.toHaveBeenCalled(); + expect( + db().prepare('SELECT state,error FROM server_deployments WHERE id=?').get(id) + ).toEqual({ + state: 'failed', + error: 'Original instance was retired or replaced; no command was sent.' + }); + expect(db().prepare('SELECT * FROM native_instances ORDER BY id').all()).toEqual( + before + ); + expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]); + expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); + expect( + db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id) + ).toEqual({ port: null }); } - insert.run('replacement-instance', server, hostId, '2025-01-02', null); - const before = db().prepare('SELECT * FROM native_instances ORDER BY id').all(); - const stored = recipeStorage(captureRecipePlan('t5', 'theater')); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)') - .run(id, stored.recipe, stored.recipeSummary); - const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); - const placement = vi.spyOn(preparation, 'evaluateNativePlacement').mockRejectedValue(new Error('Unexpected placement')); - const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); - await processNativeProvisioning(() => null); - expect(placement).not.toHaveBeenCalled(); - expect(db().prepare('SELECT state,error FROM server_deployments WHERE id=?').get(id)) - .toEqual({ state: 'failed', error: 'Original instance was retired or replaced; no command was sent.' }); - expect(db().prepare('SELECT * FROM native_instances ORDER BY id').all()).toEqual(before); - expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]); - expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); - expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null }); - }); + ); it('starts work on a ready host when an older pinned intent is waiting for an offline host', async () => { vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); @@ -125,35 +184,93 @@ describe('native creation intents before placement', () => { const hosts = await import('../../src/lib/server/hosts/index.js'); const offline = hosts.createHost({ name: 'Offline host' }); const ready = hosts.createHost({ name: 'Ready host' }); - const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, - cpuCount: 16, cpuUsagePercent: 5, memoryTotal: 32 * 1024 ** 3, memoryAvailable: 30 * 1024 ** 3, - storageAvailable: 100_000_000_000, runtimeImages: [image] }; + const inventory = { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + cpuCount: 16, + cpuUsagePercent: 5, + memoryTotal: 32 * 1024 ** 3, + memoryAvailable: 30 * 1024 ** 3, + storageAvailable: 100_000_000_000, + runtimeImages: [image] + }; hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory); const libraries = await import('../../src/lib/server/game-files/index.js'); const imported = libraries.createLibraryImport({ hostId: ready.id, game: 't5' }); const bytes = Buffer.from('verified base game'); - const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; + const file = { + path: 'zone/base.ff', + size: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex') + }; libraries.setImportManifest(ready.id, imported.id, [file]); - await libraries.uploadImportFile(ready.id, imported.id, file.path, new Response(bytes).body); + await libraries.uploadImportFile( + ready.id, + imported.id, + file.path, + new Response(bytes).body + ); const library = await libraries.publishImport(ready.id, imported.id); - libraries.updateHostLibraryCaches(ready.id, [{ libraryVersionId: library.id, status: 'ready', manifestSha256: library.manifestSha256 }]); - hosts.heartbeat(ready.id, { ...inventory, libraries: [{ id: library.id, state: 'ready' }] }); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + libraries.updateHostLibraryCaches(ready.id, [ + { + libraryVersionId: library.id, + status: 'ready', + manifestSha256: library.manifestSha256 + } + ]); + hosts.heartbeat(ready.id, { + ...inventory, + libraries: [{ id: library.id, state: 'ready' }] + }); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); - const { queueNativeCreation, processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); - const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', libraryId: library.id, image, plutoKey: 'saved-key' }; + const { queueNativeCreation, processNativeProvisioning } = + await import('../../src/lib/server/provisioning/native.js'); + const input = { + game: 't5' as const, + gameMode: 'zombie' as const, + mapSlug: 'theater', + libraryId: library.id, + image, + plutoKey: 'saved-key' + }; const blocked = queueNativeCreation({ ...input, hostId: offline.id }); - const available = queueNativeCreation({ ...input, hostId: ready.id, plutoKey: 'ready-key' }); + const available = queueNativeCreation({ + ...input, + hostId: ready.id, + plutoKey: 'ready-key' + }); expect(blocked.ok && available.ok).toBe(true); if (!blocked.ok || !available.ok) throw new Error('Could not queue launch fixtures.'); await processNativeProvisioning(() => null); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(blocked.deploymentId)).toEqual({ state: 'queued' }); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(available.deploymentId)).toEqual({ state: 'running' }); - expect(db().prepare("SELECT host_id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([{ host_id: ready.id }]); - expect(db().prepare('SELECT server,host_id FROM native_instances WHERE retired_at IS NULL').all()) - .toEqual([{ server: available.name, host_id: ready.id }]); - expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(blocked.deploymentId)).toEqual({ port: null }); + expect( + db() + .prepare('SELECT state FROM server_deployments WHERE id=?') + .get(blocked.deploymentId) + ).toEqual({ state: 'queued' }); + expect( + db() + .prepare('SELECT state FROM server_deployments WHERE id=?') + .get(available.deploymentId) + ).toEqual({ state: 'running' }); + expect( + db() + .prepare("SELECT host_id FROM ec_host_operations WHERE kind='instance.create'") + .all() + ).toEqual([{ host_id: ready.id }]); + expect( + db() + .prepare('SELECT server,host_id FROM native_instances WHERE retired_at IS NULL') + .all() + ).toEqual([{ server: available.name, host_id: ready.id }]); + expect( + db() + .prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?') + .get(blocked.deploymentId) + ).toEqual({ port: null }); }); it('handles exact-match removal of an unplaced launch as local cancellation', async () => { @@ -171,8 +288,12 @@ describe('native creation intents before placement', () => { it('migrates existing port reservations without losing rows, references, indexes, or triggers', async () => { const legacy = new Database(fixture.dbPath); try { - legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_creation_jobs \([\s\S]*?\n\);/, - table => table.replace(' port INTEGER,', ' port INTEGER NOT NULL,'))); + legacy.exec( + schema.replace( + /CREATE TABLE IF NOT EXISTS server_creation_jobs \([\s\S]*?\n\);/, + (table) => table.replace(' port INTEGER,', ' port INTEGER NOT NULL,') + ) + ); legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('existing','old-server','create','queued','Queued','2025-01-01','fixture'); INSERT INTO server_creation_jobs(rowid,deployment_id,server,port,game_mode,map_slug,expected_map) @@ -183,18 +304,33 @@ describe('native creation intents before placement', () => { CREATE TABLE fixture_creation_audit(port INTEGER); CREATE TRIGGER fixture_creation_update AFTER UPDATE OF port ON server_creation_jobs BEGIN INSERT INTO fixture_creation_audit(port) VALUES(NEW.port); END;`); - } finally { legacy.close(); } + } finally { + legacy.close(); + } const { db, closeDb } = await import('../../src/lib/server/db/index.js'); for (let open = 0; open < 2; open++) { const handle = db(); - expect(handle.prepare('SELECT rowid,deployment_id,port FROM server_creation_jobs').get()) - .toEqual({ rowid: 8, deployment_id: 'existing', port: 28960 }); - expect(handle.prepare('SELECT id FROM fixture_creation_reference').get()).toEqual({ id: 'existing' }); - expect(handle.prepare("SELECT name FROM sqlite_master WHERE name IN ('fixture_creation_port','fixture_creation_update')").all()).toHaveLength(2); + expect( + handle.prepare('SELECT rowid,deployment_id,port FROM server_creation_jobs').get() + ).toEqual({ rowid: 8, deployment_id: 'existing', port: 28960 }); + expect(handle.prepare('SELECT id FROM fixture_creation_reference').get()).toEqual({ + id: 'existing' + }); + expect( + handle + .prepare( + "SELECT name FROM sqlite_master WHERE name IN ('fixture_creation_port','fixture_creation_update')" + ) + .all() + ).toHaveLength(2); expect(handle.pragma('foreign_key_check')).toEqual([]); closeDb(); } - db().prepare('UPDATE server_creation_jobs SET port=NULL WHERE deployment_id=?').run('existing'); - expect(db().prepare('SELECT port FROM fixture_creation_audit').all()).toEqual([{ port: null }]); + db() + .prepare('UPDATE server_creation_jobs SET port=NULL WHERE deployment_id=?') + .run('existing'); + expect(db().prepare('SELECT port FROM fixture_creation_audit').all()).toEqual([ + { port: null } + ]); }); }); diff --git a/tests/unit/native-drift-manifest.test.ts b/tests/unit/native-drift-manifest.test.ts index 56fadab..0326d0e 100644 --- a/tests/unit/native-drift-manifest.test.ts +++ b/tests/unit/native-drift-manifest.test.ts @@ -12,127 +12,263 @@ beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); vi.restoreAllMocks(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.restoreAllMocks(); +}); async function setup(capable = true) { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Drift manifest host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { - protocolVersion: 1, capabilities: capable ? ['files.manifest'] : [] + protocolVersion: 1, + capabilities: capable ? ['files.manifest'] : [] }); const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); - database.prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,created_by) - VALUES(?,'t5','zombie_theater','zombie','theater','ui')`).run(server); - database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES(?,?,?,'library','image',28960,?)`).run(randomUUID(), server, host.id, new Date().toISOString()); - database.prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) - SELECT 'launch',host_id,id,generation,'instance.create','{}','completed',created_at,created_at FROM native_instances WHERE server=?`).run(server); + database + .prepare( + `INSERT INTO servers(name,game,map,game_mode,map_slug,created_by) + VALUES(?,'t5','zombie_theater','zombie','theater','ui')` + ) + .run(server); + database + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,'library','image',28960,?)` + ) + .run(randomUUID(), server, host.id, new Date().toISOString()); + database + .prepare( + `INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) + SELECT 'launch',host_id,id,generation,'instance.create','{}','completed',created_at,created_at FROM native_instances WHERE server=?` + ) + .run(server); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); const contents = new Map(); - function promoted(kind: 'script' | 'plugin' | 'baseline', name: string, content: Buffer, targetRelPath?: string) { - const artifact = createScript({ name, kind, targetRelPath, scopeType: 'game', scopeValue: 't5', content }); - database.prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id); + function promoted( + kind: 'script' | 'plugin' | 'baseline', + name: string, + content: Buffer, + targetRelPath?: string + ) { + const artifact = createScript({ + name, + kind, + targetRelPath, + scopeType: 'game', + scopeValue: 't5', + content + }); + database + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(artifact.version.id, artifact.script.id); return artifact; } - const script = promoted('script', 'Managed script', Buffer.from('main() { wait 1; }'), 'managed.gsc'); - const plugin = promoted('plugin', 'Managed plugin', Buffer.from('promoted plugin'), 'managed.dll'); + const script = promoted( + 'script', + 'Managed script', + Buffer.from('main() { wait 1; }'), + 'managed.gsc' + ); + const plugin = promoted( + 'plugin', + 'Managed plugin', + Buffer.from('promoted plugin'), + 'managed.dll' + ); const baseline = promoted('baseline', 'Baseline', Buffer.from('set sv_voice "1"\n')); - for (const [root, artifact] of [[scriptRoot, script], [pluginRoot, plugin]] as const) { + for (const [root, artifact] of [ + [scriptRoot, script], + [pluginRoot, plugin] + ] as const) { const { version } = artifact; - contents.set(`${root}/${artifact.script.target_rel_path}`, (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id)); - recordScript(server, artifact.script.target_rel_path, version.id, version.sha256, version.size, 'test'); + contents.set( + `${root}/${artifact.script.target_rel_path}`, + (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id) + ); + recordScript( + server, + artifact.script.target_rel_path, + version.id, + version.sha256, + version.size, + 'test' + ); } - const { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } = await import('../../src/lib/server/fleet/server-config.js'); + const { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } = + await import('../../src/lib/server/fleet/server-config.js'); const config = getPromotedServerConfig('t5', 'zombie_theater', 'theater')!; contents.set('configs/dedicated_sp.cfg', config); recordScript(server, 'dedicated_sp.cfg', null, hash(config), config.length, 'test'); - contents.set(`configs/${MAP_CONFIG_FILE}`, getServerMapConfig('t5', 'zombie_theater', 'theater')!); + contents.set( + `configs/${MAP_CONFIG_FILE}`, + getServerMapConfig('t5', 'zombie_theater', 'theater')! + ); // The runtime owns this generated file; neither scanner should classify it as drift. contents.set('configs/dedicated.cfg', Buffer.from('runtime generated')); const backend = await import('../../src/lib/server/backends/native.js'); - const request = vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind, payload = {}) => { - if (kind === 'files.manifest') { - const roots = payload.roots as string[]; - const ignored = payload.ignoreNames as string[]; - return { files: [...contents].filter(([path]) => roots.some(root => path.startsWith(`${root}/`)) - && !path.split('/').some(part => ignored.includes(part))) - .map(([path, content]) => ({ path, size: content.length, sha256: hash(content) })) } as never; - } - if (capable) throw new Error(`Manifest scan requested an extra remote operation: ${kind}`); - const path = String(payload.path); - const content = contents.get(path); - if (kind === 'files.read') return { contentBase64: content?.toString('base64') ?? null } as never; - if (kind === 'files.stat') { - const isDirectory = path === '.' || [...contents.keys()].some(file => file.startsWith(`${path}/`)); - return { exists: content !== undefined || isDirectory, size: content?.length ?? 0, - mtime: new Date().toISOString(), isDirectory, isFile: content !== undefined } as never; - } - if (kind === 'files.list') { - const entries = new Map(); - for (const file of contents.keys()) { - if (!file.startsWith(`${path}/`)) continue; - const [name, ...nested] = file.slice(path.length + 1).split('/'); - entries.set(name, { name, isDirectory: nested.length > 0, isFile: nested.length === 0 }); + const request = vi + .spyOn(backend, 'nativeRequest') + .mockImplementation(async (_instance, kind, payload = {}) => { + if (kind === 'files.manifest') { + const roots = payload.roots as string[]; + const ignored = payload.ignoreNames as string[]; + return { + files: [...contents] + .filter( + ([path]) => + roots.some((root) => path.startsWith(`${root}/`)) && + !path.split('/').some((part) => ignored.includes(part)) + ) + .map(([path, content]) => ({ + path, + size: content.length, + sha256: hash(content) + })) + } as never; } - return { entries: [...entries.values()] } as never; - } - throw new Error(`Unexpected remote operation: ${kind}`); - }); + if (capable) + throw new Error(`Manifest scan requested an extra remote operation: ${kind}`); + const path = String(payload.path); + const content = contents.get(path); + if (kind === 'files.read') + return { contentBase64: content?.toString('base64') ?? null } as never; + if (kind === 'files.stat') { + const isDirectory = + path === '.' || + [...contents.keys()].some((file) => file.startsWith(`${path}/`)); + return { + exists: content !== undefined || isDirectory, + size: content?.length ?? 0, + mtime: new Date().toISOString(), + isDirectory, + isFile: content !== undefined + } as never; + } + if (kind === 'files.list') { + const entries = new Map< + string, + { name: string; isDirectory: boolean; isFile: boolean } + >(); + for (const file of contents.keys()) { + if (!file.startsWith(`${path}/`)) continue; + const [name, ...nested] = file.slice(path.length + 1).split('/'); + entries.set(name, { + name, + isDirectory: nested.length > 0, + isFile: nested.length === 0 + }); + } + return { entries: [...entries.values()] } as never; + } + throw new Error(`Unexpected remote operation: ${kind}`); + }); const drift = await import('../../src/lib/server/fleet/drift.js'); - return { ...drift, database, request, contents, script, plugin, baseline, promoted, mapConfigFile: MAP_CONFIG_FILE }; + return { + ...drift, + database, + request, + contents, + script, + plugin, + baseline, + promoted, + mapConfigFile: MAP_CONFIG_FILE + }; } function expectOneManifest(request: Awaited>['request']) { expect(request).toHaveBeenCalledTimes(1); expect(request.mock.calls[0][1]).toBe('files.manifest'); expect(request.mock.calls[0][2]).toEqual({ - roots: [scriptRoot, 'configs', pluginRoot], ignoreNames: ['dedicated.cfg'], - paths: [`${scriptRoot}/managed.gsc`, `${pluginRoot}/managed.dll`, 'configs/dedicated_sp.cfg', 'configs/electric_cherry_map.cfg'] + roots: [scriptRoot, 'configs', pluginRoot], + ignoreNames: ['dedicated.cfg'], + paths: [ + `${scriptRoot}/managed.gsc`, + `${pluginRoot}/managed.dll`, + 'configs/dedicated_sp.cfg', + 'configs/electric_cherry_map.cfg' + ] }); } describe('native drift manifests', () => { - it.each([true, false])('reconciles verified startup password changes without writing server files (manifest: %s)', async capable => { - const api = await setup(capable); - const { applyRuntimePassword } = await import('../../src/lib/server/fleet/runtime-config.js'); - const { getServerScript, recordScript } = await import('../../src/lib/server/fleet/inventory.js'); - const promoted = Buffer.from('set rcon_password "password"\nset sv_maprotation "map zombie_theater"\n'); - const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - // Replace the promoted baseline bytes, retaining a legacy inventory hash. - const next = uploadNewVersion(api.baseline.script.id, promoted); - api.database.prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(next.id, api.baseline.script.id); - const password = 'a'.repeat(64); - api.database.prepare('UPDATE ec_host_operations SET payload=? WHERE id=?').run(JSON.stringify({ rconPassword: password }), 'launch'); - recordScript(server, 'dedicated_sp.cfg', null, hash(promoted), promoted.length, 'native-create'); - const effective = Buffer.from(applyRuntimePassword(promoted.toString(), password)); - expect(effective.length - promoted.length).toBe(56); - api.contents.set('configs/dedicated_sp.cfg', effective); + it.each([true, false])( + 'reconciles verified startup password changes without writing server files (manifest: %s)', + async (capable) => { + const api = await setup(capable); + const { applyRuntimePassword } = + await import('../../src/lib/server/fleet/runtime-config.js'); + const { getServerScript, recordScript } = + await import('../../src/lib/server/fleet/inventory.js'); + const promoted = Buffer.from( + 'set rcon_password "password"\nset sv_maprotation "map zombie_theater"\n' + ); + const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); + // Replace the promoted baseline bytes, retaining a legacy inventory hash. + const next = uploadNewVersion(api.baseline.script.id, promoted); + api.database + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(next.id, api.baseline.script.id); + const password = 'a'.repeat(64); + api.database + .prepare('UPDATE ec_host_operations SET payload=? WHERE id=?') + .run(JSON.stringify({ rconPassword: password }), 'launch'); + recordScript( + server, + 'dedicated_sp.cfg', + null, + hash(promoted), + promoted.length, + 'native-create' + ); + const effective = Buffer.from(applyRuntimePassword(promoted.toString(), password)); + expect(effective.length - promoted.length).toBe(56); + api.contents.set('configs/dedicated_sp.cfg', effective); - expect(await api.scanServerDrift(server)).toEqual([]); - expect(getServerScript(server, 'dedicated_sp.cfg')).toMatchObject({ sha256: hash(effective), size: effective.length, - source: 'runtime-config-reconciliation' }); - if (capable) expectOneManifest(api.request); - expect((await api.prepareDriftCampaign(server)).plan.files).toEqual([]); + expect(await api.scanServerDrift(server)).toEqual([]); + expect(getServerScript(server, 'dedicated_sp.cfg')).toMatchObject({ + sha256: hash(effective), + size: effective.length, + source: 'runtime-config-reconciliation' + }); + if (capable) expectOneManifest(api.request); + expect((await api.prepareDriftCampaign(server)).plan.files).toEqual([]); - // A wrong password of the same length is still real drift. - api.contents.set('configs/dedicated_sp.cfg', Buffer.from(applyRuntimePassword(promoted.toString(), 'b'.repeat(64)))); - expect(await api.scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]); - if (!capable) { - const repair = await api.prepareDriftCampaign(server); - expect(repair.files[0].after).toEqual(effective); + // A wrong password of the same length is still real drift. + api.contents.set( + 'configs/dedicated_sp.cfg', + Buffer.from(applyRuntimePassword(promoted.toString(), 'b'.repeat(64))) + ); + expect(await api.scanServerDrift(server)).toEqual([ + expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' }) + ]); + if (!capable) { + const repair = await api.prepareDriftCampaign(server); + expect(repair.files[0].after).toEqual(effective); + } + api.contents.set( + 'configs/dedicated_sp.cfg', + Buffer.concat([effective, Buffer.from('set sv_voice "0"\n')]) + ); + expect(await api.scanServerDrift(server)).toHaveLength(1); } - api.contents.set('configs/dedicated_sp.cfg', Buffer.concat([effective, Buffer.from('set sv_voice "0"\n')])); - expect(await api.scanServerDrift(server)).toHaveLength(1); - }); + ); it('reports unavailable credentials as a scan failure rather than scheduling a destructive repair', async () => { const api = await setup(); - api.database.prepare("UPDATE ec_host_operations SET history_expired_at='expired',payload='{}' WHERE id='launch'").run(); - await expect(api.scanServerDrift(server)).rejects.toThrow('launch credentials are unavailable'); + api.database + .prepare( + "UPDATE ec_host_operations SET history_expired_at='expired',payload='{}' WHERE id='launch'" + ) + .run(); + await expect(api.scanServerDrift(server)).rejects.toThrow( + 'launch credentials are unavailable' + ); }); it('checks managed files, composed config, generated config, and untracked trees in one request', async () => { @@ -141,7 +277,10 @@ describe('native drift manifests', () => { expectOneManifest(api.request); api.request.mockClear(); - expect(await api.prepareDriftCampaign(server)).toMatchObject({ plan: { files: [] }, files: [] }); + expect(await api.prepareDriftCampaign(server)).toMatchObject({ + plan: { files: [] }, + files: [] + }); expectOneManifest(api.request); }); @@ -155,23 +294,49 @@ describe('native drift manifests', () => { const changes = await api.scanServerDrift(server); expect(changes).toHaveLength(4); - expect(changes).toEqual(expect.arrayContaining([ - expect.objectContaining({ kind: 'script', relPath: 'managed.gsc', driftKind: 'replace', sha256: api.script.version.sha256 }), - expect.objectContaining({ kind: 'plugin', relPath: 'managed.dll', driftKind: 'missing', sha256: api.plugin.version.sha256 }), - expect.objectContaining({ kind: 'config', relPath: 'dedicated_sp.cfg', driftKind: 'replace' }), - expect.objectContaining({ kind: 'script', relPath: 'custom/stray.gsc', driftKind: 'untracked', size: stray.length, sha256: hash(stray) }) - ])); + expect(changes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'script', + relPath: 'managed.gsc', + driftKind: 'replace', + sha256: api.script.version.sha256 + }), + expect.objectContaining({ + kind: 'plugin', + relPath: 'managed.dll', + driftKind: 'missing', + sha256: api.plugin.version.sha256 + }), + expect.objectContaining({ + kind: 'config', + relPath: 'dedicated_sp.cfg', + driftKind: 'replace' + }), + expect.objectContaining({ + kind: 'script', + relPath: 'custom/stray.gsc', + driftKind: 'untracked', + size: stray.length, + sha256: hash(stray) + }) + ]) + ); expectOneManifest(api.request); }); it('still repairs missing inventory and generated map config', async () => { const api = await setup(); - api.database.prepare('DELETE FROM server_scripts WHERE server=? AND rel_path=?').run(server, 'managed.gsc'); + api.database + .prepare('DELETE FROM server_scripts WHERE server=? AND rel_path=?') + .run(server, 'managed.gsc'); api.contents.delete(`configs/${api.mapConfigFile}`); - expect(await api.scanServerDrift(server)).toEqual(expect.arrayContaining([ - expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }), - expect.objectContaining({ relPath: api.mapConfigFile, driftKind: 'missing' }) - ])); + expect(await api.scanServerDrift(server)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }), + expect.objectContaining({ relPath: api.mapConfigFile, driftKind: 'missing' }) + ]) + ); expectOneManifest(api.request); }); @@ -180,20 +345,37 @@ describe('native drift manifests', () => { api.promoted('plugin', 'Managed ignored name', Buffer.from('expected'), 'dedicated.cfg'); api.contents.set(`${pluginRoot}/dedicated.cfg`, Buffer.from('changed')); expect(await api.scanServerDrift(server)).toEqual([ - expect.objectContaining({ kind: 'plugin', relPath: 'dedicated.cfg', driftKind: 'replace' }) + expect.objectContaining({ + kind: 'plugin', + relPath: 'dedicated.cfg', + driftKind: 'replace' + }) ]); expect(api.request).toHaveBeenCalledTimes(1); - expect(api.request.mock.calls[0][2]).toMatchObject({ ignoreNames: [], paths: expect.arrayContaining([`${pluginRoot}/dedicated.cfg`]) }); + expect(api.request.mock.calls[0][2]).toMatchObject({ + ignoreNames: [], + paths: expect.arrayContaining([`${pluginRoot}/dedicated.cfg`]) + }); }); - it.each(['running', 'ready-promote'])('skips remote reads while a %s beta owns the server', async state => { - const api = await setup(); - const beta = api.database.prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual',?)") - .run(api.script.script.id, api.script.version.id, state).lastInsertRowid; - api.database.prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(beta, server); - expect(await api.scanServerDrift(server)).toEqual([]); - expect(api.request).not.toHaveBeenCalled(); - }); + it.each(['running', 'ready-promote'])( + 'skips remote reads while a %s beta owns the server', + async (state) => { + const api = await setup(); + const beta = api.database + .prepare( + "INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual',?)" + ) + .run(api.script.script.id, api.script.version.id, state).lastInsertRowid; + api.database + .prepare( + "INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')" + ) + .run(beta, server); + expect(await api.scanServerDrift(server)).toEqual([]); + expect(api.request).not.toHaveBeenCalled(); + } + ); it('preserves unavailable evidence as a failed scan without falling back to per-file reads', async () => { const api = await setup(); @@ -213,12 +395,16 @@ describe('native drift manifests', () => { const api = await setup(false); api.contents.set(`${scriptRoot}/managed.gsc`, Buffer.from('old script')); api.contents.set('configs/stray.cfg', Buffer.from('untracked config')); - expect(await api.scanServerDrift(server)).toEqual(expect.arrayContaining([ - expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }), - expect.objectContaining({ relPath: 'stray.cfg', driftKind: 'untracked' }) - ])); + expect(await api.scanServerDrift(server)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ relPath: 'managed.gsc', driftKind: 'replace' }), + expect.objectContaining({ relPath: 'stray.cfg', driftKind: 'untracked' }) + ]) + ); const operations = api.request.mock.calls.map(([, kind]) => kind); expect(operations).not.toContain('files.manifest'); - expect(operations).toEqual(expect.arrayContaining(['files.stat', 'files.list', 'files.read'])); + expect(operations).toEqual( + expect.arrayContaining(['files.stat', 'files.list', 'files.read']) + ); }); }); diff --git a/tests/unit/native-game-files-selection.test.ts b/tests/unit/native-game-files-selection.test.ts index b1b86e0..2dfd9b3 100644 --- a/tests/unit/native-game-files-selection.test.ts +++ b/tests/unit/native-game-files-selection.test.ts @@ -2,27 +2,70 @@ import { describe, expect, it } from 'vitest'; import type { GameLibrary, GameLibraryGame, GameLibraryVersion } from '../../src/lib/game-files.js'; import { latestGameFiles, selectLatestGameFiles } from '../../src/lib/ui/native-game-files.js'; -function upload(id: string, game: GameLibraryGame = 't5', state: GameLibraryVersion['state'] = 'ready'): GameLibraryVersion { +function upload( + id: string, + game: GameLibraryGame = 't5', + state: GameLibraryVersion['state'] = 'ready' +): GameLibraryVersion { return { - id, libraryId: game, game, version: id, description: '', state, - sizeBytes: 10, fileCount: 1, manifestSha256: 'manifest', files: [], - createdAt: '2026-10-04T12:00:00Z', publishedAt: state === 'ready' ? '2026-10-04T12:05:00Z' : null, - storageLocation: '', sourceHostId: null, sourceVolume: null, error: null, operationId: null, caches: [] + id, + libraryId: game, + game, + version: id, + description: '', + state, + sizeBytes: 10, + fileCount: 1, + manifestSha256: 'manifest', + files: [], + createdAt: '2026-10-04T12:00:00Z', + publishedAt: state === 'ready' ? '2026-10-04T12:05:00Z' : null, + storageLocation: '', + sourceHostId: null, + sourceVolume: null, + error: null, + operationId: null, + caches: [] }; } -function slot(game: GameLibraryGame, latestVersionId: string | null, versions: GameLibraryVersion[]): GameLibrary { - return { id: game, name: game.toUpperCase(), game, description: '', createdAt: '', latestVersionId, versions }; +function slot( + game: GameLibraryGame, + latestVersionId: string | null, + versions: GameLibraryVersion[] +): GameLibrary { + return { + id: game, + name: game.toUpperCase(), + game, + description: '', + createdAt: '', + latestVersionId, + versions + }; } describe('automatic game file selection for new launches', () => { it('follows the controller latest upload even when older files are already cached', () => { const old = upload('old'); - old.caches = [{ hostId: 'host', libraryVersionId: old.id, status: 'ready', bytesDownloaded: 10, - bytesTotal: 10, manifestSha256: 'manifest', error: null, updatedAt: '', operationId: null }]; + old.caches = [ + { + hostId: 'host', + libraryVersionId: old.id, + status: 'ready', + bytesDownloaded: 10, + bytesTotal: 10, + manifestSha256: 'manifest', + error: null, + updatedAt: '', + operationId: null + } + ]; const current = upload('latest'); // The API can retain older uploads and unfinished uploads in its history. - const libraries = [slot('t5', current.id, [upload('unfinished', 't5', 'importing'), old, current])]; + const libraries = [ + slot('t5', current.id, [upload('unfinished', 't5', 'importing'), old, current]) + ]; const selected = { t5: old.id }; const latest = latestGameFiles(libraries); @@ -33,7 +76,10 @@ describe('automatic game file selection for new launches', () => { }); it('updates both games and removes a stale selection when a game has no available upload', () => { - const latest = latestGameFiles([slot('t5', null, [upload('unavailable')]), slot('t6', 'new-t6', [upload('new-t6', 't6')])]); + const latest = latestGameFiles([ + slot('t5', null, [upload('unavailable')]), + slot('t6', 'new-t6', [upload('new-t6', 't6')]) + ]); const selected: Partial> = { t5: 'old-t5', t6: 'old-t6' }; expect(selectLatestGameFiles(selected, latest, ['t5', 't6'])).toBe(true); @@ -41,7 +87,11 @@ describe('automatic game file selection for new launches', () => { }); it('never selects an unfinished or missing latest upload from history', () => { - expect(latestGameFiles([slot('t5', 'pending', [upload('old'), upload('pending', 't5', 'importing')])])).toEqual({}); + expect( + latestGameFiles([ + slot('t5', 'pending', [upload('old'), upload('pending', 't5', 'importing')]) + ]) + ).toEqual({}); expect(latestGameFiles([slot('t6', 'missing', [upload('old', 't6')])])).toEqual({}); }); diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts index e2507b2..1713e53 100644 --- a/tests/unit/native-hosts.test.ts +++ b/tests/unit/native-hosts.test.ts @@ -3,99 +3,220 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { randomBytes } from 'node:crypto'; let fixture: Fixture; -const inventory = { hostname: 'games-1', agentVersion: '1.0.0', protocolVersion: 1, os: 'Debian 13', architecture: 'amd64', dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }] }; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.useRealTimers(); vi.unstubAllEnvs(); await fixture.cleanup(); }); +const inventory = { + hostname: 'games-1', + agentVersion: '1.0.0', + protocolVersion: 1, + os: 'Debian 13', + architecture: 'amd64', + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * 1024 ** 3, + runtimeImages: ['plutainer:test'], + libraries: [{ id: 'test-library', state: 'ready' }] +}; +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + await fixture.cleanup(); +}); async function enrolled() { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Game host' }); const enrollment = hosts.createEnrollment(host.id); - return { ...hosts, host, enrollment, credentials: hosts.enrollHost(enrollment.token, inventory) }; + return { + ...hosts, + host, + enrollment, + credentials: hosts.enrollHost(enrollment.token, inventory) + }; } describe('native host enrollment and scoped authentication', () => { - it.each(['instance.inspect','rcon.status-batch'])('expires undelivered %s on timeout and delivers the next fresh check', async kind => { - const api = await enrolled(); - const { db } = await import('../../src/lib/server/db/index.js'); - api.heartbeat(api.host.id,{...inventory,capabilities:['operations.health-lane']}); - const stale = api.queueHostOperation(api.host.id,kind,{}); - await expect(api.waitHostOperationStatus(stale.id,0)).rejects.toThrow('expired before delivery'); - expect(api.getHostOperationStatus(stale.id)).toMatchObject({state:'failed',error:'Expired before delivery.'}); - expect(await api.waitHostOperationStatus(stale.id,0)).toMatchObject({state:'failed'}); - expect(db().prepare("SELECT state,error FROM ec_host_operation_events WHERE operation_id=? AND state='failed'").all(stale.id)) - .toEqual([{state:'failed',error:'Expired before delivery.'}]); - const fresh = api.queueHostOperation(api.host.id,kind,{}); - expect(api.pendingHostOperations(api.host.id,'health').map(op=>op.id)).toEqual([fresh.id]); - }); + it.each(['instance.inspect', 'rcon.status-batch'])( + 'expires undelivered %s on timeout and delivers the next fresh check', + async (kind) => { + const api = await enrolled(); + const { db } = await import('../../src/lib/server/db/index.js'); + api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] }); + const stale = api.queueHostOperation(api.host.id, kind, {}); + await expect(api.waitHostOperationStatus(stale.id, 0)).rejects.toThrow( + 'expired before delivery' + ); + expect(api.getHostOperationStatus(stale.id)).toMatchObject({ + state: 'failed', + error: 'Expired before delivery.' + }); + expect(await api.waitHostOperationStatus(stale.id, 0)).toMatchObject({ + state: 'failed' + }); + expect( + db() + .prepare( + "SELECT state,error FROM ec_host_operation_events WHERE operation_id=? AND state='failed'" + ) + .all(stale.id) + ).toEqual([{ state: 'failed', error: 'Expired before delivery.' }]); + const fresh = api.queueHostOperation(api.host.id, kind, {}); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + fresh.id + ]); + } + ); - it.each(['pending','accepted','running'])('preserves delivered health checks in %s state after the caller times out', async state => { - const api = await enrolled(); - const operation = api.queueHostOperation(api.host.id,'instance.inspect',{}); - api.pendingHostOperations(api.host.id,'health'); - if (state !== 'pending') api.reportHostOperationStatus(api.host.id,operation.id,{state}); - await expect(api.waitHostOperationStatus(operation.id,0)).rejects.toThrow('awaiting acknowledgement'); - expect(api.getHostOperationStatus(operation.id)?.state).toBe(state); - api.reportHostOperationStatus(api.host.id,operation.id,{state:'completed',result:{healthy:true}}); - expect(await api.waitHostOperationStatus(operation.id)).toMatchObject({state:'completed',result:{healthy:true}}); - }); + it.each(['pending', 'accepted', 'running'])( + 'preserves delivered health checks in %s state after the caller times out', + async (state) => { + const api = await enrolled(); + const operation = api.queueHostOperation(api.host.id, 'instance.inspect', {}); + api.pendingHostOperations(api.host.id, 'health'); + if (state !== 'pending') + api.reportHostOperationStatus(api.host.id, operation.id, { state }); + await expect(api.waitHostOperationStatus(operation.id, 0)).rejects.toThrow( + 'awaiting acknowledgement' + ); + expect(api.getHostOperationStatus(operation.id)?.state).toBe(state); + api.reportHostOperationStatus(api.host.id, operation.id, { + state: 'completed', + result: { healthy: true } + }); + expect(await api.waitHostOperationStatus(operation.id)).toMatchObject({ + state: 'completed', + result: { healthy: true } + }); + } + ); - it.each(['instance.restart','files.stage','rcon.execute'])('preserves undelivered %s when its caller times out', async kind => { - const api = await enrolled(); - const operation = api.queueHostOperation(api.host.id,kind,{command:'status'}); - await expect(api.waitHostOperationStatus(operation.id,0)).rejects.toThrow('awaiting acknowledgement'); - expect(api.getHostOperationStatus(operation.id)?.state).toBe('pending'); - }); + it.each(['instance.restart', 'files.stage', 'rcon.execute'])( + 'preserves undelivered %s when its caller times out', + async (kind) => { + const api = await enrolled(); + const operation = api.queueHostOperation(api.host.id, kind, { command: 'status' }); + await expect(api.waitHostOperationStatus(operation.id, 0)).rejects.toThrow( + 'awaiting acknowledgement' + ); + expect(api.getHostOperationStatus(operation.id)?.state).toBe('pending'); + } + ); it('keeps the host online and delivers work when individual state observations are invalid', async () => { const api = await enrolled(); - const good = { instanceId: 'healthy-server', generation: 1, state: 'running', healthy: true, observedAt: new Date().toISOString() }; + const good = { + instanceId: 'healthy-server', + generation: 1, + state: 'running', + healthy: true, + observedAt: new Date().toISOString() + }; const operation = api.queueHostOperation(api.host.id, 'host.inspect', {}); - const host = api.heartbeat(api.host.id, { ...inventory, instanceStates: [good, - { ...good, instanceId: 'crashing', state: 'restarting' }, - { ...good, instanceId: 'paused', state: 'paused' }, null, - { ...good, generation: 0 }, { ...good, observedAt: 'invalid' }] }); + const host = api.heartbeat(api.host.id, { + ...inventory, + instanceStates: [ + good, + { ...good, instanceId: 'crashing', state: 'restarting' }, + { ...good, instanceId: 'paused', state: 'paused' }, + null, + { ...good, generation: 0 }, + { ...good, observedAt: 'invalid' } + ] + }); expect(host.inventory.instanceStates).toEqual([good]); expect(host.readiness).toBe('ready'); - expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(operation.id); + expect(api.pendingHostOperations(api.host.id).map((row) => row.id)).toContain(operation.id); }); it('queues serialized launch files and processes receipts without hydrating plugin payloads', async () => { const api = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js'); - const payload = { files: [{ path: 'plugins/fixture.dll', contentBase64: Buffer.alloc(8 * 1024 ** 2, 42).toString('base64') }] }; + const payload = { + files: [ + { + path: 'plugins/fixture.dll', + contentBase64: Buffer.alloc(8 * 1024 ** 2, 42).toString('base64') + } + ] + }; const { putFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); const hash = putFileBlob(Buffer.from(payload.files[0].contentBase64, 'base64')); // Launch preparation transfers bytes separately and serializes only references. - const bytes = Buffer.from(JSON.stringify({ files: [{ path: payload.files[0].path, contentSha256: hash }] })); - const options = { instanceId: 'immutable-instance', generation: 1, idempotencyKey: 'large-launch' }; + const bytes = Buffer.from( + JSON.stringify({ files: [{ path: payload.files[0].path, contentSha256: hash }] }) + ); + const options = { + instanceId: 'immutable-instance', + generation: 1, + idempotencyKey: 'large-launch' + }; const parse = JSON.parse; const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => { - if (value.length > 1024 * 1024) throw new Error('Launch payload was unnecessarily parsed.'); + if (value.length > 1024 * 1024) + throw new Error('Launch payload was unnecessarily parsed.'); return parse(value, reviver); }); let id: string; try { - const operation = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options); + const operation = api.queueHostOperationSerialized( + api.host.id, + 'instance.create', + bytes, + options + ); id = operation.id; expect(operation).not.toHaveProperty('payload'); - expect(api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options).id).toBe(id); - expect(() => api.queueHostOperationSerialized(api.host.id, 'instance.create', Buffer.from('{}'), options)).toThrow('different request'); - const stored = db().prepare('SELECT typeof(payload) AS kind,count(*) AS count FROM ec_host_operations WHERE idempotency_key=?').get(options.idempotencyKey); + expect( + api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options).id + ).toBe(id); + expect(() => + api.queueHostOperationSerialized( + api.host.id, + 'instance.create', + Buffer.from('{}'), + options + ) + ).toThrow('different request'); + const stored = db() + .prepare( + 'SELECT typeof(payload) AS kind,count(*) AS count FROM ec_host_operations WHERE idempotency_key=?' + ) + .get(options.idempotencyKey); expect(stored).toEqual({ kind: 'text', count: 1 }); const waiting = api.waitHostOperationStatus(id); - expect(api.reportHostOperationStatus(api.host.id, id, { state: 'accepted' }).state).toBe('accepted'); - expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('running'); - api.reportHostOperationStatus(api.host.id, id, { state: 'completed', result: { healthy: true } }); - expect(await waiting).toMatchObject({ id, state: 'completed', result: { healthy: true } }); - expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('completed'); + expect( + api.reportHostOperationStatus(api.host.id, id, { state: 'accepted' }).state + ).toBe('accepted'); + expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe( + 'running' + ); + api.reportHostOperationStatus(api.host.id, id, { + state: 'completed', + result: { healthy: true } + }); + expect(await waiting).toMatchObject({ + id, + state: 'completed', + result: { healthy: true } + }); + expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe( + 'completed' + ); expect(api.getHostOperationStatus(id)).not.toHaveProperty('payload'); const other = api.createHost({ name: 'Other host' }); - expect(() => api.reportHostOperationStatus(other.id, id, { state: 'completed' })).toThrow('this host'); - } finally { parsing.mockRestore(); } + expect(() => + api.reportHostOperationStatus(other.id, id, { state: 'completed' }) + ).toThrow('this host'); + } finally { + parsing.mockRestore(); + } // Full-data APIs retain their original contract for explicit readers. expect(api.getHostOperation(id!)?.payload).toEqual(payload); - expect(api.reportHostOperation(api.host.id, id!, { state: 'running' }).payload).toEqual(payload); + expect(api.reportHostOperation(api.host.id, id!, { state: 'running' }).payload).toEqual( + payload + ); expect((await api.waitHostOperation(id!)).payload).toEqual(payload); }); @@ -105,29 +226,61 @@ describe('native host enrollment and scoped authentication', () => { const routing = { libraryId: 'test-library', name: 't5_zombie_theater_1', game: 't5' }; const payload = { ...routing, files: [{ contentBase64: 'A'.repeat(512 * 1024) }] }; const bytes = Buffer.from(JSON.stringify(payload)); - const launch = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, {}, routing); - const inline = api.queueHostOperation(api.host.id, 'instance.inspect', { name: 'inline-legacy' }); + const launch = api.queueHostOperationSerialized( + api.host.id, + 'instance.create', + bytes, + {}, + routing + ); + const inline = api.queueHostOperation(api.host.id, 'instance.inspect', { + name: 'inline-legacy' + }); expect(api.getHostOperation(launch.id)?.payload).toEqual(payload); - expect(api.listHostOperations(api.host.id).find(operation => operation.id === launch.id)?.payload).toEqual(payload); - expect(api.pendingHostOperations(api.host.id, 'write').map(operation => operation.payload)).toEqual([payload]); + expect( + api.listHostOperations(api.host.id).find((operation) => operation.id === launch.id) + ?.payload + ).toEqual(payload); + expect( + api.pendingHostOperations(api.host.id, 'write').map((operation) => operation.payload) + ).toEqual([payload]); expect(api.pendingHostOperations(api.host.id, 'read')).toEqual([inline]); - expect(api.pendingHostOperationRows(api.host.id, 'write', false)[0]).not.toHaveProperty('payload'); + expect(api.pendingHostOperationRows(api.host.id, 'write', false)[0]).not.toHaveProperty( + 'payload' + ); api.reportHostOperationStatus(api.host.id, launch.id, { state: 'accepted' }); - api.reportHostOperationStatus(api.host.id, launch.id, { state: 'running', result: { phase: 'starting' } }); - const stored = db().prepare(`SELECT payload,last_delivered_at FROM ec_host_operations WHERE id=?`).get(launch.id) as - { payload: string; last_delivered_at: string | null }; + api.reportHostOperationStatus(api.host.id, launch.id, { + state: 'running', + result: { phase: 'starting' } + }); + const stored = db() + .prepare(`SELECT payload,last_delivered_at FROM ec_host_operations WHERE id=?`) + .get(launch.id) as { payload: string; last_delivered_at: string | null }; expect(stored.payload).toBe(JSON.stringify(routing)); expect(stored.last_delivered_at).not.toBeNull(); - const saved = db().prepare('SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?').get(launch.id) as { bytes: number }; + const saved = db() + .prepare( + 'SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?' + ) + .get(launch.id) as { bytes: number }; expect(saved.bytes).toBeLessThan(1024); expect(api.getHostOperationPayload(api.host.id, launch.id)).toEqual(bytes); - expect(api.getHostOperationPayload(api.host.id, inline.id).toString('utf8')).toBe(JSON.stringify(inline.payload)); - expect(db().prepare('SELECT 1 FROM ec_host_operation_payloads WHERE operation_id=?').get(inline.id)).toBeUndefined(); + expect(api.getHostOperationPayload(api.host.id, inline.id).toString('utf8')).toBe( + JSON.stringify(inline.payload) + ); + expect( + db() + .prepare('SELECT 1 FROM ec_host_operation_payloads WHERE operation_id=?') + .get(inline.id) + ).toBeUndefined(); }); it('narrows cancellation lookups by kind instead of scanning completed health history', async () => { const api = await enrolled(); - api.heartbeat(api.host.id, { ...inventory, capabilities: ['instance.cancel-create', 'operations.health-lane'] }); + api.heartbeat(api.host.id, { + ...inventory, + capabilities: ['instance.cancel-create', 'operations.health-lane'] + }); const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); const insert = handle.prepare(`INSERT INTO ec_host_operations @@ -138,60 +291,106 @@ describe('native host enrollment and scoped authentication', () => { })(); const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); const prepare = vi.spyOn(handle, 'prepare'); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]); - const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o WHERE host_id=?'))!; + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + inspect.id + ]); + const query = prepare.mock.calls + .map(([sql]) => sql) + .find((sql) => sql.includes('FROM ec_host_operations o WHERE host_id=?'))!; prepare.mockRestore(); // Assert the actual poll's access path, avoiding hardware-dependent timing thresholds. - const plan = handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(api.host.id, 1, 1, new Date().toISOString()) as { detail: string }[]; - expect(plan.find(row => row.detail.includes('SEARCH cancellation'))?.detail) - .toContain('(host_id=? AND kind=?)'); + const plan = handle + .prepare(`EXPLAIN QUERY PLAN ${query}`) + .all(api.host.id, 1, 1, new Date().toISOString()) as { detail: string }[]; + expect(plan.find((row) => row.detail.includes('SEARCH cancellation'))?.detail).toContain( + '(host_id=? AND kind=?)' + ); }); - it.each(['accepted', 'running'])('withholds %s health work and replays its original ID after the recovery lease', async state => { - const api = await enrolled(); - vi.useFakeTimers({ toFake: ['Date'] }); - const at = Date.now(); - api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] }); - const busy = api.queueHostOperation(api.host.id, 'instance.inspect', {}); - api.pendingHostOperations(api.host.id, 'health'); - api.reportHostOperation(api.host.id, busy.id, { state }); - const next = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' }); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([next.id]); - api.reportHostOperation(api.host.id, next.id, { state: 'completed' }); - expect(api.pendingHostOperations(api.host.id, 'health')).toEqual([]); - vi.setSystemTime(at + 30_001); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([busy.id]); - expect(api.pendingHostOperations(api.host.id, 'health')).toEqual([]); - api.reportHostOperation(api.host.id, busy.id, { state: 'unknown' }); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([busy.id]); - }); + it.each(['accepted', 'running'])( + 'withholds %s health work and replays its original ID after the recovery lease', + async (state) => { + const api = await enrolled(); + vi.useFakeTimers({ toFake: ['Date'] }); + const at = Date.now(); + api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] }); + const busy = api.queueHostOperation(api.host.id, 'instance.inspect', {}); + api.pendingHostOperations(api.host.id, 'health'); + api.reportHostOperation(api.host.id, busy.id, { state }); + const next = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' }); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + next.id + ]); + api.reportHostOperation(api.host.id, next.id, { state: 'completed' }); + expect(api.pendingHostOperations(api.host.id, 'health')).toEqual([]); + vi.setSystemTime(at + 30_001); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + busy.id + ]); + expect(api.pendingHostOperations(api.host.id, 'health')).toEqual([]); + api.reportHostOperation(api.host.id, busy.id, { state: 'unknown' }); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + busy.id + ]); + } + ); it('reserves health delivery while a file backlog is pending and bounds background batches', async () => { const api = await enrolled(); api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] }); - const files = Array.from({ length: 30 }, (_, i) => api.queueHostOperation(api.host.id, 'files.read', { path: `file-${i}` })); + const files = Array.from({ length: 30 }, (_, i) => + api.queueHostOperation(api.host.id, 'files.read', { path: `file-${i}` }) + ); const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' }); const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); - const mutation = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' }); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([status.id]); + const mutation = api.queueHostOperation(api.host.id, 'rcon.execute', { + command: 'map_restart' + }); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + status.id + ]); api.reportHostOperation(api.host.id, status.id, { state: 'completed', result: {} }); - expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]); - expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual(files.slice(0, 4).map(op => op.id)); - expect(api.pendingHostOperations(api.host.id, 'write').map(op => op.id)).toEqual([mutation.id]); + expect(api.pendingHostOperations(api.host.id, 'health').map((op) => op.id)).toEqual([ + inspect.id + ]); + expect(api.pendingHostOperations(api.host.id, 'read').map((op) => op.id)).toEqual( + files.slice(0, 4).map((op) => op.id) + ); + expect(api.pendingHostOperations(api.host.id, 'write').map((op) => op.id)).toEqual([ + mutation.id + ]); // Rolling upgrades: older agents still receive health through read. api.heartbeat(api.host.id, inventory); - files.forEach(file => api.reportHostOperation(api.host.id, file.id, { state: 'completed', result: {} })); - expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual([inspect.id]); + files.forEach((file) => + api.reportHostOperation(api.host.id, file.id, { state: 'completed', result: {} }) + ); + expect(api.pendingHostOperations(api.host.id, 'read').map((op) => op.id)).toEqual([ + inspect.id + ]); }); it('separates observation and mutation delivery and wakes long polls on newly queued work', async () => { const api = await enrolled(); - const inspect = api.queueHostOperation(api.host.id, 'files.manifest', { roots: ['configs'] }); + const inspect = api.queueHostOperation(api.host.id, 'files.manifest', { + roots: ['configs'] + }); const restart = api.queueHostOperation(api.host.id, 'instance.restart', {}); const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' }); - const command = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' }); - expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id).sort()).toEqual([inspect.id, status.id].sort()); - expect(api.pendingHostOperations(api.host.id, 'write').map(op => op.id).sort()).toEqual([restart.id, command.id].sort()); + const command = api.queueHostOperation(api.host.id, 'rcon.execute', { + command: 'map_restart' + }); + expect( + api + .pendingHostOperations(api.host.id, 'read') + .map((op) => op.id) + .sort() + ).toEqual([inspect.id, status.id].sort()); + expect( + api + .pendingHostOperations(api.host.id, 'write') + .map((op) => op.id) + .sort() + ).toEqual([restart.id, command.id].sort()); // Let earlier enqueue notifications finish before subscribing. await Promise.resolve(); const controller = new AbortController(); @@ -208,45 +407,87 @@ describe('native host enrollment and scoped authentication', () => { expect(api.authenticateCredential(api.enrollment.token)).toBeNull(); expect(() => api.enrollHost(api.enrollment.token, inventory)).toThrow('invalid or expired'); const { db } = await import('../../src/lib/server/db/index.js'); - const stored = JSON.stringify(db().prepare('SELECT * FROM ec_host_credentials').all()) + JSON.stringify(db().prepare('SELECT * FROM ec_host_enrollments').all()); - expect(stored).not.toContain(api.credentials.credential); expect(stored).not.toContain(api.enrollment.token); + const stored = + JSON.stringify(db().prepare('SELECT * FROM ec_host_credentials').all()) + + JSON.stringify(db().prepare('SELECT * FROM ec_host_enrollments').all()); + expect(stored).not.toContain(api.credentials.credential); + expect(stored).not.toContain(api.enrollment.token); }); it('expires enrollments and invalidates earlier unused enrollment tokens', async () => { const api = await enrolled(); - const first = api.createEnrollment(api.host.id); const next = api.createEnrollment(api.host.id, 10); + const first = api.createEnrollment(api.host.id); + const next = api.createEnrollment(api.host.id, 10); expect(() => api.enrollHost(first.token, inventory)).toThrow('invalid or expired'); - vi.useFakeTimers(); vi.setSystemTime(Date.now() + 20); + vi.useFakeTimers(); + vi.setSystemTime(Date.now() + 20); expect(() => api.enrollHost(next.token, inventory)).toThrow('invalid or expired'); }); it('rotates credentials independently and immediately revokes access', async () => { - const api = await enrolled(); const other = api.createHost({ name: 'Other' }); const second = api.enrollHost(api.createEnrollment(other.id).token, inventory); + const api = await enrolled(); + const other = api.createHost({ name: 'Other' }); + const second = api.enrollHost(api.createEnrollment(other.id).token, inventory); const rotated = api.rotateCredential(api.host.id); - expect(api.authenticateCredential(api.credentials.credential)).toBeNull(); expect(api.authenticateCredential(rotated)?.id).toBe(api.host.id); + expect(api.authenticateCredential(api.credentials.credential)).toBeNull(); + expect(api.authenticateCredential(rotated)?.id).toBe(api.host.id); api.revokeCredentials(api.host.id); - expect(api.authenticateCredential(rotated)).toBeNull(); expect(api.authenticateCredential(second.credential)?.id).toBe(other.id); + expect(api.authenticateCredential(rotated)).toBeNull(); + expect(api.authenticateCredential(second.credential)?.id).toBe(other.id); }); it('requires HTTPS and blocks cross-origin management and agent credentials on management APIs', async () => { - const api = await enrolled(); const { authenticateAgent, requireManagementOrigin } = await import('../../src/lib/server/hosts/http.js'); + const api = await enrolled(); + const { authenticateAgent, requireManagementOrigin } = + await import('../../src/lib/server/hosts/http.js'); const headers = { authorization: `Bearer ${api.credentials.credential}` }; - expect(authenticateAgent(new Request('https://ec.test/api/agent/operations', { headers })).id).toBe(api.host.id); - expect(() => authenticateAgent(new Request('http://ec.test/api/agent/operations', { headers }))).toThrow(); - expect(() => authenticateAgent(new Request('https://ec.test/api/agent/operations', { headers: { authorization: `Bearer ${api.enrollment.token}` } }))).toThrow(); - expect(() => requireManagementOrigin(new Request('https://ec.test/api/hosts', { headers }))).toThrow(); - expect(() => requireManagementOrigin(new Request('https://ec.test/api/hosts', { headers: { Origin: 'https://attacker.test' } }))).toThrow(); + expect( + authenticateAgent(new Request('https://ec.test/api/agent/operations', { headers })).id + ).toBe(api.host.id); + expect(() => + authenticateAgent(new Request('http://ec.test/api/agent/operations', { headers })) + ).toThrow(); + expect(() => + authenticateAgent( + new Request('https://ec.test/api/agent/operations', { + headers: { authorization: `Bearer ${api.enrollment.token}` } + }) + ) + ).toThrow(); + expect(() => + requireManagementOrigin(new Request('https://ec.test/api/hosts', { headers })) + ).toThrow(); + expect(() => + requireManagementOrigin( + new Request('https://ec.test/api/hosts', { + headers: { Origin: 'https://attacker.test' } + }) + ) + ).toThrow(); }); it('recovers credential rotation when acknowledgement is lost using the agent-persisted replacement', async () => { - const api = await enrolled(); const next = 'ec_host_' + randomBytes(32).toString('base64url'); + const api = await enrolled(); + const next = 'ec_host_' + randomBytes(32).toString('base64url'); expect(api.rotateCredential(api.host.id, next)).toBe(next); expect(api.authenticateCredential(api.credentials.credential)).toBeNull(); expect(api.authenticateCredential(next)?.id).toBe(api.host.id); expect(api.rotateCredential(api.host.id, next)).toBe(next); - api.revokeCredentials(api.host.id); expect(() => api.rotateCredential(api.host.id, next)).toThrow('cannot be reused'); + api.revokeCredentials(api.host.id); + expect(() => api.rotateCredential(api.host.id, next)).toThrow('cannot be reused'); }); it('bounds chunked request bodies before consuming the complete upload', async () => { const { objectBody } = await import('../../src/lib/server/hosts/http.js'); let cancelled = false; - const body = new ReadableStream({ pull(controller) { controller.enqueue(new TextEncoder().encode('x'.repeat(100))); }, cancel() { cancelled = true; } }); - const request = new Request('https://ec.test/api/agent/operations/x', { method: 'POST', body, duplex: 'half' } as RequestInit); + const body = new ReadableStream({ + pull(controller) { + controller.enqueue(new TextEncoder().encode('x'.repeat(100))); + }, + cancel() { + cancelled = true; + } + }); + const request = new Request('https://ec.test/api/agent/operations/x', { + method: 'POST', + body, + duplex: 'half' + } as RequestInit); await expect(objectBody(request, 50)).rejects.toMatchObject({ status: 413 }); expect(cancelled).toBe(true); }); @@ -256,68 +497,111 @@ describe('native host readiness and reservations', () => { it('separates a connected agent from ready Docker, storage, runtime and library state', async () => { const api = await enrolled(); expect(api.requireReadyHost(api.host.id).readiness).toBe('ready'); - expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime'); - expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library'); - expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('needs-game-files'); + expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow( + 'runtime' + ); + expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow( + 'library' + ); + expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe( + 'needs-game-files' + ); const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false }); - expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded'); + expect(host.connected).toBe(true); + expect(host.readiness).toBe('degraded'); expect(() => api.requireReadyHost(host.id)).toThrow('Docker'); api.heartbeat(host.id, inventory); - vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000); - expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale'); + vi.useFakeTimers(); + vi.setSystemTime(Date.now() + 91_000); + expect(api.getHost(host.id)?.connectionState).toBe('offline'); + expect(() => api.requireReadyHost(host.id)).toThrow('stale'); }); it.each([ { missing: { libraries: [] }, expected: 'needs-game-files' }, { missing: { runtimeImages: [] }, expected: 'needs-runtime' }, { missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' } - ])('names missing content as $expected until a prepared heartbeat arrives', async ({ missing, expected }) => { - const api = await enrolled(); - const host = api.heartbeat(api.host.id, { ...inventory, ...missing }); - expect(host.connected).toBe(true); - expect(host.readiness).toBe(expected); - expect(() => api.requireReadyHost(host.id)).toThrow(); - expect(api.heartbeat(host.id, inventory).readiness).toBe('ready'); - expect(api.requireReadyHost(host.id).id).toBe(host.id); - }); + ])( + 'names missing content as $expected until a prepared heartbeat arrives', + async ({ missing, expected }) => { + const api = await enrolled(); + const host = api.heartbeat(api.host.id, { ...inventory, ...missing }); + expect(host.connected).toBe(true); + expect(host.readiness).toBe(expected); + expect(() => api.requireReadyHost(host.id)).toThrow(); + expect(api.heartbeat(host.id, inventory).readiness).toBe('ready'); + expect(api.requireReadyHost(host.id).id).toBe(host.id); + } + ); it('keeps setup work deliverable while new instance creation waits for missing game files', async () => { const api = await enrolled(); const create = api.queueHostOperation(api.host.id, 'instance.create', {}); api.heartbeat(api.host.id, { ...inventory, libraries: [] }); - expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('game-file library'); + expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow( + 'game-file library' + ); const download = api.queueHostOperation(api.host.id, 'library.download', {}); const runtime = api.queueHostOperation(api.host.id, 'runtime.pull', {}); - expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]); + expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([ + download.id, + runtime.id + ]); api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] }); - expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]); + expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([ + download.id, + runtime.id + ]); api.heartbeat(api.host.id, inventory); - expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(create.id); + expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toContain( + create.id + ); }); it('gives enrollment, infrastructure failure and host modes priority over missing content', async () => { const api = await enrolled(); expect(api.createHost({ name: 'New host' }).readiness).toBe('preparing'); - expect(api.heartbeat(api.host.id, { ...inventory, libraries: [], dockerRunning: false }).readiness).toBe('degraded'); + expect( + api.heartbeat(api.host.id, { ...inventory, libraries: [], dockerRunning: false }) + .readiness + ).toBe('degraded'); api.heartbeat(api.host.id, { ...inventory, libraries: [] }); expect(api.updateHost(api.host.id, { mode: 'draining' }).readiness).toBe('draining'); expect(api.updateHost(api.host.id, { mode: 'disabled' }).readiness).toBe('disabled'); }); it('blocks new deployments on a draining or disabled host without altering existing operations', async () => { - const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 }); + const api = await enrolled(); + const existing = api.queueHostOperation( + api.host.id, + 'instance.inspect', + {}, + { instanceId: 'immutable-id', generation: 1 } + ); api.updateHost(api.host.id, { mode: 'draining' }); - expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('draining'); + expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow( + 'draining' + ); expect(api.getHostOperation(existing.id)?.state).toBe('pending'); - api.updateHost(api.host.id, { mode: 'disabled' }); expect(api.pendingHostOperations(api.host.id)).toEqual([]); + api.updateHost(api.host.id, { mode: 'disabled' }); + expect(api.pendingHostOperations(api.host.id)).toEqual([]); }); it('allocates per host, bind address and protocol while protecting wildcard bindings', async () => { - const api = await enrolled(); const second = api.createHost({ name: 'Other' }); + const api = await enrolled(); + const second = api.createHost({ name: 'Other' }); expect(api.reserveHostPort(api.host.id, 'a', { port: 28960 })).toBe(28960); expect(api.reserveHostPort(second.id, 'b', { port: 28960 })).toBe(28960); expect(api.reserveHostPort(api.host.id, 'c', { port: 28960, protocol: 'tcp' })).toBe(28960); - expect(() => api.reserveHostPort(api.host.id, 'd', { port: 28960, bindAddress: '127.0.0.1' })).toThrow('available'); + expect(() => + api.reserveHostPort(api.host.id, 'd', { port: 28960, bindAddress: '127.0.0.1' }) + ).toThrow('available'); expect(api.reserveHostPort(api.host.id, 'a', { port: 28960 })).toBe(28960); api.releaseHostPorts('a'); - expect(api.reserveHostPort(api.host.id, 'e', { port: 28960, bindAddress: '127.0.0.1' })).toBe(28960); - expect(api.reserveHostPort(api.host.id, 'f', { port: 28960, bindAddress: '127.0.0.2' })).toBe(28960); - expect(() => api.reserveHostPort(api.host.id, 'g', { port: 28960, bindAddress: '0.0.0.0' })).toThrow('available'); + expect( + api.reserveHostPort(api.host.id, 'e', { port: 28960, bindAddress: '127.0.0.1' }) + ).toBe(28960); + expect( + api.reserveHostPort(api.host.id, 'f', { port: 28960, bindAddress: '127.0.0.2' }) + ).toBe(28960); + expect(() => + api.reserveHostPort(api.host.id, 'g', { port: 28960, bindAddress: '0.0.0.0' }) + ).toThrow('available'); expect(() => api.reserveHostPort(api.host.id, 'h', { port: 28000 })).toThrow('available'); }); }); @@ -325,65 +609,125 @@ describe('native host readiness and reservations', () => { describe('durable host operations', () => { it('drains an old agent for a cancellation update while allowing receipts, heartbeats, and other hosts', async () => { const api = await enrolled(); - const original = api.queueHostOperation(api.host.id, 'instance.create', {}, { instanceId: 'cancelled-instance', generation: 1 }); + const original = api.queueHostOperation( + api.host.id, + 'instance.create', + {}, + { instanceId: 'cancelled-instance', generation: 1 } + ); const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); - expect(api.pendingHostOperations(api.host.id).map(op => op.id)).toContain(original.id); + expect(api.pendingHostOperations(api.host.id).map((op) => op.id)).toContain(original.id); api.reportHostOperation(api.host.id, original.id, { state: 'running' }); api.reportHostOperation(api.host.id, inspect.id, { state: 'unknown' }); - const cleanup = api.queueHostOperation(api.host.id, 'instance.cancel-create', { createOperationId: original.id }, { instanceId: 'cancelled-instance', generation: 1 }); + const cleanup = api.queueHostOperation( + api.host.id, + 'instance.cancel-create', + { createOperationId: original.id }, + { instanceId: 'cancelled-instance', generation: 1 } + ); const logs = api.queueHostOperation(api.host.id, 'logs.read', {}); const preparation = api.queueHostOperation(api.host.id, 'library.download', {}); const other = api.createHost({ name: 'Unaffected host' }); api.enrollHost(api.createEnrollment(other.id).token, inventory); const otherWork = api.queueHostOperation(other.id, 'host.inspect', {}); const { db } = await import('../../src/lib/server/db/index.js'); - const before = db().prepare('SELECT id,state,last_delivered_at FROM ec_host_operations WHERE host_id=? ORDER BY id').all(api.host.id); - for (let poll = 0; poll < 3; poll++) expect(api.pendingHostOperations(api.host.id)).toEqual([]); - expect(db().prepare('SELECT id,state,last_delivered_at FROM ec_host_operations WHERE host_id=? ORDER BY id').all(api.host.id)).toEqual(before); - expect(api.pendingHostOperations(other.id).map(op => op.id)).toEqual([otherWork.id]); - api.reportHostOperation(api.host.id, inspect.id, { state: 'completed', result: { healthy: true } }); + const before = db() + .prepare( + 'SELECT id,state,last_delivered_at FROM ec_host_operations WHERE host_id=? ORDER BY id' + ) + .all(api.host.id); + for (let poll = 0; poll < 3; poll++) + expect(api.pendingHostOperations(api.host.id)).toEqual([]); + expect( + db() + .prepare( + 'SELECT id,state,last_delivered_at FROM ec_host_operations WHERE host_id=? ORDER BY id' + ) + .all(api.host.id) + ).toEqual(before); + expect(api.pendingHostOperations(other.id).map((op) => op.id)).toEqual([otherWork.id]); + api.reportHostOperation(api.host.id, inspect.id, { + state: 'completed', + result: { healthy: true } + }); api.heartbeat(api.host.id, inventory); expect(api.pendingHostOperations(api.host.id)).toEqual([]); // The controller hold must clear on capability, without waiting for cleanup // or the original creation to finish (which would recreate the deadlock). api.heartbeat(api.host.id, { ...inventory, capabilities: ['instance.cancel-create'] }); - expect(api.pendingHostOperations(api.host.id).map(op => op.id)).toEqual([cleanup.id, logs.id, preparation.id]); + expect(api.pendingHostOperations(api.host.id).map((op) => op.id)).toEqual([ + cleanup.id, + logs.id, + preparation.id + ]); expect(api.getHostOperation(original.id)?.state).toBe('running'); expect(api.getHostOperation(cleanup.id)?.state).toBe('pending'); }); - it.each(['completed', 'failed'] as const)('does not pause an old host for a terminal %s cancellation', async state => { - const api = await enrolled(); - const cleanup = api.queueHostOperation(api.host.id, 'instance.cancel-create', { createOperationId: 'creation' }); - api.reportHostOperation(api.host.id, cleanup.id, { state }); - const work = api.queueHostOperation(api.host.id, 'host.inspect', {}); - expect(api.pendingHostOperations(api.host.id).map(op => op.id)).toEqual([work.id]); - }); + it.each(['completed', 'failed'] as const)( + 'does not pause an old host for a terminal %s cancellation', + async (state) => { + const api = await enrolled(); + const cleanup = api.queueHostOperation(api.host.id, 'instance.cancel-create', { + createOperationId: 'creation' + }); + api.reportHostOperation(api.host.id, cleanup.id, { state }); + const work = api.queueHostOperation(api.host.id, 'host.inspect', {}); + expect(api.pendingHostOperations(api.host.id).map((op) => op.id)).toEqual([work.id]); + } + ); it('rotates unresolved operation delivery so an old batch cannot starve new work', async () => { const api = await enrolled(); - const operations = Array.from({ length: 30 }, () => api.queueHostOperation(api.host.id, 'instance.inspect', {})); - operations.forEach(operation => api.reportHostOperation(api.host.id, operation.id, { state: 'unknown' })); - const first = api.pendingHostOperations(api.host.id).map(operation => operation.id); - const second = api.pendingHostOperations(api.host.id).map(operation => operation.id); + const operations = Array.from({ length: 30 }, () => + api.queueHostOperation(api.host.id, 'instance.inspect', {}) + ); + operations.forEach((operation) => + api.reportHostOperation(api.host.id, operation.id, { state: 'unknown' }) + ); + const first = api.pendingHostOperations(api.host.id).map((operation) => operation.id); + const second = api.pendingHostOperations(api.host.id).map((operation) => operation.id); expect(new Set([...first, ...second]).size).toBe(30); - expect(operations.every(operation => api.getHostOperation(operation.id)?.state === 'unknown')).toBe(true); + expect( + operations.every((operation) => api.getHostOperation(operation.id)?.state === 'unknown') + ).toBe(true); }); it('reconciles the same operation ID after loss of acknowledgement and prevents duplicate side effects', async () => { - const api = await enrolled(); const options = { instanceId: 'immutable-id', generation: 4, idempotencyKey: 'provisioning:123' }; + const api = await enrolled(); + const options = { + instanceId: 'immutable-id', + generation: 4, + idempotencyKey: 'provisioning:123' + }; const operation = api.queueHostOperation(api.host.id, 'instance.restart', {}, options); - api.reportHostOperation(api.host.id, operation.id, { state: 'accepted' }); api.reportHostOperation(api.host.id, operation.id, { state: 'running' }); - expect(api.queueHostOperation(api.host.id, 'instance.restart', {}, options).id).toBe(operation.id); - expect(api.pendingHostOperations(api.host.id).map(value => value.id)).toEqual([operation.id]); - expect(() => api.queueHostOperation(api.host.id, 'instance.stop', {}, options)).toThrow('different request'); - api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: { restarted: true } }); - expect(api.reportHostOperation(api.host.id, operation.id, { state: 'running' }).state).toBe('completed'); + api.reportHostOperation(api.host.id, operation.id, { state: 'accepted' }); + api.reportHostOperation(api.host.id, operation.id, { state: 'running' }); + expect(api.queueHostOperation(api.host.id, 'instance.restart', {}, options).id).toBe( + operation.id + ); + expect(api.pendingHostOperations(api.host.id).map((value) => value.id)).toEqual([ + operation.id + ]); + expect(() => api.queueHostOperation(api.host.id, 'instance.stop', {}, options)).toThrow( + 'different request' + ); + api.reportHostOperation(api.host.id, operation.id, { + state: 'completed', + result: { restarted: true } + }); + expect(api.reportHostOperation(api.host.id, operation.id, { state: 'running' }).state).toBe( + 'completed' + ); expect(api.pendingHostOperations(api.host.id)).toEqual([]); expect((await api.waitHostOperation(operation.id)).result).toEqual({ restarted: true }); }); it('does not allow another host to report a result, and timeouts retain the original operation', async () => { - const api = await enrolled(); const other = api.createHost({ name: 'Other' }); const operation = api.queueHostOperation(api.host.id, 'instance.stop', {}); - expect(() => api.reportHostOperation(other.id, operation.id, { state: 'completed' })).toThrow('this host'); + const api = await enrolled(); + const other = api.createHost({ name: 'Other' }); + const operation = api.queueHostOperation(api.host.id, 'instance.stop', {}); + expect(() => + api.reportHostOperation(other.id, operation.id, { state: 'completed' }) + ).toThrow('this host'); await expect(api.waitHostOperation(operation.id, 0)).rejects.toThrow(operation.id); expect(api.getHostOperation(operation.id)?.state).toBe('pending'); }); @@ -395,13 +739,15 @@ it('limits preparing hosts at delivery, preserves replays, and releases capacity api.enrollHost(api.createEnrollment(second.id).token, inventory); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 1 }); - const first = api.queueHostOperation(api.host.id, 'library.download', { libraryVersionId: 'one' }); + const first = api.queueHostOperation(api.host.id, 'library.download', { + libraryVersionId: 'one' + }); const next = api.queueHostOperation(second.id, 'library.download', { libraryVersionId: 'two' }); const inspect = api.queueHostOperation(second.id, 'host.inspect', {}); - expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id); - expect(api.pendingHostOperations(second.id).map(row => row.id)).not.toContain(next.id); - expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(inspect.id); - expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id); + expect(api.pendingHostOperations(api.host.id).map((row) => row.id)).toContain(first.id); + expect(api.pendingHostOperations(second.id).map((row) => row.id)).not.toContain(next.id); + expect(api.pendingHostOperations(second.id).map((row) => row.id)).toContain(inspect.id); + expect(api.pendingHostOperations(api.host.id).map((row) => row.id)).toContain(first.id); api.reportHostOperation(api.host.id, first.id, { state: 'completed', result: {} }); - expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(next.id); + expect(api.pendingHostOperations(second.id).map((row) => row.id)).toContain(next.id); }); diff --git a/tests/unit/native-launch-preparation.test.ts b/tests/unit/native-launch-preparation.test.ts index 0383693..b09e2ee 100644 --- a/tests/unit/native-launch-preparation.test.ts +++ b/tests/unit/native-launch-preparation.test.ts @@ -3,9 +3,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ - checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) + checkRelease: vi.fn(async () => {}), + freshRelease: () => 5354, + probeVersion: vi.fn(async () => 5354) +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; let hostId: string; @@ -15,23 +20,40 @@ const releases: (() => void)[] = []; const passes: Promise[] = []; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); + vi.resetModules(); + vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); - vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('PLUTAINER_IMAGE', image); + vi.stubEnv('RCON_PASSWORD', 'test-password'); - const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { createHost, createEnrollment, enrollHost, heartbeat } = + await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id; - const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3, - memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, - directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] }; + const inventory = { + cpuCount: 32, + cpuUsagePercent: 0, + memoryTotal: 64 * 1024 ** 3, + memoryAvailable: 60 * 1024 ** 3, + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 1_000_000_000_000, + runtimeImages: [image] + }; enrollHost(createEnrollment(hostId).token, inventory); - const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + const { createLibraryImport, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); libraryId = createLibraryImport({ hostId, game: 't5' }).id; - db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?") + db() + .prepare( + "UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?" + ) .run('a'.repeat(64), new Date().toISOString(), libraryId); - updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }]); + updateHostLibraryCaches(hostId, [ + { libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) } + ]); heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] }); }); @@ -46,14 +68,22 @@ afterEach(async () => { let queuedKeys = 0; async function queue() { const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', - hostId, libraryId, image, plutoKey: `saved-key-${++queuedKeys}` }); + const result = queueNativeCreation({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + hostId, + libraryId, + image, + plutoKey: `saved-key-${++queuedKeys}` + }); if (!result.ok) throw new Error(result.message); return result; } async function runPass() { - const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); + const { processNativeProvisioning } = + await import('../../src/lib/server/provisioning/native.js'); const pass = processNativeProvisioning(() => null); passes.push(pass); await pass; @@ -61,8 +91,12 @@ async function runPass() { async function operationFor(deploymentId: string) { const { db } = await import('../../src/lib/server/db/index.js'); - return db().prepare(`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id - WHERE n.deployment_id=?`).get(deploymentId) as { id: string } | undefined; + return db() + .prepare( + `SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id + WHERE n.deployment_id=?` + ) + .get(deploymentId) as { id: string } | undefined; } /** Pause after real bytes are captured so mutations exercise the final admission checks. */ @@ -71,15 +105,21 @@ async function pausePreparation() { const prepare = preparation.prepareNativeLaunch; let started!: () => void; let release!: () => void; - const ready = new Promise(resolve => { started = resolve; }); - const gate = new Promise(resolve => { release = resolve; }); - releases.push(release); - const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => { - const result = await prepare(input); - started(); - await gate; - return result; + const ready = new Promise((resolve) => { + started = resolve; }); + const gate = new Promise((resolve) => { + release = resolve; + }); + releases.push(release); + const spy = vi + .spyOn(preparation, 'prepareNativeLaunch') + .mockImplementationOnce(async (input) => { + const result = await prepare(input); + started(); + await gate; + return result; + }); return { ready, release, spy }; } @@ -90,21 +130,28 @@ describe('asynchronous native launch preparation', () => { const evaluate = preparation.evaluateNativePlacement; let started!: () => void; let release!: () => void; - const ready = new Promise(resolve => { started = resolve; }); - const gate = new Promise(resolve => { release = resolve; }); + const ready = new Promise((resolve) => { + started = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); releases.push(release); - vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async input => { + vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async (input) => { const result = await evaluate(input); - started(); await gate; + started(); + await gate; return result; }); const pass = runPass(); await ready; const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)).toBeNull(); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); - release(); await pass; + release(); + await pass; expect(nativeInstance(job.name)).toBeNull(); expect(await operationFor(job.deploymentId)).toBeUndefined(); const { db } = await import('../../src/lib/server/db/index.js'); @@ -114,52 +161,77 @@ describe('asynchronous native launch preparation', () => { it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 }); - const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll', - scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) }); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); + const { script, version } = createScript({ + name: 'Large launch plugin', + kind: 'plugin', + targetRelPath: 'fixture.dll', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.alloc(8 * 1024 ** 2, 42) + }); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(version.id, script.id); const jobs = await Promise.all(Array.from({ length: 4 }, queue)); const hosts = await import('../../src/lib/server/hosts/index.js'); const queueCommand = hosts.queueHostOperationSerialized; let yielded = true; let uninterruptedCommits = 0; - const committing = vi.spyOn(hosts, 'queueHostOperationSerialized').mockImplementation((...args) => { - if (!yielded) uninterruptedCommits++; - yielded = false; - setImmediate(() => { yielded = true; }); - return queueCommand(...args); - }); + const committing = vi + .spyOn(hosts, 'queueHostOperationSerialized') + .mockImplementation((...args) => { + if (!yielded) uninterruptedCommits++; + yielded = false; + setImmediate(() => { + yielded = true; + }); + return queueCommand(...args); + }); const parse = JSON.parse; const stringify = JSON.stringify; const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => { - if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread'); + if (typeof raw === 'string' && raw.length > 1_000_000) + throw new Error('Large file payload parsed on the controller thread'); return parse(raw, reviver); }); - const encoding = vi.spyOn(JSON, 'stringify').mockImplementation((...args: Parameters) => { - const encoded = stringify(...args); - if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread'); - return encoded; - }); + const encoding = vi + .spyOn(JSON, 'stringify') + .mockImplementation((...args: Parameters) => { + const encoded = stringify(...args); + if (encoded && encoded.length > 1_000_000) + throw new Error('Large file payload serialized on the controller thread'); + return encoded; + }); let ticks = 0; - const timer = setInterval(() => { ticks++; }, 1); + const timer = setInterval(() => { + ticks++; + }, 1); try { const pass = runPass(); - await new Promise(resolve => setTimeout(resolve, 0)); - for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files) - .toContainEqual(expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 })); + await new Promise((resolve) => setTimeout(resolve, 0)); + for (const job of jobs) + expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual( + expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 }) + ); await pass; expect(ticks).toBeGreaterThan(1); expect(committing).toHaveBeenCalledTimes(4); expect(uninterruptedCommits).toBe(0); for (const job of jobs) { expect(await operationFor(job.deploymentId)).toBeDefined(); - expect(getDeployment(job.deploymentId)?.recipe?.files) - .toContainEqual(expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 })); + expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual( + expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 }) + ); } } finally { - clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore(); + clearInterval(timer); + parsing.mockRestore(); + encoding.mockRestore(); + committing.mockRestore(); } }, 30_000); @@ -168,7 +240,8 @@ describe('asynchronous native launch preparation', () => { const paused = await pausePreparation(); const pass = runPass(); await paused.ready; - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); paused.release(); await pass; @@ -176,43 +249,92 @@ describe('asynchronous native launch preparation', () => { expect(await operationFor(job.deploymentId)).toBeUndefined(); expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined(); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' }); + expect( + db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId) + ).toEqual({ state: 'cancelled' }); }); - it.each(['baseline', 'config', 'script', 'plugin'] as const)('discards prepared bytes when the promoted %s changes before dispatch', async kind => { - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const baseline = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', - content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') }); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(baseline.version.id, baseline.script.id); - const isConfig = kind === 'baseline' || kind === 'config'; - const { script, version } = kind === 'baseline' ? baseline : createScript({ name: `Launch ${kind}`, kind, - targetRelPath: kind === 'script' ? 'fixture.gsc' : kind === 'plugin' ? 'fixture.dll' : undefined, - scopeType: kind === 'config' ? 'map' : 'game', scopeValue: kind === 'config' ? 'zombie_theater' : 't5', - content: Buffer.from(isConfig ? 'set sv_voice "1"\n' : 'before promotion') }); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); - const job = await queue(); - const paused = await pausePreparation(); - const pass = runPass(); - await paused.ready; - const content = Buffer.from(isConfig ? 'set rcon_password "password"\nset sv_voice "0"\n' : 'newly promoted file'); - const latest = uploadNewVersion(script.id, content); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id); - paused.release(); - await pass; - expect(await operationFor(job.deploymentId)).toBeUndefined(); - await runPass(); - const operation = await operationFor(job.deploymentId); - expect(operation).toBeDefined(); - const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] }; - if (isConfig) { - const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js'); - const expected = getPromotedServerConfig('t5', 'zombie_theater', 'theater', 'test-password')!; - expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: expected.toString('base64') })); - expect(expected.toString()).toContain('set sv_voice "0"'); - } else expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 })); - }); + it.each(['baseline', 'config', 'script', 'plugin'] as const)( + 'discards prepared bytes when the promoted %s changes before dispatch', + async (kind) => { + const { createScript, uploadNewVersion } = + await import('../../src/lib/server/fleet/scripts.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const baseline = createScript({ + name: 'Launch baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') + }); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(baseline.version.id, baseline.script.id); + const isConfig = kind === 'baseline' || kind === 'config'; + const { script, version } = + kind === 'baseline' + ? baseline + : createScript({ + name: `Launch ${kind}`, + kind, + targetRelPath: + kind === 'script' + ? 'fixture.gsc' + : kind === 'plugin' + ? 'fixture.dll' + : undefined, + scopeType: kind === 'config' ? 'map' : 'game', + scopeValue: kind === 'config' ? 'zombie_theater' : 't5', + content: Buffer.from(isConfig ? 'set sv_voice "1"\n' : 'before promotion') + }); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(version.id, script.id); + const job = await queue(); + const paused = await pausePreparation(); + const pass = runPass(); + await paused.ready; + const content = Buffer.from( + isConfig + ? 'set rcon_password "password"\nset sv_voice "0"\n' + : 'newly promoted file' + ); + const latest = uploadNewVersion(script.id, content); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(latest.id, script.id); + paused.release(); + await pass; + expect(await operationFor(job.deploymentId)).toBeUndefined(); + await runPass(); + const operation = await operationFor(job.deploymentId); + expect(operation).toBeDefined(); + const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const payload = getHostOperation(operation!.id)!.payload as { + files: { contentBase64: string; sha256: string }[]; + }; + if (isConfig) { + const { getPromotedServerConfig } = + await import('../../src/lib/server/fleet/server-config.js'); + const expected = getPromotedServerConfig( + 't5', + 'zombie_theater', + 'theater', + 'test-password' + )!; + expect(payload.files).toContainEqual( + expect.objectContaining({ contentBase64: expected.toString('base64') }) + ); + expect(expected.toString()).toContain('set sv_voice "0"'); + } else + expect(payload.files).toContainEqual( + expect.objectContaining({ + contentBase64: content.toString('base64'), + sha256: latest.sha256 + }) + ); + } + ); it('discards prepared bytes when the server key changes before dispatch', async () => { const job = await queue(); @@ -220,8 +342,12 @@ describe('asynchronous native launch preparation', () => { const pass = runPass(); await paused.ready; const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`UPDATE server_keys SET pluto_key='changed-key' WHERE id IN - (SELECT key_id FROM server_key_leases WHERE server_id=? AND is_current=1)`).run(job.name); + db() + .prepare( + `UPDATE server_keys SET pluto_key='changed-key' WHERE id IN + (SELECT key_id FROM server_key_leases WHERE server_id=? AND is_current=1)` + ) + .run(job.name); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); @@ -229,7 +355,9 @@ describe('asynchronous native launch preparation', () => { const operation = await operationFor(job.deploymentId); expect(operation).toBeDefined(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - expect(getHostOperation(operation!.id)!.payload).toMatchObject({ env: { PLUTO_SERVER_KEY: 'changed-key' } }); + expect(getHostOperation(operation!.id)!.payload).toMatchObject({ + env: { PLUTO_SERVER_KEY: 'changed-key' } + }); }); it('prepares only one launch while the single startup slot is reserved', async () => { @@ -239,7 +367,8 @@ describe('asynchronous native launch preparation', () => { const pass = runPass(); await paused.ready; expect(paused.spy).toHaveBeenCalledTimes(1); - const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); + const { processNativeProvisioning } = + await import('../../src/lib/server/provisioning/native.js'); await processNativeProvisioning(() => null, { wait: false }); expect(paused.spy).toHaveBeenCalledTimes(1); paused.release(); @@ -258,13 +387,17 @@ describe('asynchronous native launch preparation', () => { paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); - expect(updateHost(hostId, { mode: 'active' })).toMatchObject({ mode: 'disabled', readiness: 'preparing' }); + expect(updateHost(hostId, { mode: 'active' })).toMatchObject({ + mode: 'disabled', + readiness: 'preparing' + }); await runPass(); expect(await operationFor(job.deploymentId)).toBeUndefined(); }); it('rechecks a reduced startup limit before submitting prepared files', async () => { - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); const first = await queue(); await runPass(); diff --git a/tests/unit/native-observations.test.ts b/tests/unit/native-observations.test.ts index 8eb7ce8..0cc49a5 100644 --- a/tests/unit/native-observations.test.ts +++ b/tests/unit/native-observations.test.ts @@ -3,8 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); async function setup(capabilities: string[] = []) { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -12,52 +17,94 @@ async function setup(capabilities: string[] = []) { hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, capabilities }); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO servers(name) VALUES('test')").run(); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('instance','test',?,'library','image',28960,?)`).run(host.id, new Date().toISOString()); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('instance','test',?,'library','image',28960,?)` + ) + .run(host.id, new Date().toISOString()); const native = await import('../../src/lib/server/backends/native.js'); const instance = native.nativeInstance('test')!; return { ...hosts, ...native, db: db(), host, instance }; } describe('overlapping native observations', () => { - it.each([false, true])('negotiates incremental log reads with agent support: %s', async incremental => { - const api = await setup(incremental ? ['logs.read-offset'] : []); - const { nativeLogChunk } = await import('../../src/lib/server/backends/files.js'); - const cursor = { fileId: '1:2', offset: 3, size: 3, modified: '4', anchorLength: 3, anchorHash: 'hash' }; - const request = nativeLogChunk(join(fixture.t5, 'test', 'logs/games_sp.log'), cursor); - const operation = api.listHostOperations(api.host.id)[0]; - expect(operation.payload).toEqual({ path: 'logs/games_sp.log', ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) }); - const result = incremental ? { contentBase64: '', cursor, reset: false } : { lines: ['legacy tail'] }; - api.reportHostOperationStatus(api.host.id, operation.id, { state: 'completed', result }); - expect(await request).toEqual(result); - expect(api.getHostOperation(operation.id)?.result).toMatchObject({ retained: false }); - }); + it.each([false, true])( + 'negotiates incremental log reads with agent support: %s', + async (incremental) => { + const api = await setup(incremental ? ['logs.read-offset'] : []); + const { nativeLogChunk } = await import('../../src/lib/server/backends/files.js'); + const cursor = { + fileId: '1:2', + offset: 3, + size: 3, + modified: '4', + anchorLength: 3, + anchorHash: 'hash' + }; + const request = nativeLogChunk(join(fixture.t5, 'test', 'logs/games_sp.log'), cursor); + const operation = api.listHostOperations(api.host.id)[0]; + expect(operation.payload).toEqual({ + path: 'logs/games_sp.log', + ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) + }); + const result = incremental + ? { contentBase64: '', cursor, reset: false } + : { lines: ['legacy tail'] }; + api.reportHostOperationStatus(api.host.id, operation.id, { + state: 'completed', + result + }); + expect(await request).toEqual(result); + expect(api.getHostOperation(operation.id)?.result).toMatchObject({ retained: false }); + } + ); - it.each(['files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read', 'instance.inspect'])('shares %s only until completion', async kind => { + it.each([ + 'files.read', + 'files.list', + 'files.stat', + 'files.manifest', + 'logs.read', + 'instance.inspect' + ])('shares %s only until completion', async (kind) => { const api = await setup(); - const requests = Array.from({ length: 4 }, () => api.nativeRequest(api.instance, kind, { path: 'same' })); + const requests = Array.from({ length: 4 }, () => + api.nativeRequest(api.instance, kind, { path: 'same' }) + ); const operations = api.listHostOperations(api.host.id); expect(operations).toHaveLength(1); - api.reportHostOperation(api.host.id, operations[0].id, { state: 'completed', result: { value: 'first' } }); + api.reportHostOperation(api.host.id, operations[0].id, { + state: 'completed', + result: { value: 'first' } + }); expect(await Promise.all(requests)).toEqual(Array(4).fill({ value: 'first' })); const fresh = api.nativeRequest(api.instance, kind, { path: 'same' }); const next = api.listHostOperations(api.host.id)[0]; expect(next.id).not.toBe(operations[0].id); - api.reportHostOperation(api.host.id, next.id, { state: 'completed', result: { value: 'changed' } }); + api.reportHostOperation(api.host.id, next.id, { + state: 'completed', + result: { value: 'changed' } + }); expect(await fresh).toEqual({ value: 'changed' }); }); it('does not merge different paths, mutations, or reads across lifecycle submissions', async () => { const api = await setup(); - const requests = [api.nativeRequest(api.instance, 'files.read', { path: 'one' }), + const requests = [ + api.nativeRequest(api.instance, 'files.read', { path: 'one' }), api.nativeRequest(api.instance, 'files.read', { path: 'two' }), api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' }), - api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' })]; - api.db.prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('test',?)").run(new Date().toISOString()); + api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' }) + ]; + api.db + .prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('test',?)") + .run(new Date().toISOString()); requests.push(api.nativeRequest(api.instance, 'files.read', { path: 'one' })); const operations = api.listHostOperations(api.host.id); expect(operations).toHaveLength(5); - for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: {} }); + for (const operation of operations) + api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: {} }); await Promise.all(requests); }); @@ -65,11 +112,17 @@ describe('overlapping native observations', () => { const api = await setup(); const first = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); const rejected = expect(first).rejects.toThrow('unavailable'); - api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'failed', error: 'unavailable' }); + api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { + state: 'failed', + error: 'unavailable' + }); await rejected; const next = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); expect(api.listHostOperations(api.host.id)).toHaveLength(2); - api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'completed', result: {} }); + api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { + state: 'completed', + result: {} + }); await next; }); @@ -78,12 +131,16 @@ describe('overlapping native observations', () => { const old = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); const rejected = expect(old).rejects.toThrow('Instance was replaced'); api.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run(); - const current = api.nativeRequest(api.nativeInstance('test')!, 'files.read', { path: 'one' }); + const current = api.nativeRequest(api.nativeInstance('test')!, 'files.read', { + path: 'one' + }); const operations = api.listHostOperations(api.host.id); expect(operations).toHaveLength(2); - for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, { - state: 'completed', result: { generation: operation.generation } - }); + for (const operation of operations) + api.reportHostOperation(api.host.id, operation.id, { + state: 'completed', + result: { generation: operation.generation } + }); await rejected; expect(await current).toEqual({ generation: 2 }); }); @@ -98,30 +155,48 @@ describe('native file reads through stored agent receipts', () => { const { readFile } = await import('../../src/lib/server/backends/files.js'); const reading = readFile(join(fixture.t5, 'test', 'configs/server.cfg')); const operation = api.listHostOperations(api.host.id)[0]; - expect(operation).toMatchObject({ kind: 'files.read', payload: { path: 'configs/server.cfg' } }); - api.reportHostOperationStatus(api.host.id, operation.id, { - state: 'completed', result: { contentBase64: content.toString('base64') } + expect(operation).toMatchObject({ + kind: 'files.read', + payload: { path: 'configs/server.cfg' } + }); + api.reportHostOperationStatus(api.host.id, operation.id, { + state: 'completed', + result: { contentBase64: content.toString('base64') } + }); + expect(api.getHostOperationStatus(operation.id)?.result).toEqual({ + contentSha256: expect.any(String) }); - expect(api.getHostOperationStatus(operation.id)?.result).toEqual({ contentSha256: expect.any(String) }); expect(await reading).toEqual(content); }); it('preserves missing-file errors from the agent', async () => { const api = await setup(); const { readFile } = await import('../../src/lib/server/backends/files.js'); - const reading = expect(readFile(join(fixture.t5, 'test', 'missing'))).rejects.toMatchObject({ code: 'ENOENT' }); + const reading = expect(readFile(join(fixture.t5, 'test', 'missing'))).rejects.toMatchObject( + { code: 'ENOENT' } + ); api.reportHostOperationStatus(api.host.id, api.listHostOperations(api.host.id)[0].id, { - state: 'completed', result: { contentBase64: null } + state: 'completed', + result: { contentBase64: null } }); await reading; }); - it.each([null, {}, { contentBase64: 123 }, { contentSha256: 123 }])('reports an invalid response for malformed file content: %j', async result => { - const api = await setup(); - const { readFile } = await import('../../src/lib/server/backends/files.js'); - const reading = expect(readFile(join(fixture.t5, 'test', 'configs/server.cfg'))) - .rejects.toThrow('Agent returned an invalid response for files.read: configs/server.cfg'); - api.reportHostOperationStatus(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'completed', result }); - await reading; - }); + it.each([null, {}, { contentBase64: 123 }, { contentSha256: 123 }])( + 'reports an invalid response for malformed file content: %j', + async (result) => { + const api = await setup(); + const { readFile } = await import('../../src/lib/server/backends/files.js'); + const reading = expect( + readFile(join(fixture.t5, 'test', 'configs/server.cfg')) + ).rejects.toThrow( + 'Agent returned an invalid response for files.read: configs/server.cfg' + ); + api.reportHostOperationStatus(api.host.id, api.listHostOperations(api.host.id)[0].id, { + state: 'completed', + result + }); + await reading; + } + ); }); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index e5e73fc..e35875d 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -6,26 +6,61 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } })), - rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) + rconStatusResult: vi.fn(async () => ({ + ok: true, + endpoint: { host: 'agent', port: 28960 }, + status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } + })), + rconGetPlayerCount: vi.fn(async () => remote.players), + rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) +})); +vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ + checkRelease: vi.fn(async () => {}), + freshRelease: () => 5354, + probeVersion: vi.fn(async () => 5354) +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({ - rebalanceAdmission: vi.fn(() => null), requestRebalanceStopForOperation: vi.fn() + rebalanceAdmission: vi.fn(() => null), + requestRebalanceStopForOperation: vi.fn() })); let queuedKeys = 0; -let fixture: Fixture; let hostId: string; let libraryId: string; +let fixture: Fixture; +let hostId: string; +let libraryId: string; const image = 'example.test/plutainer:test'; -const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, capabilities: ['instance.unhealthy-recovery'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] }); +const inventory = () => ({ + cpuCount: 16, + cpuUsagePercent: 10, + memoryTotal: 64 * 1024 ** 3, + memoryAvailable: 60 * 1024 ** 3, + protocolVersion: 1, + capabilities: ['instance.unhealthy-recovery'], + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100_000_000_000, + runtimeImages: [image], + libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] +}); beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0; + vi.resetModules(); + vi.clearAllMocks(); + remote.state = 'running'; + remote.players = 0; + remote.map = 'zombie_theater'; + libraryId = ''; + queuedKeys = 0; fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); - vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('PLUTAINER_IMAGE', image); + vi.stubEnv('RCON_PASSWORD', 'test-password'); - const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + const { createHost, createEnrollment, enrollHost } = + await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id; enrollHost(createEnrollment(hostId).token, inventory()); const backend = await import('../../src/lib/server/backends/native.js'); @@ -34,42 +69,69 @@ beforeEach(async () => { return { healthy: remote.state === 'running', state: remote.state, exists: true } as never; }); }); -afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { + vi.useRealTimers(); + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); async function prepareLibrary(game: 't5' | 't6' = 't5') { const libs = await import('../../src/lib/server/game-files/index.js'); const version = libs.createLibraryImport({ hostId, game }); const bytes = Buffer.from('verified base game'); - const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; + const file = { + path: 'zone/base.ff', + size: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex') + }; libs.setImportManifest(hostId, version.id, [file]); await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body); - const ready = await libs.publishImport(hostId, version.id); libraryId = ready.id; - libs.updateHostLibraryCaches(hostId, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); + const ready = await libs.publishImport(hostId, version.id); + libraryId = ready.id; + libs.updateHostLibraryCaches(hostId, [ + { libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 } + ]); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + heartbeat(hostId, inventory()); return ready; } async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') { - await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit'; + await prepareLibrary(game); + remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit'; const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); - const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', - hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}` }); + const result = queueCreateServer({ + game, + gameMode: 'zombie', + mapSlug: game === 't5' ? 'theater' : 'town', + hostId: placement === 'pinned' ? hostId : undefined, + libraryId, + image, + plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}` + }); if (!result.ok) throw new Error(result.message); return result; } async function operationFor(deploymentId: string) { const { db } = await import('../../src/lib/server/db/index.js'); - const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(deploymentId) as { operation_id: string | null }; + const row = db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(deploymentId) as { operation_id: string | null }; const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); return row?.operation_id ? getHostOperation(row.operation_id)! : null; } async function advance() { - const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); + const { processNativeProvisioning } = + await import('../../src/lib/server/provisioning/native.js'); await processNativeProvisioning(() => null); } async function acknowledge(deploymentId: string) { - const operation = await operationFor(deploymentId); if (!operation) throw new Error('Missing durable native operation'); + const operation = await operationFor(deploymentId); + if (!operation) throw new Error('Missing durable native operation'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: remote.state, healthy: remote.state === 'running' } }); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { state: remote.state, healthy: remote.state === 'running' } + }); await advance(); return operation; } @@ -78,22 +140,36 @@ async function detail(id: string) { return getDeployment(id)!.deployment; } - async function observations(server: string) { const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { serverObservation, observationReader } = await import('../../src/lib/server/fleet/status.js'); - const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js'); + const { serverObservation, observationReader } = + await import('../../src/lib/server/fleet/status.js'); + const { MonitoringSnapshot } = + await import('../../src/lib/server/fleet/monitoring-snapshot.js'); const row = getServer(server)!; - return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)]; + return [ + serverObservation(row), + observationReader()(row), + new MonitoringSnapshot().observe(row) + ]; } async function moveTarget() { const hosts = await import('../../src/lib/server/hosts/index.js'); const libraries = await import('../../src/lib/server/game-files/index.js'); - const target = hosts.createHost({ name: 'Rebalance destination', portMin: 29000, portMax: 29005 }); + const target = hosts.createHost({ + name: 'Rebalance destination', + portMin: 29000, + portMax: 29005 + }); hosts.enrollHost(hosts.createEnrollment(target.id).token, inventory()); - libraries.updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', - manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]); + libraries.updateHostLibraryCaches(target.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 + } + ]); return target; } @@ -101,69 +177,144 @@ describe('rebalance instance migrations', () => { it('preserves deployed configuration, patched file versions, identity and credentials on an active source', async () => { const { config } = await import('../../src/lib/server/config.js'); Object.assign(config.rcon, { password: '' }); - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, uploadNewVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const baseline = createScript({ name: 'Deployed baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', - content: Buffer.from('set sv_voice "1"\n') }); - const script = createScript({ name: 'Deployed script', kind: 'script', targetRelPath: 'preserved.gsc', - scopeType: 'game', scopeValue: 't5', content: Buffer.from('original deployed script') }); - for (const artifact of [baseline, script]) db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id); - const job = await queue(); await advance(); - const launch = (await operationFor(job.deploymentId))!.payload as { rconPassword: string; files: { path: string; sha256: string }[] }; + const baseline = createScript({ + name: 'Deployed baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('set sv_voice "1"\n') + }); + const script = createScript({ + name: 'Deployed script', + kind: 'script', + targetRelPath: 'preserved.gsc', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('original deployed script') + }); + for (const artifact of [baseline, script]) + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(artifact.version.id, artifact.script.id); + const job = await queue(); + await advance(); + const launch = (await operationFor(job.deploymentId))!.payload as { + rconPassword: string; + files: { path: string; sha256: string }[]; + }; await acknowledge(job.deploymentId); - const patched = uploadNewVersion(script.script.id, Buffer.from('confirmed patch after initial launch')); + const patched = uploadNewVersion( + script.script.id, + Buffer.from('confirmed patch after initial launch') + ); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); - recordScript(job.name, 'preserved.gsc', patched.id, patched.sha256, patched.size, 'campaign'); - const newest = uploadNewVersion(script.script.id, Buffer.from('new promotion must not be installed')); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newest.id, script.script.id); + recordScript( + job.name, + 'preserved.gsc', + patched.id, + patched.sha256, + patched.size, + 'campaign' + ); + const newest = uploadNewVersion( + script.script.id, + Buffer.from('new promotion must not be installed') + ); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(newest.id, script.script.id); const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n')); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newBaseline.id, baseline.script.id); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(newBaseline.id, baseline.script.id); Object.assign(config.rcon, { password: 'new-controller-password' }); const target = await moveTarget(); - const { captureMoveRecipe, requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js'); + const { captureMoveRecipe, requestInstanceMove, hostMigration } = + await import('../../src/lib/server/provisioning/host-drain.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const original = nativeInstance(job.name)!; const recipe = captureMoveRecipe(job.name); - expect(recipe.files.every(file => file.content === undefined)).toBe(true); - const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test', recipe }); + expect(recipe.files.every((file) => file.content === undefined)).toBe(true); + const id = requestInstanceMove(job.name, target.id, { + triggeredBy: 'rebalance:test', + recipe + }); await advance(); const removal = hostMigration(id)!.remove_operation_id!; expect(removal).toBeTruthy(); - const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob, canCancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(canCancelProvisioningJob(id)).toBe(false); expect(cancelProvisioningJob(id)).toBe(false); - const { getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js'); + const { getDeployment, listDeployments } = + await import('../../src/lib/server/provisioning/history.js'); expect(getDeployment(id)?.canCancel).toBe(false); - expect(listDeployments().find(item => item.id === id)?.canCancel).toBe(0); + expect(listDeployments().find((item) => item.id === id)?.canCancel).toBe(0); const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js'); const response = POST({ params: { id } }); expect(response.status).toBe(409); - expect(await response.json()).toMatchObject({ error: expect.stringContaining('must finish recreating') }); - const { reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js'); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('must finish recreating') + }); + const { reportHostOperation, getHost } = + await import('../../src/lib/server/hosts/index.js'); reportHostOperation(hostId, removal, { state: 'completed' }); - await advance(); await advance(); + await advance(); + await advance(); const creation = (await operationFor(id))!; - expect(creation).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2, - payload: { name: job.name, libraryId, image, rconPassword: launch.rconPassword, env: { PLUTO_SERVER_KEY: 'saved-key' } } }); + expect(creation).toMatchObject({ + hostId: target.id, + kind: 'instance.create', + generation: 2, + payload: { + name: job.name, + libraryId, + image, + rconPassword: launch.rconPassword, + env: { PLUTO_SERVER_KEY: 'saved-key' } + } + }); const payload = creation.payload as { files: { path: string; sha256: string }[] }; - expect(payload.files).toContainEqual(expect.objectContaining({ path: expect.stringContaining('preserved.gsc'), sha256: patched.sha256 })); - expect(payload.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256) - .toBe(launch.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256); - reportHostOperation(target.id, creation.id, { state: 'completed', result: { state: 'running', healthy: true } }); + expect(payload.files).toContainEqual( + expect.objectContaining({ + path: expect.stringContaining('preserved.gsc'), + sha256: patched.sha256 + }) + ); + expect(payload.files.find((file) => file.path === 'configs/dedicated_sp.cfg')?.sha256).toBe( + launch.files.find((file) => file.path === 'configs/dedicated_sp.cfg')?.sha256 + ); + reportHostOperation(target.id, creation.id, { + state: 'completed', + result: { state: 'running', healthy: true } + }); await advance(); expect((await detail(id)).state).toBe('done'); - expect(nativeInstance(job.name)).toMatchObject({ id: original.id, host_id: target.id, generation: 2 }); + expect(nativeInstance(job.name)).toMatchObject({ + id: original.id, + host_id: target.id, + generation: 2 + }); expect(getHost(hostId)?.mode).toBe('active'); expect(JSON.stringify(getDeployment(id)?.recipe)).not.toContain(launch.rconPassword); }); it('rechecks occupancy, target capacity and run admission before removing the source', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const target = await moveTarget(); - const { requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js'); - expect(() => requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' })).toThrow('different destination'); + const { requestInstanceMove, hostMigration } = + await import('../../src/lib/server/provisioning/host-drain.js'); + expect(() => + requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' }) + ).toThrow('different destination'); const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' }); - remote.players = 2; await advance(); + remote.players = 2; + await advance(); expect(hostMigration(id)?.remove_operation_id).toBeNull(); remote.players = 0; const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); @@ -171,7 +322,8 @@ describe('rebalance instance migrations', () => { await advance(); expect(hostMigration(id)?.remove_operation_id).toBeNull(); heartbeat(target.id, inventory()); - const { rebalanceAdmission } = await import('../../src/lib/server/rebalancing/admission.js'); + const { rebalanceAdmission } = + await import('../../src/lib/server/rebalancing/admission.js'); vi.mocked(rebalanceAdmission).mockReturnValueOnce('Run stopped during occupancy check.'); await advance(); expect(hostMigration(id)?.remove_operation_id).toBeNull(); @@ -180,113 +332,190 @@ describe('rebalance instance migrations', () => { }); it('withdraws only uncommitted moves and preserves their server keys', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const target = await moveTarget(); - const { requestInstanceMove } = await import('../../src/lib/server/provisioning/host-drain.js'); + const { requestInstanceMove } = + await import('../../src/lib/server/provisioning/host-drain.js'); const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' }); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(id)).toBe(true); - const { requestRebalanceStopForOperation } = await import('../../src/lib/server/rebalancing/admission.js'); + const { requestRebalanceStopForOperation } = + await import('../../src/lib/server/rebalancing/admission.js'); expect(requestRebalanceStopForOperation).toHaveBeenCalledWith(id); await advance(); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(job.name)).toBe('saved-key'); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toEqual([]); }); it('refuses to move when recorded deployed bytes cannot be restored', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { captureMoveRecipe } = await import('../../src/lib/server/provisioning/host-drain.js'); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { captureMoveRecipe } = + await import('../../src/lib/server/provisioning/host-drain.js'); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_scripts SET sha256=? WHERE server=?").run('0'.repeat(64), job.name); + db() + .prepare('UPDATE server_scripts SET sha256=? WHERE server=?') + .run('0'.repeat(64), job.name); expect(() => captureMoveRecipe(job.name)).toThrow('deployed bytes are unavailable'); }); }); describe('runtime configuration at dispatch', () => { it.each([ - ['t5', 'shared'], ['t5', 'generated'], ['t6', 'shared'], ['t6', 'generated'] - ] as const)('dispatches %s with a promoted baseline and %s RCON password', async (game, source) => { - const { config, configFileForGame } = await import('../../src/lib/server/config.js'); - if (source === 'generated') Object.assign(config.rcon, { password: '' }); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { script, version } = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: game, - content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') }); - db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); + ['t5', 'shared'], + ['t5', 'generated'], + ['t6', 'shared'], + ['t6', 'generated'] + ] as const)( + 'dispatches %s with a promoted baseline and %s RCON password', + async (game, source) => { + const { config, configFileForGame } = await import('../../src/lib/server/config.js'); + if (source === 'generated') Object.assign(config.rcon, { password: '' }); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { script, version } = createScript({ + name: 'Launch baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: game, + content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') + }); + db() + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(version.id, script.id); - const job = await queue(game); - await advance(); - const operation = await operationFor(job.deploymentId); - expect(operation).toMatchObject({ kind: 'instance.create', state: 'pending' }); - const payload = operation!.payload as { rconPassword: string; files: { path: string; contentBase64: string; sha256: string }[] }; - expect(payload.rconPassword).toEqual(source === 'shared' ? 'test-password' : expect.stringMatching(/^[a-f0-9]{64}$/)); - const configFile = configFileForGame(game); - const file = payload.files.find(file => file.path === `configs/${configFile}`)!; - const content = Buffer.from(file.contentBase64, 'base64'); - expect(content.toString()).toContain(`set rcon_password "${payload.rconPassword}"`); - expect(content.toString()).not.toContain('set rcon_password "password"'); - const hash = createHash('sha256').update(content).digest('hex'); - expect(file.sha256).toBe(hash); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - expect(getServerScript(job.name, configFile)).toMatchObject({ sha256: hash, size: content.length, source: 'native-create' }); - const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); - expect(recipeForDeployment(job.deploymentId)!.recipe.files).toContainEqual(expect.objectContaining({ - path: configFile, sha256: hash, size: content.length, content: content.toString('base64') - })); - }); + const job = await queue(game); + await advance(); + const operation = await operationFor(job.deploymentId); + expect(operation).toMatchObject({ kind: 'instance.create', state: 'pending' }); + const payload = operation!.payload as { + rconPassword: string; + files: { path: string; contentBase64: string; sha256: string }[]; + }; + expect(payload.rconPassword).toEqual( + source === 'shared' ? 'test-password' : expect.stringMatching(/^[a-f0-9]{64}$/) + ); + const configFile = configFileForGame(game); + const file = payload.files.find((file) => file.path === `configs/${configFile}`)!; + const content = Buffer.from(file.contentBase64, 'base64'); + expect(content.toString()).toContain(`set rcon_password "${payload.rconPassword}"`); + expect(content.toString()).not.toContain('set rcon_password "password"'); + const hash = createHash('sha256').update(content).digest('hex'); + expect(file.sha256).toBe(hash); + const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); + expect(getServerScript(job.name, configFile)).toMatchObject({ + sha256: hash, + size: content.length, + source: 'native-create' + }); + const { recipeForDeployment } = + await import('../../src/lib/server/fleet/launch-recipes.js'); + expect(recipeForDeployment(job.deploymentId)!.recipe.files).toContainEqual( + expect.objectContaining({ + path: configFile, + sha256: hash, + size: content.length, + content: content.toString('base64') + }) + ); + } + ); }); describe('provisioning health handoff', () => { - it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => { - const job = await queue(); await advance(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - let id = job.deploymentId; - if (action !== 'create') { - await acknowledge(id); - db().prepare('UPDATE servers SET container_state=? WHERE name=?').run(action === 'start' ? 'stopped' : 'running', job.name); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const request = requestLifecycle(job.name, action); - if (!request.ok) throw new Error(request.error); - id = request.deploymentId; + it.each(['create', 'start', 'restart', 'rebuild'] as const)( + '%s publishes verified game health before completion becomes visible', + async (action) => { + const job = await queue(); await advance(); - } - const old = new Date(Date.now() - 120_000).toISOString(); - db().prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]', - last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`).run(old, old, old, job.name); - vi.useFakeTimers({ toFake: ['Date'] }); - vi.setSystemTime(Date.now() + 1000); - const observedAt = new Date().toISOString(); - const roster = [{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }]; - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'agent', port: 28960 }, - status: { players: 1, map: remote.map, gametype: 'zom', roster } }); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); - const visible: string[] = []; - const unsubscribe = subscribe(event => { - if (event.type === 'fleet-invalidated' && getDeployment(id)?.deployment.state === 'done') { - visible.push(serverObservation(getServer(job.name)!).health); + const { db } = await import('../../src/lib/server/db/index.js'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + let id = job.deploymentId; + if (action !== 'create') { + await acknowledge(id); + db() + .prepare('UPDATE servers SET container_state=? WHERE name=?') + .run(action === 'start' ? 'stopped' : 'running', job.name); + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + const request = requestLifecycle(job.name, action); + if (!request.ok) throw new Error(request.error); + id = request.deploymentId; + await advance(); } - }); - try { await acknowledge(id); } finally { unsubscribe(); } - expect((await detail(id)).state).toBe('done'); - expect(visible).toEqual(['online']); - expect(getServer(job.name)).toMatchObject({ last_checked_at: observedAt, last_seen_at: observedAt, - last_players: 1, last_roster: JSON.stringify(roster), last_reply_snippet: null, - last_error: null, rcon_failures: 0, active_since: observedAt, inactive_since: null }); - for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 1, roster }); - }); + const old = new Date(Date.now() - 120_000).toISOString(); + db() + .prepare( + `UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]', + last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?` + ) + .run(old, old, old, job.name); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 1000); + const observedAt = new Date().toISOString(); + const roster = [ + { num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 } + ]; + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ + ok: true, + endpoint: { host: 'agent', port: 28960 }, + status: { players: 1, map: remote.map, gametype: 'zom', roster } + }); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); + const visible: string[] = []; + const unsubscribe = subscribe((event) => { + if ( + event.type === 'fleet-invalidated' && + getDeployment(id)?.deployment.state === 'done' + ) { + visible.push(serverObservation(getServer(job.name)!).health); + } + }); + try { + await acknowledge(id); + } finally { + unsubscribe(); + } + expect((await detail(id)).state).toBe('done'); + expect(visible).toEqual(['online']); + expect(getServer(job.name)).toMatchObject({ + last_checked_at: observedAt, + last_seen_at: observedAt, + last_players: 1, + last_roster: JSON.stringify(roster), + last_reply_snippet: null, + last_error: null, + rcon_failures: 0, + active_since: observedAt, + inactive_since: null + }); + for (const status of await observations(job.name)) + expect(status).toMatchObject({ health: 'online', last_players: 1, roster }); + } + ); it('completes directly from persisted telemetry without inventing an RCON observation', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const operation = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } }); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { state: 'running', healthy: true } + }); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 1000); const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); const { gameEvent } = await import('../helpers/telemetry.js'); const receivedAt = new Date().toISOString(); @@ -294,13 +523,23 @@ describe('provisioning health handoff', () => { await advance(); expect((await detail(job.deploymentId)).state).toBe('done'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - expect(getServer(job.name)).toMatchObject({ last_seen_at: null, inactive_since: receivedAt }); - for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', status_source: 'telemetry', last_seen_at: receivedAt }); + expect(getServer(job.name)).toMatchObject({ + last_seen_at: null, + inactive_since: receivedAt + }); + for (const status of await observations(job.name)) + expect(status).toMatchObject({ + health: 'online', + status_source: 'telemetry', + last_seen_at: receivedAt + }); }); it('rechecks expired verification evidence instead of renewing its timestamps', async () => { - const job = await queue(); await advance(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); + const job = await queue(); + await advance(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 1000); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); vi.mocked(probeVersion).mockImplementationOnce(async () => { @@ -312,60 +551,108 @@ describe('provisioning health handoff', () => { expect((await detail(job.deploymentId)).state).toBe('awaiting-health'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); expect(getServer(job.name)?.last_seen_at).toBeNull(); - for (const status of await observations(job.name)) expect(status.health).toBe('provisioning'); + for (const status of await observations(job.name)) + expect(status.health).toBe('provisioning'); await advance(); expect((await detail(job.deploymentId)).state).toBe('done'); expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString()); }); - it.each(['instance', 'lifecycle'] as const)('discards verification when the %s changes during later checks', async change => { - const job = await queue(); await advance(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - vi.mocked(probeVersion).mockImplementationOnce(async () => { - if (change === 'instance') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); - else db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() + 1000).toISOString(), job.deploymentId); - return 5354; - }); - await acknowledge(job.deploymentId); - expect((await detail(job.deploymentId)).state).toBe('awaiting-health'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - expect(getServer(job.name)).toMatchObject({ last_seen_at: null, container_checked_at: null }); - }); + it.each(['instance', 'lifecycle'] as const)( + 'discards verification when the %s changes during later checks', + async (change) => { + const job = await queue(); + await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + vi.mocked(probeVersion).mockImplementationOnce(async () => { + if (change === 'instance') + db() + .prepare( + 'UPDATE native_instances SET generation=generation+1 WHERE server=?' + ) + .run(job.name); + else + db() + .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') + .run(new Date(Date.now() + 1000).toISOString(), job.deploymentId); + return 5354; + }); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('awaiting-health'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)).toMatchObject({ + last_seen_at: null, + container_checked_at: null + }); + } + ); - it.each(['online', 'offline'] as const)('preserves newer %s monitoring evidence received during version verification', async health => { - const job = await queue(); await advance(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); - const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - let newer: string; - vi.mocked(probeVersion).mockImplementationOnce(async () => { - vi.setSystemTime(Date.now() + 1000); newer = new Date().toISOString(); - db().prepare(`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?, - last_players=2,last_roster='[]',last_error=NULL WHERE name=?`).run(health === 'online' ? 'running' : 'stopped', newer, newer, newer, job.name); - return 5354; - }); - await acknowledge(job.deploymentId); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - expect(getServer(job.name)).toMatchObject({ last_seen_at: newer!, container_checked_at: newer!, last_players: 2 }); - expect((await detail(job.deploymentId)).state).toBe(health === 'online' ? 'done' : 'awaiting-health'); - if (health === 'online') for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 2 }); - }); + it.each(['online', 'offline'] as const)( + 'preserves newer %s monitoring evidence received during version verification', + async (health) => { + const job = await queue(); + await advance(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 1000); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + let newer: string; + vi.mocked(probeVersion).mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 1000); + newer = new Date().toISOString(); + db() + .prepare( + `UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?, + last_players=2,last_roster='[]',last_error=NULL WHERE name=?` + ) + .run( + health === 'online' ? 'running' : 'stopped', + newer, + newer, + newer, + job.name + ); + return 5354; + }); + await acknowledge(job.deploymentId); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)).toMatchObject({ + last_seen_at: newer!, + container_checked_at: newer!, + last_players: 2 + }); + expect((await detail(job.deploymentId)).state).toBe( + health === 'online' ? 'done' : 'awaiting-health' + ); + if (health === 'online') + for (const status of await observations(job.name)) + expect(status).toMatchObject({ health: 'online', last_players: 2 }); + } + ); it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const request = requestLifecycle(job.name, 'restart'); if (!request.ok) throw new Error(request.error); await advance(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 20_000); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 20_000); for (const status of await observations(job.name)) expect(status.health).toBe('starting'); const { db } = await import('../../src/lib/server/db/index.js'); for (const state of ['failed', 'cancelled', 'interrupted']) { - db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, request.deploymentId); - for (const status of await observations(job.name)) expect(status.health).toBe('offline'); + db() + .prepare('UPDATE server_deployments SET state=? WHERE id=?') + .run(state, request.deploymentId); + for (const status of await observations(job.name)) + expect(status.health).toBe('offline'); } - db().prepare("UPDATE server_deployments SET state='running' WHERE id=?").run(request.deploymentId); + db() + .prepare("UPDATE server_deployments SET state='running' WHERE id=?") + .run(request.deploymentId); await acknowledge(request.deploymentId); expect((await detail(request.deploymentId)).state).toBe('done'); for (const status of await observations(job.name)) expect(status.health).toBe('online'); @@ -376,35 +663,66 @@ describe('native instance provisioning', () => { it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => { const job = await queue(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { nativeDeployment, nativeInstance } = await import('../../src/lib/server/backends/native.js'); + const { nativeDeployment, nativeInstance } = + await import('../../src/lib/server/backends/native.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); expect(job.port).toBeNull(); expect(nativeDeployment(job.deploymentId)).toBe(true); expect(nativeInstance(job.name)).toBeNull(); - expect(getDeployment(job.deploymentId)).toMatchObject({ backend: 'native', canCancelCreation: true }); - expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([{ deployment_id: job.deploymentId }]); + expect(getDeployment(job.deploymentId)).toMatchObject({ + backend: 'native', + canCancelCreation: true + }); + expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([ + { deployment_id: job.deploymentId } + ]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); - closeDb(); vi.resetModules(); + closeDb(); + vi.resetModules(); const backend = await import('../../src/lib/server/backends/native.js'); expect(backend.nativeDeployment(job.deploymentId)).toBe(true); await advance(); const original = await operationFor(job.deploymentId); - expect(original).toMatchObject({ kind: 'instance.create', payload: { name: job.name, port: 28960 } }); + expect(original).toMatchObject({ + kind: 'instance.create', + payload: { name: job.name, port: 28960 } + }); await advance(); expect((await operationFor(job.deploymentId))?.id).toBe(original!.id); const restartedDb = (await import('../../src/lib/server/db/index.js')).db(); - expect(restartedDb.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1); - expect(restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); + expect( + restartedDb + .prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'") + .all() + ).toHaveLength(1); + expect( + restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all() + ).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); }); it('still rejects missing keys and stale lobby identities before creating an intent', async () => { await prepareLibrary(); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', hostId, libraryId, image }; - expect(queueNativeCreation(input)).toMatchObject({ ok: false, message: expect.stringContaining('server key') }); - expect(queueNativeCreation({ ...input, plutoKey: 'saved-key', expectedName: 't5_zombie_theater_2' })) - .toMatchObject({ ok: false }); + const input = { + game: 't5' as const, + gameMode: 'zombie' as const, + mapSlug: 'theater', + hostId, + libraryId, + image + }; + expect(queueNativeCreation(input)).toMatchObject({ + ok: false, + message: expect.stringContaining('server key') + }); + expect( + queueNativeCreation({ + ...input, + plutoKey: 'saved-key', + expectedName: 't5_zombie_theater_2' + }) + ).toMatchObject({ ok: false }); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); @@ -413,17 +731,21 @@ describe('native instance provisioning', () => { it('accepts creation cleanup through the provisioning API and rejects an already completed creation', async () => { const job = await queue(); - const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js'); + const { POST } = + await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js'); const response = POST({ params: { id: job.deploymentId } }); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) }); - const next = await queue(); await advance(); await acknowledge(next.deploymentId); + const next = await queue(); + await advance(); + await acknowledge(next.deploymentId); expect(POST({ params: { id: next.deploymentId } }).status).toBe(409); expect(POST({ params: { id: job.deploymentId } }).status).toBe(200); }); it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => { const job = await queue(); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true); @@ -439,22 +761,26 @@ describe('native instance provisioning', () => { }); it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => { - const job = await queue(); await advance(); remote.players = 4; + const job = await queue(); + await advance(); + remote.players = 4; const original = (await operationFor(job.deploymentId))!; - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); - const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { heartbeat, pendingHostOperations, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] }); // A previously delivered command might still be executing when cancellation arrives. - expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id); + expect(pendingHostOperations(hostId).map((op) => op.id)).toContain(original.id); const cleanup = cancelCreation(job.deploymentId)!; const removal = (await operationFor(cleanup))!; expect(removal.kind).toBe('instance.cancel-create'); expect(removal.payload).toEqual({ createOperationId: original.id }); expect(cancelCreation(job.deploymentId)).toBe(cleanup); - expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id); - expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id); + expect(pendingHostOperations(hostId).map((op) => op.id)).toContain(removal.id); + expect(pendingHostOperations(hostId).map((op) => op.id)).not.toContain(original.id); await advance(); expect(nativeRequest).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); @@ -468,47 +794,79 @@ describe('native instance provisioning', () => { expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy(); expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined(); - expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) }); + expect( + db() + .prepare('SELECT retired_at FROM native_instances WHERE id=?') + .get(original.instanceId) + ).toMatchObject({ retired_at: expect.any(String) }); }); it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => { - const job = await queue(); await advance(); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); - const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js'); + const job = await queue(); + await advance(); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { heartbeat, pendingHostOperations } = + await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const cleanup = cancelCreation(job.deploymentId)!; - expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]); + expect( + pendingHostOperations(hostId).filter((op) => op.kind.startsWith('instance.')) + ).toEqual([]); await advance(); expect((await detail(cleanup)).stage).toContain('Update the host agent'); expect((await detail(cleanup)).stage).toContain('Automatic updates are not enabled'); - heartbeat(hostId, { ...inventory(), capabilities: ['agent.update'], - agentUpdate: { enabled: true, state: 'deferred', currentVersion: '1.2.2', targetVersion: '1.3.0' } }); + heartbeat(hostId, { + ...inventory(), + capabilities: ['agent.update'], + agentUpdate: { + enabled: true, + state: 'deferred', + currentVersion: '1.2.2', + targetVersion: '1.3.0' + } + }); await advance(); expect((await detail(cleanup)).stage).toContain('New command delivery is paused'); expect((await detail(cleanup)).stage).toContain('automatic updater can run'); - db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId); + db() + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run('2000-01-01T00:00:00.000Z', hostId); await advance(); expect((await detail(cleanup)).stage).toContain('unavailable'); expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] }); - expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1); + expect( + pendingHostOperations(hostId).filter((op) => op.kind === 'instance.cancel-create') + ).toHaveLength(1); await acknowledge(cleanup); expect((await detail(cleanup)).state).toBe('done'); }); it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const operation = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } }); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { healthy: true } + }); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = advance(); try { - const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const { destroyServer } = + await import('../../src/lib/server/provisioning/instances.js'); const waitingRemoval = destroyServer(job.name).deploymentId!; - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const cleanup = cancelCreation(job.deploymentId)!; expect(cleanup).toBeTruthy(); expect((await detail(waitingRemoval)).state).toBe('cancelled'); @@ -516,24 +874,35 @@ describe('native instance provisioning', () => { await pending; expect((await detail(job.deploymentId)).state).toBe('cancelled'); expect((await detail(cleanup)).state).toBe('running'); - } finally { release({ healthy: true, state: 'running' } as never); await pending; } + } finally { + release({ healthy: true, state: 'running' } as never); + await pending; + } }); it('does not offer creation cancellation after success or against a replacement instance', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { canCancelCreation, cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(canCancelCreation(job.deploymentId)).toBe(false); expect(cancelCreation(job.deploymentId)).toBeNull(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId); - db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); + db() + .prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?") + .run(job.deploymentId); + db() + .prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?') + .run(job.name); expect(cancelCreation(job.deploymentId)).toBeNull(); }); it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => { const job = await queue(); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js'); + const { isProvisioningDetail, provisioningGroups } = + await import('../../src/lib/ui/provisioning-presentation.js'); const queued = getDeployment(job.deploymentId)!; expect(queued.progress?.phase).toBe('eligibility'); expect(queued.progress?.blocker).toBeNull(); @@ -544,48 +913,73 @@ describe('native instance provisioning', () => { const result = getDeployment(job.deploymentId)!; expect(result.deployment.state).toBe('done'); expect(isProvisioningDetail(result, job.deploymentId)).toBe(true); - expect(result.progress?.checks.map(check => check.key)).toEqual(['container', 'game', 'map', 'plutonium-release', 'plutonium-version']); - expect(result.progress?.checks.every(check => check.result === 'pass')).toBe(true); + expect(result.progress?.checks.map((check) => check.key)).toEqual([ + 'container', + 'game', + 'map', + 'plutonium-release', + 'plutonium-version' + ]); + expect(result.progress?.checks.every((check) => check.result === 'pass')).toBe(true); expect(result.progress?.checks[0].source).toBe('agent'); - const steps = provisioningGroups(result).flatMap(group => group.steps); - expect(steps.every(step => step.state === 'done')).toBe(true); + const steps = provisioningGroups(result).flatMap((group) => group.steps); + expect(steps.every((step) => step.state === 'done')).toBe(true); expect(result.agentOperation?.state).toBe('completed'); }); it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const operation = (await operationFor(job.deploymentId))!; const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) { - reportHostOperation(hostId, operation.id, { state, result: { state: 'running', healthy: true } }); + reportHostOperation(hostId, operation.id, { + state, + result: { state: 'running', healthy: true } + }); } const result = getDeployment(job.deploymentId)!; expect(result.deployment.state).toBe('running'); expect(result.agentOperation?.state).toBe('completed'); - expect(result.events.filter(event => event.kind === 'agent')).toHaveLength(3); - expect(result.progress?.checkpoints.map(point => point.key)).toEqual(expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed'])); + expect(result.events.filter((event) => event.kind === 'agent')).toHaveLength(3); + expect(result.progress?.checkpoints.map((point) => point.key)).toEqual( + expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed']) + ); expect(result.progress?.command?.state).toBe('accepted'); }); it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => { - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); - const first = await queue(); await advance(); + const first = await queue(); + await advance(); const operation = (await operationFor(first.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } }); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { healthy: true } + }); const second = await queue(); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = advance(); try { const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container'); - await vi.waitFor(async () => { - expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); - }, { timeout: 5_000 }); + await vi.waitFor( + async () => { + expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); + }, + { timeout: 5_000 } + ); } finally { release({ healthy: true, state: 'running' } as never); await pending; @@ -593,23 +987,36 @@ describe('native instance provisioning', () => { }); it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => { - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); - const first = await queue(); await advance(); + const first = await queue(); + await advance(); const operation = (await operationFor(first.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } }); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { healthy: true } + }); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning({ wait: false }); try { const second = await queue(); await processProvisioning({ wait: false }); - await vi.waitFor(async () => { - expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); - }, { timeout: 5_000 }); + await vi.waitFor( + async () => { + expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); + }, + { timeout: 5_000 } + ); expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1); } finally { release({ healthy: true, state: 'running' } as never); @@ -618,18 +1025,30 @@ describe('native instance provisioning', () => { }); it('ignores a late health result after removal retires the operation', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const operation = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } }); + reportHostOperation(hostId, operation.id, { + state: 'completed', + result: { healthy: true } + }); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = advance(); try { await vi.waitFor(() => expect(release).toBeTypeOf('function')); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?") + db() + .prepare( + "UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?" + ) .run(new Date().toISOString(), job.deploymentId); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const before = getDeployment(job.deploymentId)!; @@ -639,21 +1058,33 @@ describe('native instance provisioning', () => { expect(after.deployment.state).toBe('cancelled'); expect(after.progress).toEqual(before.progress); expect(after.activity).toEqual(before.activity); - } finally { release({ healthy: true, state: 'running' } as never); await pending; } + } finally { + release({ healthy: true, state: 'running' } as never); + await pending; + } }); it('records map mismatch evidence while retaining passed container and game checks', async () => { - const job = await queue(); await advance(); remote.map = 'zombie_moon'; + const job = await queue(); + await advance(); + remote.map = 'zombie_moon'; await acknowledge(job.deploymentId); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const result = getDeployment(job.deploymentId)!; expect(result.deployment.state).toBe('awaiting-health'); expect(result.progress?.phase).toBe('verify-map'); - expect(result.progress?.checks).toEqual(expect.arrayContaining([ - expect.objectContaining({ key: 'container', result: 'pass' }), - expect.objectContaining({ key: 'game', result: 'pass' }), - expect.objectContaining({ key: 'map', result: 'waiting', actual: 'zombie_moon', expected: 'zombie_theater' }) - ])); + expect(result.progress?.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'container', result: 'pass' }), + expect.objectContaining({ key: 'game', result: 'pass' }), + expect.objectContaining({ + key: 'map', + result: 'waiting', + actual: 'zombie_moon', + expected: 'zombie_theater' + }) + ]) + ); }); it('selects the latest game files automatically and rejects stale explicit selections', async () => { @@ -663,97 +1094,188 @@ describe('native instance provisioning', () => { expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(old.id); const latest = await prepareLibrary('t5'); expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(latest.id); - expect(() => nativeSelection({ hostId, image, libraryId: old.id }, 't5')).toThrow('Refresh the launch preview'); + expect(() => nativeSelection({ hostId, image, libraryId: old.id }, 't5')).toThrow( + 'Refresh the launch preview' + ); }); it('rejects a launch preview made before newer game files were published without reserving a server', async () => { await prepareLibrary('t5'); - const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); + const { previewNativeCreation, queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); const preview = previewNativeCreation({ hostId, image, game: 't5', mapSlug: 'theater' }); await prepareLibrary('t5'); - const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', previewId: preview.id, plutoKey: 'new-key' }); - expect(result).toMatchObject({ ok: false, message: expect.stringContaining('Refresh the launch preview') }); + const result = queueNativeCreation({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + previewId: preview.id, + plutoKey: 'new-key' + }); + expect(result).toMatchObject({ + ok: false, + message: expect.stringContaining('Refresh the launch preview') + }); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 0 }); - expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 0 }); + expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ + count: 0 + }); + expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ + count: 0 + }); }); - it.each(['pinned', 'automatic'] as const)('keeps an unplaced %s launch pinned to its accepted files during cleanup', async placement => { - const job = await queue('t5', placement); - const acceptedId = libraryId; - const latest = await prepareLibrary('t5'); - expect(latest.id).not.toBe(acceptedId); - const { db } = await import('../../src/lib/server/db/index.js'); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - const { reconcileHostLibraryCleanup, cleanupControllerLibraries, getLibraryVersion, updateHostLibraryCaches } - = await import('../../src/lib/server/game-files/index.js'); - const accepted = getLibraryVersion(acceptedId)!; - heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] }); - expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); - reconcileHostLibraryCleanup(hostId); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?") - .all(acceptedId)).toEqual([]); + it.each(['pinned', 'automatic'] as const)( + 'keeps an unplaced %s launch pinned to its accepted files during cleanup', + async (placement) => { + const job = await queue('t5', placement); + const acceptedId = libraryId; + const latest = await prepareLibrary('t5'); + expect(latest.id).not.toBe(acceptedId); + const { db } = await import('../../src/lib/server/db/index.js'); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { + reconcileHostLibraryCleanup, + cleanupControllerLibraries, + getLibraryVersion, + updateHostLibraryCaches + } = await import('../../src/lib/server/game-files/index.js'); + const accepted = getLibraryVersion(acceptedId)!; + heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] }); + expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); + reconcileHostLibraryCleanup(hostId); + expect( + db() + .prepare( + "SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?" + ) + .all(acceptedId) + ).toEqual([]); - // The unplaced intent also protects the controller's copy when every - // host reports it absent; its accepted version remains downloadable. - updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]); - await cleanupControllerLibraries(); - expect(db().prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?').get(acceptedId)).toBeUndefined(); - const { stat } = await import('node:fs/promises'); - expect((await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))).isFile()).toBe(true); - updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'ready', manifestSha256: accepted.manifestSha256 }]); - await advance(); - expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - expect(nativeInstance(job.name)?.library_id).toBe(acceptedId); - }); + // The unplaced intent also protects the controller's copy when every + // host reports it absent; its accepted version remains downloadable. + updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]); + await cleanupControllerLibraries(); + expect( + db() + .prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?') + .get(acceptedId) + ).toBeUndefined(); + const { stat } = await import('node:fs/promises'); + expect( + ( + await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff')) + ).isFile() + ).toBe(true); + updateHostLibraryCaches(hostId, [ + { + libraryVersionId: acceptedId, + status: 'ready', + manifestSha256: accepted.manifestSha256 + } + ]); + await advance(); + expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.library_id).toBe(acceptedId); + } + ); - it.each(['t5', 't6'] as const)('queues and verifies a %s deployment through its host agent', async game => { - const job = await queue(game); await advance(); - const operation = await operationFor(job.deploymentId); - expect(job.port).toBeNull(); - expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: 28960 } }); - expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })])); - await acknowledge(job.deploymentId); - expect((await detail(job.deploymentId)).state).toBe('done'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(operation!.instanceId)).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); - }); + it.each(['t5', 't6'] as const)( + 'queues and verifies a %s deployment through its host agent', + async (game) => { + const job = await queue(game); + await advance(); + const operation = await operationFor(job.deploymentId); + expect(job.port).toBeNull(); + expect(operation).toMatchObject({ + kind: 'instance.create', + hostId, + generation: 1, + state: 'pending', + payload: { game, libraryId, image, port: 28960 } + }); + expect(operation?.payload.files).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' }) + ]) + ); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare( + 'SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol' + ) + .all(operation!.instanceId) + ).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); + } + ); it('reconciles the same operation after controller restart and an ambiguous remote result', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const original = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, original.id, { state: 'unknown', error: 'Reply lost after execution' }); - const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb(); - const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments(); - await advance(); await advance(); + reportHostOperation(hostId, original.id, { + state: 'unknown', + error: 'Reply lost after execution' + }); + const { closeDb, db } = await import('../../src/lib/server/db/index.js'); + closeDb(); + const { interruptDeployments } = + await import('../../src/lib/server/provisioning/history.js'); + interruptDeployments(); + await advance(); + await advance(); expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'").get()).toEqual({ count: 1 }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'" + ) + .get() + ).toEqual({ count: 1 }); expect((await detail(job.deploymentId)).state).toBe('running'); await acknowledge(job.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); }); it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const original = (await operationFor(job.deploymentId))!; const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', hostId); + db() + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run('2000-01-01T00:00:00Z', hostId); await advance(); - expect(db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)).toEqual({ container_state: 'unknown' }); + expect( + db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name) + ).toEqual({ container_state: 'unknown' }); expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + heartbeat(hostId, inventory()); await acknowledge(job.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); }); it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); - remote.players = 2; await advance(); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); + remote.players = 2; + await advance(); expect(await operationFor(rebuild.deploymentId)).toBeNull(); - remote.players = 0; await advance(); + remote.players = 0; + await advance(); const operation = (await operationFor(rebuild.deploymentId))!; - expect(operation).toMatchObject({ kind: 'instance.rebuild', generation: 1, payload: { nextGeneration: 2, libraryId } }); + expect(operation).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + payload: { nextGeneration: 2, libraryId } + }); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)?.generation).toBe(1); await acknowledge(rebuild.deploymentId); @@ -763,341 +1285,702 @@ describe('native instance provisioning', () => { expect(nativeInstance(job.name)?.generation).toBe(2); }); it('carries confirmed admin override through serialized rebuild preparation', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), capabilities: ['instance.safety-override'] }); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); - remote.players = 2; await advance(); - expect(await operationFor(rebuild.deploymentId)).toBeNull(); - const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js'); - expect(enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' })).toEqual({ ok: true }); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); + remote.players = 2; await advance(); - expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1, - payload: { nextGeneration: 2, libraryId, safetyOverride: true } }); + expect(await operationFor(rebuild.deploymentId)).toBeNull(); + const { enableSafetyOverride } = + await import('../../src/lib/server/provisioning/safety-override.js'); + expect( + enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' }) + ).toEqual({ ok: true }); + await advance(); + expect(await operationFor(rebuild.deploymentId)).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + payload: { nextGeneration: 2, libraryId, safetyOverride: true } + }); await acknowledge(rebuild.deploymentId); expect((await detail(rebuild.deploymentId)).state).toBe('done'); }); it('holds a rebuild when players join while its launch files are being prepared', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const prepare = preparation.prepareNativeLaunch; let started!: () => void; let release!: () => void; - const ready = new Promise(resolve => { started = resolve; }); - const gate = new Promise(resolve => { release = resolve; }); - const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => { - const result = await prepare(input); - started(); await gate; - return result; + const ready = new Promise((resolve) => { + started = resolve; }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const spy = vi + .spyOn(preparation, 'prepareNativeLaunch') + .mockImplementationOnce(async (input) => { + const result = await prepare(input); + started(); + await gate; + return result; + }); const pending = advance(); try { await ready; remote.players = 2; - release(); await pending; + release(); + await pending; expect(await operationFor(rebuild.deploymentId)).toBeNull(); remote.players = 0; await advance(); expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild'); - } finally { release(); await pending; spy.mockRestore(); } + } finally { + release(); + await pending; + spy.mockRestore(); + } }); it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); + const actual = await vi.importActual< + typeof import('../../src/lib/server/integrations/plutonium.js') + >('../../src/lib/server/integrations/plutonium.js'); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); // Version evidence must follow the submitted command, even on a fast test host. - db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); - db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); + db() + .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') + .run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); + db() + .prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?') + .run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); await acknowledge(rebuild.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); expect((await detail(rebuild.deploymentId)).state).toBe('done'); - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)) - .toEqual({ healthy_at: expect.any(String) }); - expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354); + expect( + db() + .prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?') + .get(rebuild.deploymentId) + ).toEqual({ healthy_at: expect.any(String) }); + expect( + actual.runningVersions().find((version) => version.server === job.name)?.revision + ).toBe(5354); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning'); - expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]); + expect( + fleetOperations().filter( + (operation) => operation.server === job.name && operation.state !== 'done' + ) + ).toEqual([]); }); it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); await advance(); const operation = (await operationFor(rebuild.deploymentId))!; const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); + const actual = await vi.importActual< + typeof import('../../src/lib/server/integrations/plutonium.js') + >('../../src/lib/server/integrations/plutonium.js'); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' }); const submittedAt = new Date(Date.now() - 1000).toISOString(); - db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId); - db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId); - vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' }); + db() + .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') + .run(submittedAt, rebuild.deploymentId); + db() + .prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?') + .run(submittedAt, rebuild.deploymentId); + vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ + revision: null, + error: 'Version response unavailable' + }); vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); await acknowledge(rebuild.deploymentId); - expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' }); - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null }); + expect(await detail(rebuild.deploymentId)).toMatchObject({ + state: 'awaiting-health', + stage: 'Waiting to verify the Plutonium runtime revision.' + }); + expect( + db() + .prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?') + .get(rebuild.deploymentId) + ).toEqual({ healthy_at: null }); closeDb(); vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); await advance(); expect((await detail(rebuild.deploymentId)).state).toBe('done'); expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'" + ) + .get() + ).toEqual({ count: 1 }); }); it('drains by reserving destination capacity, waiting for empty lobbies, and rebuilding once on another host', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { updateHost, createHost, createEnrollment, enrollHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js'); + const { + updateHost, + createHost, + createEnrollment, + enrollHost, + reportHostOperation, + getHost + } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' }); const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/); const { db } = await import('../../src/lib/server/db/index.js'); - const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + const { getLibraryVersion, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); await processProvisioning(); - const migration = db().prepare("SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?").get(job.name) as { id: string }; + const migration = db() + .prepare( + "SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?" + ) + .get(job.name) as { id: string }; expect((await detail(migration.id)).stage).toContain('No host has capacity'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toEqual([]); const target = createHost({ name: 'Destination', portMin: 29000, portMax: 29005 }); enrollHost(createEnrollment(target.id).token, inventory()); - updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]); - remote.players = 2; await processProvisioning(); + updateHostLibraryCaches(target.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 + } + ]); + remote.players = 2; + await processProvisioning(); expect((await detail(migration.id)).stage).toContain('2 player(s)'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]); - remote.players = 0; await processProvisioning(); - const remove = db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").get() as { id: string }; + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toEqual([]); + remote.players = 0; + await processProvisioning(); + const remove = db() + .prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'") + .get() as { id: string }; expect(remove).toBeTruthy(); - expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2); + expect( + db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id) + ).toHaveLength(2); expect(() => updateHost(target.id, { mode: 'draining' })).toThrow('incoming'); reportHostOperation(hostId, remove.id, { state: 'unknown' }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); await processProvisioning(); - expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId }); + expect( + db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name) + ).toEqual({ host_id: hostId }); reportHostOperation(hostId, remove.id, { state: 'completed' }); await processProvisioning(); - expect(db().prepare('SELECT host_id,generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: target.id, generation: 2 }); + expect( + db() + .prepare('SELECT host_id,generation FROM native_instances WHERE server=?') + .get(job.name) + ).toEqual({ host_id: target.id, generation: 2 }); expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]); expect(getHost(hostId)?.mode).toBe('draining'); - const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js'); + const { cancelProvisioning } = + await import('../../src/lib/server/provisioning/requests.js'); expect(cancelProvisioning(migration.id)).toBe(false); await processProvisioning(); const create = (await operationFor(migration.id))!; - expect(create).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2, payload: { name: job.name, port: 29000 } }); - reportHostOperation(target.id, create.id, { state: 'completed', result: { state: 'running', healthy: true } }); + expect(create).toMatchObject({ + hostId: target.id, + kind: 'instance.create', + generation: 2, + payload: { name: job.name, port: 29000 } + }); + reportHostOperation(target.id, create.id, { + state: 'completed', + result: { state: 'running', healthy: true } + }); await processProvisioning(); expect((await detail(migration.id)).state).toBe('done'); expect(getHost(hostId)?.mode).toBe('disabled'); - expect(db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ generation: 2 }); + expect( + db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name) + ).toEqual({ generation: 2 }); await advance(); expect((await operationFor(migration.id))?.id).toBe(create.id); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(1); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toHaveLength(1); expect(getLibraryVersion(libraryId)?.state).toBe('ready'); }); it('reserves migration capacity before source removal and retains it while a replacement awaits startup', async () => { - const first = await queue(); await advance(); await acknowledge(first.deploymentId); - const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); - const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' }); + const first = await queue(); + await advance(); + await acknowledge(first.deploymentId); + const { queueCreateServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const second = queueCreateServer({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + hostId, + libraryId, + image, + plutoKey: 'second-key' + }); if (!second.ok) throw new Error(second.message); - await advance(); await acknowledge(second.deploymentId); - const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + await advance(); + await acknowledge(second.deploymentId); + const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + const { getLibraryVersion, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); const target = createHost({ name: 'One free slot' }); - enrollHost(createEnrollment(target.id).token, { ...inventory(), memoryTotal: 4 * 1024 ** 3, memoryAvailable: 1.75 * 1024 ** 3 }); - updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]); + enrollHost(createEnrollment(target.id).token, { + ...inventory(), + memoryTotal: 4 * 1024 ** 3, + memoryAvailable: 1.75 * 1024 ** 3 + }); + updateHostLibraryCaches(target.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 + } + ]); updateHost(hostId, { mode: 'draining' }); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); - const removals = () => db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() as { id: string }[]; + const removals = () => + db() + .prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'") + .all() as { id: string }[]; expect(removals()).toHaveLength(1); - expect(db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2); + expect( + db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id) + ).toHaveLength(2); const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js'); - expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity'); + expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow( + 'memory capacity' + ); reportHostOperation(hostId, removals()[0].id, { state: 'completed' }); await advance(); - expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity'); + expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow( + 'memory capacity' + ); expect(removals()).toHaveLength(1); }); it('retries definite migration failures without replaying successful source removal', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js'); - const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { + createHost, + createEnrollment, + enrollHost, + updateHost, + reportHostOperation, + getHost + } = await import('../../src/lib/server/hosts/index.js'); + const { getLibraryVersion, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); const target = createHost({ name: 'Destination' }); enrollHost(createEnrollment(target.id).token, inventory()); - updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]); + updateHostLibraryCaches(target.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 + } + ]); updateHost(hostId, { mode: 'draining' }); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); - const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as { deployment_id: string; remove_operation_id: string }; - const expireRetry = (id: string) => db().prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(new Date(Date.now() - 61_000).toISOString(), id); - reportHostOperation(hostId, migration.remove_operation_id, { state: 'failed', error: 'Docker unavailable before removal' }); + const migration = db() + .prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations') + .get() as { deployment_id: string; remove_operation_id: string }; + const expireRetry = (id: string) => + db() + .prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?') + .run(new Date(Date.now() - 61_000).toISOString(), id); + reportHostOperation(hostId, migration.remove_operation_id, { + state: 'failed', + error: 'Docker unavailable before removal' + }); await advance(); - expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId }); + expect( + db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name) + ).toEqual({ host_id: hostId }); expireRetry(migration.remove_operation_id); - await advance(); await advance(); - const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as { remove_operation_id: string }; + await advance(); + await advance(); + const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as { + remove_operation_id: string; + }; expect(retry.remove_operation_id).not.toBe(migration.remove_operation_id); reportHostOperation(hostId, retry.remove_operation_id, { state: 'completed' }); - await advance(); await advance(); + await advance(); + await advance(); const creation = (await operationFor(migration.deployment_id))!; - reportHostOperation(target.id, creation.id, { state: 'failed', error: 'Docker unavailable before creation' }); + reportHostOperation(target.id, creation.id, { + state: 'failed', + error: 'Docker unavailable before creation' + }); expireRetry(creation.id); - await advance(); await advance(); + await advance(); + await advance(); const replacement = (await operationFor(migration.deployment_id))!; expect(replacement.id).not.toBe(creation.id); - expect(replacement).toMatchObject({ kind: 'instance.create', hostId: target.id, generation: 2 }); + expect(replacement).toMatchObject({ + kind: 'instance.create', + hostId: target.id, + generation: 2 + }); reportHostOperation(target.id, replacement.id, { state: 'completed' }); await advance(); expect(getHost(hostId)?.mode).toBe('disabled'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(2); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toHaveLength(2); }); it('disables an empty draining host and redirects its unplaced launches to another host', async () => { const job = await queue(); - const { createHost, createEnrollment, enrollHost, updateHost, getHost } = await import('../../src/lib/server/hosts/index.js'); - const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + const { createHost, createEnrollment, enrollHost, updateHost, getHost } = + await import('../../src/lib/server/hosts/index.js'); + const { getLibraryVersion, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); const target = createHost({ name: 'Destination' }); enrollHost(createEnrollment(target.id).token, inventory()); - updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]); + updateHostLibraryCaches(target.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 + } + ]); updateHost(hostId, { mode: 'draining' }); expect(() => updateHost(hostId, { mode: 'active' })).toThrow('finish draining'); await advance(); expect(getHost(hostId)?.mode).toBe('disabled'); - expect(await operationFor(job.deploymentId)).toMatchObject({ hostId: target.id, kind: 'instance.create' }); + expect(await operationFor(job.deploymentId)).toMatchObject({ + hostId: target.id, + kind: 'instance.create' + }); }); it('rejects the removed manual disable API action', async () => { const { PATCH } = await import('../../src/routes/api/hosts/[id]/+server.js'); - const response = await PATCH({ params: { id: hostId }, request: new Request(`https://ec.test/api/hosts/${hostId}`, { - method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: 'disabled' }) - }) }); + const response = await PATCH({ + params: { id: hostId }, + request: new Request(`https://ec.test/api/hosts/${hostId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode: 'disabled' }) + }) + }); expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ error: expect.stringContaining('disable automatically') }); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('disable automatically') + }); const { getHost } = await import('../../src/lib/server/hosts/index.js'); expect(getHost(hostId)?.mode).toBe('active'); }); it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { db } = await import('../../src/lib/server/db/index.js'); - const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes'); - db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name); - const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, putFileBlob(after, db())).lastInsertRowid); - const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); - queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId }); + const before = Buffer.from('original bytes on the host'); + const after = Buffer.from('new campaign bytes'); + db() + .prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)") + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')" + ) + .run(job.name); + const fileId = Number( + db() + .prepare( + "INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)" + ) + .run(job.name, putFileBlob(after, db())).lastInsertRowid + ); + const { queueFileChange, stagedFileChanges } = + await import('../../src/lib/server/fleet/staged-files.js'); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'campaign', + server: job.name, + kind: 'script', + relPath: 'hud.gsc', + content: after, + inventorySource: 'test', + deploymentFileId: fileId + }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' }); - remote.players = 2; await advance(); - const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + remote.players = 2; + await advance(); + const staged = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { stage_operation_id: string }; expect(staged.stage_operation_id).toBeTruthy(); - const { getHostOperation, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { getHostOperation, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); const stage = getHostOperation(staged.stage_operation_id)!; - expect(stage.kind).toBe('files.stage'); expect(await operationFor(restart.id)).toBeNull(); - const routing = db().prepare('SELECT payload FROM ec_host_operations WHERE id=?').get(stage.id) as { payload: string }; + expect(stage.kind).toBe('files.stage'); + expect(await operationFor(restart.id)).toBeNull(); + const routing = db() + .prepare('SELECT payload FROM ec_host_operations WHERE id=?') + .get(stage.id) as { payload: string }; expect(JSON.parse(routing.payload)).toEqual({ stageId: restart.id }); - const plan = db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { files_json: string }; + const plan = db() + .prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { files_json: string }; expect(JSON.parse(plan.files_json)[0]).not.toHaveProperty('contentBase64'); - const storedPayload = db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id) as { payload: string }; - expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({ contentSha256: createHash('sha256').update(after).digest('hex') }); + const storedPayload = db() + .prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?') + .get(stage.id) as { payload: string }; + expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({ + contentSha256: createHash('sha256').update(after).digest('hex') + }); expect(JSON.parse(storedPayload.payload).files[0]).not.toHaveProperty('contentBase64'); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); const { pendingHostOperations } = await import('../../src/lib/server/hosts/index.js'); - expect(pendingHostOperations(hostId).find(operation => operation.id === stage.id)?.payload).toEqual(stage.payload); + expect( + pendingHostOperations(hostId).find((operation) => operation.id === stage.id)?.payload + ).toEqual(stage.payload); await advance(); - expect((db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }).stage_operation_id).toBe(stage.id); + expect( + ( + db() + .prepare( + 'SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?' + ) + .get(restart.id) as { stage_operation_id: string } + ).stage_operation_id + ).toBe(stage.id); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending'); reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } }); - await advance(); expect(await operationFor(restart.id)).toBeNull(); - remote.players = 0; await advance(); + await advance(); + expect(await operationFor(restart.id)).toBeNull(); + remote.players = 0; + await advance(); const activation = (await operationFor(restart.id))!; - expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } }); - expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 }); + expect(activation).toMatchObject({ + kind: 'instance.restart', + generation: 1, + payload: { stageId: restart.id } + }); + expect( + db() + .prepare( + 'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?' + ) + .get(fileId) + ).toEqual({ content_before: null, snapshot_pending: 1 }); const files = stage.payload.files as Array<{ id: number; path: string }>; - reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } }); + reportHostOperation(hostId, activation.id, { + state: 'completed', + result: { + before: [ + { + id: files[0].id, + path: files[0].path, + contentBase64: before.toString('base64') + } + ] + } + }); await advance(); expect((await detail(restart.id)).state).toBe('done'); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied'); - expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 }); + expect( + db() + .prepare( + 'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?' + ) + .get(fileId) + ).toEqual({ + content_before: before, + sha256_before: createHash('sha256').update(before).digest('hex'), + snapshot_pending: 0 + }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex')); + expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe( + createHash('sha256').update(after).digest('hex') + ); }); it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => { - const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6'); + const t5 = await prepareLibrary('t5'); + const t6 = await prepareLibrary('t6'); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] }); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image }); + const blueprint = saveBlueprint({ + name: 'Both games', + entries: [ + { game: 't5', mapSlug: 'theater', count: 2 }, + { game: 't6', mapSlug: 'town', count: 1 } + ] + }); + const { previewCollection, queueCollection } = + await import('../../src/lib/server/fleet/collections.js'); + const preview = previewCollection(blueprint.id, 'fill', { + hostId, + libraryIds: { t5: t5.id, t6: t6.id }, + image + }); expect(preview.ready).toBe(true); - expect(preview.servers.map(server => server.port)).toEqual([null, null, null]); - const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }); + expect(preview.servers.map((server) => server.port)).toEqual([null, null, null]); + const queued = queueCollection({ + previewId: preview.id, + keys: Object.fromEntries( + preview.servers.map((server) => [server.name!, 'key-' + server.name]) + ) + }); expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 }); await advance(); - const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all(); - expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]); - const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js'); - const secondHost = createHost({ name: 'Second Debian' }); enrollHost(createEnrollment(secondHost.id).token, inventory()); - const { updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); - updateHostLibraryCaches(secondHost.id, [{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }]); - const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); - const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' }); - expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message); + const instances = db() + .prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port') + .all(); + expect(instances).toEqual([ + { host_id: hostId, library_id: t5.id, port: 28960 }, + { host_id: hostId, library_id: t5.id, port: 28961 }, + { host_id: hostId, library_id: t6.id, port: 28962 } + ]); + const { createHost, createEnrollment, enrollHost, hostPortAvailable } = + await import('../../src/lib/server/hosts/index.js'); + const secondHost = createHost({ name: 'Second Debian' }); + enrollHost(createEnrollment(secondHost.id).token, inventory()); + const { updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); + updateHostLibraryCaches(secondHost.id, [ + { libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 } + ]); + const { queueCreateServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const second = queueCreateServer({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'moon', + hostId: secondHost.id, + libraryId: t5.id, + image, + plutoKey: 'second-host-key' + }); + expect(second.ok).toBe(true); + if (!second.ok) throw new Error(second.message); expect(second.port).toBeNull(); await advance(); expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960); expect(hostPortAvailable(hostId, 28960)).toBe(false); expect(hostPortAvailable(secondHost.id, 28960)).toBe(false); - expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 }); + expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ + count: 8 + }); }); it('reads remote instance files through the agent including missing files and epoch timestamps', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const backend = await import('../../src/lib/server/backends/native.js'); const remoteRead = vi.mocked(backend.nativeRequest); remoteRead.mockImplementation(async (_instance, kind, payload) => { - if (payload?.path === 'missing') return (kind === 'files.read' ? { contentBase64: null } : null) as never; - if (kind === 'files.stat') return { size: 15, mtime: 1_790_000_000, isFile: true, isDirectory: false } as never; - if (kind === 'files.read') return { contentBase64: Buffer.from('agent file bytes').toString('base64') } as never; - if (kind === 'files.list') return { entries: [{ name: 'configs', isDirectory: true, isFile: false }] } as never; + if (payload?.path === 'missing') + return (kind === 'files.read' ? { contentBase64: null } : null) as never; + if (kind === 'files.stat') + return { + size: 15, + mtime: 1_790_000_000, + isFile: true, + isDirectory: false + } as never; + if (kind === 'files.read') + return { + contentBase64: Buffer.from('agent file bytes').toString('base64') + } as never; + if (kind === 'files.list') + return { + entries: [{ name: 'configs', isDirectory: true, isFile: false }] + } as never; throw new Error('Unexpected file operation'); }); const files = await import('../../src/lib/server/backends/files.js'); const root = join(fixture.t5, job.name); - expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe('agent file bytes'); + expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe( + 'agent file bytes' + ); const info = await files.stat(join(root, 'configs/server.cfg')); - expect(info.mtime.getTime()).toBe(1_790_000_000_000); expect(info.isFile()).toBe(true); + expect(info.mtime.getTime()).toBe(1_790_000_000_000); + expect(info.isFile()).toBe(true); expect(await files.exists(join(root, 'missing'))).toBe(false); - await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ + code: 'ENOENT' + }); const entries = await files.readdir(root, { withFileTypes: true }); - expect(entries[0].name).toBe('configs'); expect(entries[0].isDirectory()).toBe(true); - expect(remoteRead).toHaveBeenCalledWith(expect.objectContaining({ server: job.name }), 'files.read', { path: 'configs/server.cfg' }); + expect(entries[0].name).toBe('configs'); + expect(entries[0].isDirectory()).toBe(true); + expect(remoteRead).toHaveBeenCalledWith( + expect.objectContaining({ server: job.name }), + 'files.read', + { path: 'configs/server.cfg' } + ); const { stat } = await import('node:fs/promises'); await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }); }); it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); - const removal = destroyServer(job.name); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockClear(); - remote.state = 'missing'; await advance(); + const removal = destroyServer(job.name); + if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockClear(); + remote.state = 'missing'; + await advance(); expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove'); expect(rconStatusResult).not.toHaveBeenCalled(); await acknowledge(removal.deploymentId); @@ -1106,28 +1989,72 @@ describe('native instance provisioning', () => { expect((await detail(removal.deploymentId)).state).toBe('done'); }); it('releases a failed activation slot without recording fictitious file changes', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')").run(job.name); - const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')").run(job.name).lastInsertRowid); - const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); - queueFileChange({ sourceKind: 'campaign', sourceId: 'preflight-failed', server: job.name, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('new bytes'), inventorySource: 'test', deploymentFileId: fileId }); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)" + ) + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')" + ) + .run(job.name); + const fileId = Number( + db() + .prepare( + "INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')" + ) + .run(job.name).lastInsertRowid + ); + const { queueFileChange, stagedFileChanges } = + await import('../../src/lib/server/fleet/staged-files.js'); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'preflight-failed', + server: job.name, + kind: 'script', + relPath: 'hud.gsc', + content: Buffer.from('new bytes'), + inventorySource: 'test', + deploymentFileId: fileId + }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' }); await advance(); - const stage = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + const stage = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { stage_operation_id: string }; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, stage.stage_operation_id, { state: 'completed', result: { state: 'staged' } }); + reportHostOperation(hostId, stage.stage_operation_id, { + state: 'completed', + result: { state: 'staged' } + }); await advance(); const activation = (await operationFor(restart.id))!; expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying'); - reportHostOperation(hostId, activation.id, { state: 'failed', error: 'Target changed after campaign review; no activation occurred' }); + reportHostOperation(hostId, activation.id, { + state: 'failed', + error: 'Target changed after campaign review; no activation occurred' + }); await advance(); expect((await detail(restart.id)).state).toBe('failed'); expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending'); - expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 }); - expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null }); + expect( + db() + .prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL') + .get() + ).toEqual({ count: 0 }); + expect( + db() + .prepare( + 'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?' + ) + .get(fileId) + ).toEqual({ content_before: null, sha256_before: null }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined(); }); @@ -1136,71 +2063,111 @@ describe('native instance provisioning', () => { const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)).toBeNull(); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); - const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true); + const cancelled = destroyServer(job.name); + expect(cancelled.ok).toBe(true); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); expect((await detail(job.deploymentId)).state).toBe('cancelled'); expect(nativeInstance(job.name)).toBeNull(); expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'" + ) + .get() + ).toEqual({ count: 0 }); const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js'); expect(getLibraryVersion(libraryId)?.state).toBe('ready'); - const { getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js'); + const { getServerKey, serverKeyPool } = + await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(job.name)).toBeUndefined(); expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 }); }); }); describe('automatic capacity admission and placement', () => { - const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'key' }; + const input = { + game: 't5' as const, + gameMode: 'zombie' as const, + mapSlug: 'theater', + plutoKey: 'key' + }; async function secondHost(overrides = {}) { const api = await import('../../src/lib/server/hosts/index.js'); const libs = await import('../../src/lib/server/game-files/index.js'); - const host = api.createHost({ name: 'Second host', publicAddress: 'second.example', portMin: 28960, portMax: 28970 }); + const host = api.createHost({ + name: 'Second host', + publicAddress: 'second.example', + portMin: 28960, + portMax: 28970 + }); api.updateHost(host.id, { publicAddress: 'second.example' }); api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides }); const version = libs.getLibraryVersion(libraryId)!; - libs.updateHostLibraryCaches(host.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }]); + libs.updateHostLibraryCaches(host.id, [ + { libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 } + ]); return host; } it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => { await prepareLibrary(); const other = await secondHost({ cpuUsagePercent: 0 }); - const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); + const { queueCreateServer } = + await import('../../src/lib/server/provisioning/instances.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { recipeForDeployment } = + await import('../../src/lib/server/fleet/launch-recipes.js'); const result = queueCreateServer(input); - expect(result.ok).toBe(true); if (!result.ok) return; + expect(result.ok).toBe(true); + if (!result.ok) return; expect(nativeInstance(result.name)).toBeNull(); expect(result.port).toBeNull(); await advance(); expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id); - expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' }); + expect(nativeInstance(result.name)).toMatchObject({ + host_id: other.id, + public_address: 'second.example' + }); expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id); // Once submitted, even a now-offline host cannot trigger another placement. const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id); await advance(); expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all() + ).toHaveLength(1); }); it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => { - await prepareLibrary(); await secondHost(); + await prepareLibrary(); + await secondHost(); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const result = queueNativeCreation({ ...input, publicAddress: 'nat.example', publicPort: 30000 }); - expect(result.ok).toBe(true); if (!result.ok) return; + const result = queueNativeCreation({ + ...input, + publicAddress: 'nat.example', + publicPort: 30000 + }); + expect(result.ok).toBe(true); + if (!result.ok) return; db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId); await advance(); - expect(nativeInstance(result.name)).toMatchObject({ public_address: 'nat.example', public_port: 30000 }); + expect(nativeInstance(result.name)).toMatchObject({ + public_address: 'nat.example', + public_port: 30000 + }); expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId); expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]); }); it.each([ - { cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 }, - { cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 } - ])('accepts an unplaced job and waits for unavailable resources: %j', async overrides => { + { cpuUsagePercent: undefined }, + { memoryAvailable: undefined }, + { cpuCount: 0 }, + { cpuUsagePercent: 99 }, + { memoryAvailable: 1024 ** 3 } + ])('accepts an unplaced job and waits for unavailable resources: %j', async (overrides) => { await prepareLibrary(); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), ...overrides }); @@ -1220,7 +2187,8 @@ describe('automatic capacity admission and placement', () => { const other = await secondHost(); const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(other.id, { portMin: 30000, portMax: 30002 }); - const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); + const { previewNativeCreation, queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); const preview = previewNativeCreation(input); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId); @@ -1234,12 +2202,21 @@ describe('automatic capacity admission and placement', () => { it('queues the whole collection and waits when capacity shrinks after its preview', async () => { await prepareLibrary(); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] }); + const { previewCollection, queueCollection } = + await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ + name: 'Two lobbies', + entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] + }); const preview = previewCollection(blueprint.id); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 }); - const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }); + const result = queueCollection({ + previewId: preview.id, + keys: Object.fromEntries( + preview.servers.map((server) => [server.name!, 'key-' + server.name]) + ) + }); expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) }); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); @@ -1249,8 +2226,16 @@ describe('automatic capacity admission and placement', () => { expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2); }); it.each([ - { resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } }, - { resource: 'memory', measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, over: { memoryAvailable: 1024 ** 3 + 499_999_999 } } + { + resource: 'CPU', + measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, + over: { cpuUsagePercent: 82.51 } + }, + { + resource: 'memory', + measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, + over: { memoryAvailable: 1024 ** 3 + 499_999_999 } + } ])('reserves only the requested host $resource headroom', async ({ measurements, over }) => { await prepareLibrary(); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); @@ -1266,22 +2251,38 @@ describe('automatic capacity admission and placement', () => { }); it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => { const first = await queue(); - await advance(); await acknowledge(first.deploymentId); + await advance(); + await acknowledge(first.deploymentId); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); + const { recordInstanceResources, capacityCalibration } = + await import('../../src/lib/server/hosts/calibration.js'); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 }; heartbeat(hostId, measured); const next = queueNativeCreation({ ...input, plutoKey: 'next-key' }); - expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message); + expect(next.ok).toBe(true); + if (!next.ok) throw new Error(next.message); await advance(); expect(await operationFor(next.deploymentId)).toBeNull(); const instance = nativeInstance(first.name)!; for (let minute = 0; minute < 25; minute++) { - recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1, - uptimeSeconds: minute < 4 ? minute * 60 : 600, cpuCores: 0.1, memoryBytes: 256 * 1024 ** 2 }] }, - new Date(Date.now() - (25 - minute) * 60_000).toISOString()); + recordInstanceResources( + hostId, + { + ...measured, + instanceResources: [ + { + instanceId: instance.id, + generation: 1, + uptimeSeconds: minute < 4 ? minute * 60 : 600, + cpuCores: 0.1, + memoryBytes: 256 * 1024 ** 2 + } + ] + }, + new Date(Date.now() - (25 - minute) * 60_000).toISOString() + ); } expect(capacityCalibration(hostId).startup.source).toBe('host'); await advance(); @@ -1291,11 +2292,19 @@ describe('automatic capacity admission and placement', () => { await prepareLibrary(); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const { previewCollection, queueCollection } = + await import('../../src/lib/server/fleet/collections.js'); const GiB = 1024 ** 3; - const measured = { ...inventory(), cpuCount: 4, cpuUsagePercent: 5, memoryTotal: 4 * GiB, memoryAvailable: 3.75 * GiB }; + const measured = { + ...inventory(), + cpuCount: 4, + cpuUsagePercent: 5, + memoryTotal: 4 * GiB, + memoryAvailable: 3.75 * GiB + }; heartbeat(hostId, measured); saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); const insert = db().prepare(`INSERT INTO ec_resource_allowances @@ -1304,17 +2313,42 @@ describe('automatic capacity admission and placement', () => { const start = new Date(Date.now() - 30 * 60_000).toISOString(); insert.run(hostId, 'startup', 2, 2 * GiB, start, now, now); insert.run(hostId, 'running', 0.2, GiB / 4, start, now, now); - const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] }); + const blueprint = saveBlueprint({ + name: 'Staggered lobbies', + entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] + }); const preview = previewCollection(blueprint.id); expect(preview.ready).toBe(true); - expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true); + expect( + queueCollection({ + previewId: preview.id, + keys: Object.fromEntries( + preview.servers.map((server) => [server.name!, 'key-' + server.name]) + ) + }).ok + ).toBe(true); await advance(); - const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[]; + const creates = () => + db() + .prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'") + .all() as { instance_id: string }[]; expect(creates()).toHaveLength(1); await advance(); expect(creates()).toHaveLength(1); // No heartbeat yet: the first launch still reserves startup resources. - heartbeat(hostId, { ...measured, cpuUsagePercent: 10, memoryAvailable: 3.5 * GiB, - instanceResources: [{ instanceId: creates()[0].instance_id, generation: 1, uptimeSeconds: 60, cpuCores: 0.2, memoryBytes: GiB / 4 }] }); + heartbeat(hostId, { + ...measured, + cpuUsagePercent: 10, + memoryAvailable: 3.5 * GiB, + instanceResources: [ + { + instanceId: creates()[0].instance_id, + generation: 1, + uptimeSeconds: 60, + cpuCores: 0.2, + memoryBytes: GiB / 4 + } + ] + }); await advance(); expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification. }); @@ -1323,115 +2357,211 @@ describe('automatic capacity admission and placement', () => { const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 }); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const first = queueNativeCreation(input); - expect(first.ok).toBe(true); if (!first.ok) return; + expect(first.ok).toBe(true); + if (!first.ok) return; const second = queueNativeCreation({ ...input, plutoKey: 'second-key' }); expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null }); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); cancelCreation(first.deploymentId); - expect(queueNativeCreation(input)).toMatchObject({ ok: true, name: first.name, port: null }); - }); - it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => { - await prepareLibrary(); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const GiB = 1024 ** 3; - // A recently busy 4 GiB host is now mostly empty, as in the reported - // ec-2 inventory. Admission must not pass only to stall in dispatch. - heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) }); - for (let minute = 1; minute <= 4; minute++) db().prepare(`INSERT INTO ec_host_resource_samples - (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`) - .run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10, - 4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB)); - const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` })); - expect(jobs.every(job => job.ok)).toBe(true); - await advance(); - expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull(); + expect(queueNativeCreation(input)).toMatchObject({ + ok: true, + name: first.name, + port: null + }); }); + it.each(['memory', 'cpu'] as const)( + 'dispatches immediately after %s recovers instead of waiting for five-minute history', + async (resource) => { + await prepareLibrary(); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const GiB = 1024 ** 3; + // A recently busy 4 GiB host is now mostly empty, as in the reported + // ec-2 inventory. Admission must not pass only to stall in dispatch. + heartbeat(hostId, { + ...inventory(), + cpuCount: 4, + memoryTotal: 4 * GiB, + memoryAvailable: Math.floor(3.8 * GiB) + }); + for (let minute = 1; minute <= 4; minute++) + db() + .prepare( + `INSERT INTO ec_host_resource_samples + (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)` + ) + .run( + hostId, + new Date(Date.now() - minute * 60_000).toISOString(), + resource === 'cpu' ? 100 : 10, + 4 * GiB, + resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB) + ); + const jobs = Array.from({ length: 3 }, (_, index) => + queueNativeCreation({ ...input, plutoKey: `key-${index}` }) + ); + expect(jobs.every((job) => job.ok)).toBe(true); + await advance(); + expect( + await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing') + ).not.toBeNull(); + } + ); it('dispatches one queued launch when learned startup costs outgrow the original batch reservation', async () => { await prepareLibrary(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const { db } = await import('../../src/lib/server/db/index.js'); const GiB = 1024 ** 3; - heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) }); - const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` })); - expect(jobs.every(job => job.ok)).toBe(true); + heartbeat(hostId, { + ...inventory(), + cpuCount: 4, + memoryTotal: 4 * GiB, + memoryAvailable: Math.floor(3.8 * GiB) + }); + const jobs = Array.from({ length: 3 }, (_, index) => + queueNativeCreation({ ...input, plutoKey: `key-${index}` }) + ); + expect(jobs.every((job) => job.ok)).toBe(true); // Learning increases the startup allowance after admission. All three // no longer fit concurrently, but this otherwise empty host fits one. - db().prepare(`INSERT INTO ec_resource_allowances + db() + .prepare( + `INSERT INTO ec_resource_allowances (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) - VALUES(?,'startup',0.5,?,3,?,?,?)`).run(hostId, 2 * GiB, - new Date(Date.now() - 180_000).toISOString(), new Date().toISOString(), new Date().toISOString()); + VALUES(?,'startup',0.5,?,3,?,?,?)` + ) + .run( + hostId, + 2 * GiB, + new Date(Date.now() - 180_000).toISOString(), + new Date().toISOString(), + new Date().toISOString() + ); await advance(); - const operations = await Promise.all(jobs.map(job => operationFor(job.ok ? job.deploymentId : 'missing'))); + const operations = await Promise.all( + jobs.map((job) => operationFor(job.ok ? job.deploymentId : 'missing')) + ); expect(operations.filter(Boolean)).toHaveLength(1); expect(operations[0]?.kind).toBe('instance.create'); // The dispatched operation now consumes capacity and blocks the others, // even though the next host heartbeat has not observed it yet. await advance(); - expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1); + expect( + db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all() + ).toHaveLength(1); const later = queueNativeCreation(input); - expect(later.ok).toBe(true); if (!later.ok) throw new Error(later.message); + expect(later.ok).toBe(true); + if (!later.ok) throw new Error(later.message); await advance(); expect(await operationFor(later.deploymentId)).toBeNull(); }); it('does not charge a starting container twice when its usage is already in the heartbeat', async () => { await prepareLibrary(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const GiB = 1024 ** 3; - const host = { ...inventory(), cpuCount: 4, memoryTotal: 3 * GiB, memoryAvailable: 2.8 * GiB }; + const host = { + ...inventory(), + cpuCount: 4, + memoryTotal: 3 * GiB, + memoryAvailable: 2.8 * GiB + }; heartbeat(hostId, { ...host, memoryAvailable: Math.floor(host.memoryAvailable) }); const first = queueNativeCreation(input); - expect(first.ok).toBe(true); if (!first.ok) return; + expect(first.ok).toBe(true); + if (!first.ok) return; await advance(); const instance = nativeInstance(first.name)!; // 0.2 GiB OS + 0.8 GiB container are already in host usage. // Adding a new 1 GiB server fits below 2.53 GiB usable memory. - heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id, - generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] }); + heartbeat(hostId, { + ...host, + memoryAvailable: 2 * GiB, + instanceResources: [ + { + instanceId: instance.id, + generation: instance.generation, + uptimeSeconds: 60, + cpuCores: 0.1, + memoryBytes: Math.floor(0.8 * GiB) + } + ] + }); const second = queueNativeCreation({ ...input, plutoKey: 'second-key' }); expect(second).toMatchObject({ ok: true }); await advance(); expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull(); }); - it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => { - await prepareLibrary(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const GiB = 1024 ** 3; - heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: Math.floor(2.8 * GiB) }); - const first = queueNativeCreation(input); - expect(first.ok).toBe(true); if (!first.ok) return; - await advance(); - const instance = nativeInstance(first.name)!; - heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB, - instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1, - uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] }); - const next = queueNativeCreation({ ...input, plutoKey: 'next-key' }); - expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message); - await advance(); - expect(await operationFor(next.deploymentId)).toBeNull(); - expect((await detail(next.deploymentId)).stage).toContain('3.00 GiB projected, 2.53 GiB usable'); - }); + it.each(['missing', 'old-generation'] as const)( + 'keeps full reservations for a submitted launch with %s measurements', + async (evidence) => { + await prepareLibrary(); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); + saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + const GiB = 1024 ** 3; + heartbeat(hostId, { + ...inventory(), + memoryTotal: 3 * GiB, + memoryAvailable: Math.floor(2.8 * GiB) + }); + const first = queueNativeCreation(input); + expect(first.ok).toBe(true); + if (!first.ok) return; + await advance(); + const instance = nativeInstance(first.name)!; + heartbeat(hostId, { + ...inventory(), + memoryTotal: 3 * GiB, + memoryAvailable: 2 * GiB, + instanceResources: + evidence === 'missing' + ? [] + : [ + { + instanceId: instance.id, + generation: instance.generation + 1, + uptimeSeconds: 60, + cpuCores: 0.1, + memoryBytes: Math.floor(0.8 * GiB) + } + ] + }); + const next = queueNativeCreation({ ...input, plutoKey: 'next-key' }); + expect(next.ok).toBe(true); + if (!next.ok) throw new Error(next.message); + await advance(); + expect(await operationFor(next.deploymentId)).toBeNull(); + expect((await detail(next.deploymentId)).stage).toContain( + '3.00 GiB projected, 2.53 GiB usable' + ); + } + ); it('waits when capacity disappears after admission without sending a create command', async () => { await prepareLibrary(); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const first = queueNativeCreation(input); - expect(first.ok).toBe(true); if (!first.ok) return; + expect(first.ok).toBe(true); + if (!first.ok) return; const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 }); await advance(); @@ -1442,11 +2572,21 @@ describe('automatic capacity admission and placement', () => { await prepareLibrary(); const other = await secondHost({ cpuUsagePercent: 30 }); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) - VALUES(?,?,16,95,?,?)`).run(hostId, new Date(Date.now() - 60_000).toISOString(), 64 * 1024 ** 3, 60 * 1024 ** 3); + db() + .prepare( + `INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) + VALUES(?,?,16,95,?,?)` + ) + .run( + hostId, + new Date(Date.now() - 60_000).toISOString(), + 64 * 1024 ** 3, + 60 * 1024 ** 3 + ); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const first = queueNativeCreation(input); - expect(first.ok).toBe(true); if (!first.ok) return; + expect(first.ok).toBe(true); + if (!first.ok) return; await advance(); expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id); }); @@ -1454,221 +2594,427 @@ describe('automatic capacity admission and placement', () => { await prepareLibrary(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - db().prepare("INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')").run(hostId); + db() + .prepare( + "INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')" + ) + .run(hostId); heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined }); closeDb(); - const rows = db().prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at').all(hostId) as { cpu_usage_percent: number | null; memory_available: number; sampled_at: string }[]; - expect(rows.at(-1)).toMatchObject({ cpu_usage_percent: null, memory_available: 60 * 1024 ** 3 }); - expect(rows.some(row => row.sampled_at === '2000-01-01')).toBe(false); + const rows = db() + .prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at') + .all(hostId) as { + cpu_usage_percent: number | null; + memory_available: number; + sampled_at: string; + }[]; + expect(rows.at(-1)).toMatchObject({ + cpu_usage_percent: null, + memory_available: 60 * 1024 ** 3 + }); + expect(rows.some((row) => row.sampled_at === '2000-01-01')).toBe(false); }); it('checks both port protocols and skips draining hosts', async () => { await prepareLibrary(); const other = await secondHost(); const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js'); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - for (let port = 28960; port <= 28970; port++) reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' }); + for (let port = 28960; port <= 28970; port++) + reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' }); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const result = queueNativeCreation(input); - expect(result.ok).toBe(true); if (!result.ok) return; + expect(result.ok).toBe(true); + if (!result.ok) return; expect(nativeInstance(result.name)).toBeNull(); await advance(); expect(nativeInstance(result.name)?.host_id).toBe(other.id); updateHost(other.id, { mode: 'draining' }); const next = queueNativeCreation({ ...input, plutoKey: 'next-key' }); - expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message); + expect(next.ok).toBe(true); + if (!next.ok) throw new Error(next.message); await advance(); expect(await operationFor(next.deploymentId)).toBeNull(); }); it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => { - await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 }); + await prepareLibrary(); + await secondHost({ cpuCount: 1, cpuUsagePercent: 0 }); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 }); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] }); + const { previewCollection, queueCollection } = + await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ + name: 'Two lobbies', + entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] + }); const preview = previewCollection(blueprint.id); expect(preview.ready).toBe(true); - expect(preview.servers.map(server => server.port)).toEqual([null, null]); - expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true); + expect(preview.servers.map((server) => server.port)).toEqual([null, null]); + expect( + queueCollection({ + previewId: preview.id, + keys: Object.fromEntries( + preview.servers.map((server) => [server.name!, 'key-' + server.name]) + ) + }).ok + ).toBe(true); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); await advance(); // Concurrent ranking can select the same host before the first // reservation commits. The held job retries against the new usage. await advance(); expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2); - expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]); + expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([ + { port: 28960 }, + { port: 28960 } + ]); }); }); describe('native verification resume and explicit failed-creation retry', () => { async function failedCreation(state: 'failed' | 'interrupted' = 'failed') { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const original = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' }); + reportHostOperation(hostId, original.id, { + state: 'failed', + error: 'Host runtime preflight failed' + }); await advance(); if (state === 'interrupted') { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId); + db() + .prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?") + .run(job.deploymentId); } return { job, original }; } - it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => { - const { job, original } = await failedCreation(state); - const history = await import('../../src/lib/server/provisioning/history.js'); - expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true }); - const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); - expect(resume({ params: { id: job.deploymentId } }).status).toBe(409); - const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js'); - const response = await retry({ params: { id: job.deploymentId } }); - expect(response.status).toBe(202); - const next = await response.json() as { ok: boolean; deploymentId: string }; - expect(next.ok).toBe(true); - expect(next.deploymentId).not.toBe(job.deploymentId); - expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name }); - expect(history.getDeployment(job.deploymentId)).toMatchObject({ - canRetryWithRebuild: false, - deployment: { state, error: 'Host runtime preflight failed' }, - progress: { recovery: { state: 'transferred', operationId: next.deploymentId } } - }); - expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409); + it.each(['failed', 'interrupted'] as const)( + 'retries a %s creation through a new rebuild while retaining the original failure', + async (state) => { + const { job, original } = await failedCreation(state); + const history = await import('../../src/lib/server/provisioning/history.js'); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ + canResumeVerification: false, + canRetryWithRebuild: true + }); + const { POST: resume } = + await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); + expect(resume({ params: { id: job.deploymentId } }).status).toBe(409); + const { POST: retry } = + await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js'); + const response = await retry({ params: { id: job.deploymentId } }); + expect(response.status).toBe(202); + const next = (await response.json()) as { ok: boolean; deploymentId: string }; + expect(next.ok).toBe(true); + expect(next.deploymentId).not.toBe(job.deploymentId); + expect(await detail(next.deploymentId)).toMatchObject({ + action: 'rebuild', + state: 'waiting', + server: job.name + }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ + canRetryWithRebuild: false, + deployment: { state, error: 'Host runtime preflight failed' }, + progress: { recovery: { state: 'transferred', operationId: next.deploymentId } } + }); + expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?') + .get(job.name) + ).toEqual({ count: 2 }); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)).toMatchObject({ + id: original.instanceId, + generation: 1, + port: original.payload.port + }); + remote.state = 'missing'; + await advance(); + const rebuilding = (await operationFor(next.deploymentId))!; + expect(rebuilding).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + instanceId: original.instanceId, + payload: { + name: job.name, + port: original.payload.port, + nextGeneration: 2, + env: { PLUTO_SERVER_KEY: 'saved-key' } + } + }); + expect(rebuilding.id).not.toBe(original.id); + expect(nativeInstance(job.name)?.generation).toBe(1); + remote.state = 'running'; + await acknowledge(next.deploymentId); + expect((await detail(next.deploymentId)).state).toBe('done'); + expect(nativeInstance(job.name)?.generation).toBe(2); + // An interrupted original is reconciled to its already-failed receipt + // on the next worker pass; the failure itself is never cleared. + expect(await detail(job.deploymentId)).toMatchObject({ + state: 'failed', + error: 'Host runtime preflight failed' + }); + expect(await operationFor(job.deploymentId)).toMatchObject({ + id: original.id, + state: 'failed' + }); + expect( + db() + .prepare( + "SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid" + ) + .all() + ).toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]); + } + ); + + it.each([ + ['create', false], + ['create', true], + ['rebuild', false], + ['rebuild', true], + ['recovery-restart', false], + ['recovery-restart', true], + ['recovery-rebuild', false], + ['recovery-rebuild', true] + ] as const)( + 'resumes completed %s verification with the same command (receipt applied=%s)', + async (action, applied) => { + let job = await queue(); + await advance(); + if (action === 'rebuild') { + await acknowledge(job.deploymentId); + const { queueNativeRebuild } = + await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(job.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); + job = rebuild; + await advance(); + } else if (action.startsWith('recovery')) { + remote.state = 'unhealthy'; + await acknowledge(job.deploymentId); + if (action === 'recovery-rebuild') await acknowledge(job.deploymentId); + } + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + remote.state = 'starting'; + reportHostOperation(hostId, operation.id, { state: 'completed' }); + if (applied) await advance(); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { + ok: false, + message: 'Verification interrupted for inspection' + }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ + canResumeVerification: true, + canRetryWithRebuild: false + }); + const { db } = await import('../../src/lib/server/db/index.js'); + const commands = db() + .prepare( + "SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid" + ) + .all(); + const { POST } = + await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); + expect(POST({ params: { id: job.deploymentId } }).status).toBe(202); + remote.state = 'running'; + await advance(); + expect((await detail(job.deploymentId)).state).toBe('done'); + expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); + expect( + db() + .prepare( + "SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid" + ) + .all() + ).toEqual(commands); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe( + operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0) + ); + expect(history.getDeployment(job.deploymentId)!.events).toContainEqual( + expect.objectContaining({ + kind: 'outcome', + message: 'Failed: Verification interrupted for inspection' + }) + ); + } + ); + + it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)( + 'does not offer verification of a %s command or replay an ambiguous result', + async (state) => { + const job = await queue(); + await advance(); + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + if (state !== 'pending') reportHostOperation(hostId, operation.id, { state }); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { + ok: false, + message: 'Controller observation interrupted' + }); + expect(history.getDeployment(job.deploymentId)).toMatchObject({ + canResumeVerification: false, + canRetryWithRebuild: state === 'failed' + }); + expect(history.resumeVerification(job.deploymentId)).toBe(false); + if (state !== 'failed') { + const { requestRetryWithRebuild } = + await import('../../src/lib/server/provisioning/requests.js'); + expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ + ok: false, + status: 409 + }); + } + const { db } = await import('../../src/lib/server/db/index.js'); + if (state === 'unknown') + db() + .prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?") + .run(job.deploymentId); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); + expect( + db() + .prepare( + "SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')" + ) + .all() + ).toEqual([{ id: operation.id }]); + } + ); + + it.each([ + 'instance generation', + 'command generation', + 'command instance', + 'command host', + 'retired instance', + 'server port', + 'newer work', + 'creation cleanup', + 'recovery attempt' + ])('refuses to retry a failed creation after its %s changes', async (change) => { + const { job, original } = await failedCreation(); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 }); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port }); - remote.state = 'missing'; await advance(); - const rebuilding = (await operationFor(next.deploymentId))!; - expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId, - payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } }); - expect(rebuilding.id).not.toBe(original.id); - expect(nativeInstance(job.name)?.generation).toBe(1); - remote.state = 'running'; await acknowledge(next.deploymentId); - expect((await detail(next.deploymentId)).state).toBe('done'); - expect(nativeInstance(job.name)?.generation).toBe(2); - // An interrupted original is reconciled to its already-failed receipt - // on the next worker pass; the failure itself is never cleared. - expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' }); - expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' }); - expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all()) - .toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]); + if (change === 'instance generation') + db() + .prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?') + .run(job.name); + if (change === 'command generation') + db() + .prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?') + .run(original.id); + if (change === 'command instance') + db() + .prepare( + "UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?" + ) + .run(original.id); + if (change === 'command host') { + const { createHost } = await import('../../src/lib/server/hosts/index.js'); + db() + .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?') + .run(createHost({ name: 'Different host' }).id, original.id); + } + if (change === 'retired instance') + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE server=?') + .run(new Date().toISOString(), job.name); + if (change === 'server port') + db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); + if (change === 'newer work') { + const { beginDeployment } = + await import('../../src/lib/server/provisioning/history.js'); + beginDeployment(job.name, 'restart'); + } + if (change === 'creation cleanup') { + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); + expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); + } + if (change === 'recovery attempt') + db() + .prepare( + "UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?" + ) + .run(job.deploymentId); + const history = await import('../../src/lib/server/provisioning/history.js'); + const { requestRetryWithRebuild } = + await import('../../src/lib/server/provisioning/requests.js'); + expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false); + expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 }); + expect( + db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all() + ).toEqual([]); }); it.each([ - ['create', false], ['create', true], ['rebuild', false], ['rebuild', true], - ['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true] - ] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => { - let job = await queue(); await advance(); - if (action === 'rebuild') { - await acknowledge(job.deploymentId); - const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(job.name, 'test'); - if (!rebuild.ok) throw new Error(rebuild.message); - job = rebuild; await advance(); - } else if (action.startsWith('recovery')) { - remote.state = 'unhealthy'; await acknowledge(job.deploymentId); - if (action === 'recovery-rebuild') await acknowledge(job.deploymentId); - } - const operation = (await operationFor(job.deploymentId))!; - const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' }); - if (applied) await advance(); - const history = await import('../../src/lib/server/provisioning/history.js'); - history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' }); - expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false }); - const { db } = await import('../../src/lib/server/db/index.js'); - const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all(); - const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js'); - expect(POST({ params: { id: job.deploymentId } }).status).toBe(202); - remote.state = 'running'; await advance(); - expect((await detail(job.deploymentId)).state).toBe('done'); - expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0)); - expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({ - kind: 'outcome', message: 'Failed: Verification interrupted for inspection' - })); - }); - - it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => { - const job = await queue(); await advance(); - const operation = (await operationFor(job.deploymentId))!; - const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - if (state !== 'pending') reportHostOperation(hostId, operation.id, { state }); - const history = await import('../../src/lib/server/provisioning/history.js'); - history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' }); - expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' }); - expect(history.resumeVerification(job.deploymentId)).toBe(false); - if (state !== 'failed') { - const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js'); - expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 }); - } - const { db } = await import('../../src/lib/server/db/index.js'); - if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId); + 'instance generation', + 'command generation', + 'command instance', + 'command host', + 'retired instance', + 'server port', + 'newer work' + ])('refuses to resume a completed creation after its %s changes', async (change) => { + const job = await queue(); await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all()) - .toEqual([{ id: operation.id }]); + const original = (await operationFor(job.deploymentId))!; + const { reportHostOperation, createHost } = + await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, original.id, { state: 'completed' }); + const history = await import('../../src/lib/server/provisioning/history.js'); + history.finishDeployment(job.deploymentId, { + ok: false, + message: 'Verification interrupted' + }); + const { db } = await import('../../src/lib/server/db/index.js'); + if (change === 'instance generation') + db() + .prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?') + .run(job.name); + if (change === 'command generation') + db() + .prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?') + .run(original.id); + if (change === 'command instance') + db() + .prepare( + "UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?" + ) + .run(original.id); + if (change === 'command host') + db() + .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?') + .run(createHost({ name: 'Different host' }).id, original.id); + if (change === 'retired instance') + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE server=?') + .run(new Date().toISOString(), job.name); + if (change === 'server port') + db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); + if (change === 'newer work') history.beginDeployment(job.name, 'restart'); + expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false); + expect(history.resumeVerification(job.deploymentId)).toBe(false); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); }); - - it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt']) - ('refuses to retry a failed creation after its %s changes', async change => { - const { job, original } = await failedCreation(); - const { db } = await import('../../src/lib/server/db/index.js'); - if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); - if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id); - if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id); - if (change === 'command host') { - const { createHost } = await import('../../src/lib/server/hosts/index.js'); - db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id); - } - if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name); - if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); - if (change === 'newer work') { - const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); - beginDeployment(job.name, 'restart'); - } - if (change === 'creation cleanup') { - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); - expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); - } - if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId); - const history = await import('../../src/lib/server/provisioning/history.js'); - const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js'); - expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false); - expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 }); - expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]); - }); - - it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work']) - ('refuses to resume a completed creation after its %s changes', async change => { - const job = await queue(); await advance(); - const original = (await operationFor(job.deploymentId))!; - const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, original.id, { state: 'completed' }); - const history = await import('../../src/lib/server/provisioning/history.js'); - history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' }); - const { db } = await import('../../src/lib/server/db/index.js'); - if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); - if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id); - if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id); - if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id); - if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name); - if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name); - if (change === 'newer work') history.beginDeployment(job.name, 'restart'); - expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false); - expect(history.resumeVerification(job.deploymentId)).toBe(false); - expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - }); }); describe('unhealthy recovery within native create and rebuild operations', () => { async function launch(action: 'create' | 'rebuild') { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); if (action === 'create') return job; await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); @@ -1678,139 +3024,236 @@ describe('unhealthy recovery within native create and rebuild operations', () => return rebuild; } - it.each(['create', 'rebuild'] as const)('recovers an unhealthy %s with one restart then a rebuild using the original launch payload', async action => { - const { config } = await import('../../src/lib/server/config.js'); - Object.assign(config.rcon, { password: '' }); - const job = await launch(action); - const original = (await operationFor(job.deploymentId))!; - const currentGeneration = action === 'rebuild' ? 2 : 1; - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconStatusResult).mockClear(); - remote.state = 'unhealthy'; remote.players = 4; - Object.assign(config.rcon, { password: 'changed-after-original-launch' }); - await acknowledge(job.deploymentId); - const restart = (await operationFor(job.deploymentId))!; - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - expect(restart).toMatchObject({ kind: 'instance.restart', state: 'pending', generation: currentGeneration, - instanceId: original.instanceId, payload: { unhealthyRecovery: true } }); - expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual(expect.arrayContaining([ - expect.objectContaining({ kind: 'startup', ownsSlot: true }), - expect.objectContaining({ kind: 'restart', ownsSlot: true }) - ])); - expect(restart.id).not.toBe(original.id); - expect(nativeInstance(job.name)?.generation).toBe(currentGeneration); - expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId)) - .toEqual({ recovery_step: 'restart', generation: currentGeneration }); - expect(rconStatusResult).not.toHaveBeenCalled(); + it.each(['create', 'rebuild'] as const)( + 'recovers an unhealthy %s with one restart then a rebuild using the original launch payload', + async (action) => { + const { config } = await import('../../src/lib/server/config.js'); + Object.assign(config.rcon, { password: '' }); + const job = await launch(action); + const original = (await operationFor(job.deploymentId))!; + const currentGeneration = action === 'rebuild' ? 2 : 1; + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockClear(); + remote.state = 'unhealthy'; + remote.players = 4; + Object.assign(config.rcon, { password: 'changed-after-original-launch' }); + await acknowledge(job.deploymentId); + const restart = (await operationFor(job.deploymentId))!; + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + expect(restart).toMatchObject({ + kind: 'instance.restart', + state: 'pending', + generation: currentGeneration, + instanceId: original.instanceId, + payload: { unhealthyRecovery: true } + }); + expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'startup', ownsSlot: true }), + expect.objectContaining({ kind: 'restart', ownsSlot: true }) + ]) + ); + expect(restart.id).not.toBe(original.id); + expect(nativeInstance(job.name)?.generation).toBe(currentGeneration); + expect( + db() + .prepare( + 'SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?' + ) + .get(job.deploymentId) + ).toEqual({ recovery_step: 'restart', generation: currentGeneration }); + expect(rconStatusResult).not.toHaveBeenCalled(); - // Restarting the controller must preserve the current recovery command. - closeDb(); await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(restart.id); - await acknowledge(job.deploymentId); - const rebuild = (await operationFor(job.deploymentId))!; - expect(rebuild).toMatchObject({ kind: 'instance.rebuild', state: 'pending', generation: currentGeneration, - instanceId: original.instanceId, payload: { unhealthyRecovery: true, nextGeneration: currentGeneration + 1, - files: original.payload.files, env: original.payload.env, rconPassword: original.payload.rconPassword, - libraryId: original.payload.libraryId, image: original.payload.image } }); - expect(rebuild.id).not.toBe(restart.id); - expect(nativeInstance(job.name)?.generation).toBe(currentGeneration); - expect(rconStatusResult).not.toHaveBeenCalled(); - expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId)) - .toEqual({ recovery_step: 'rebuild', generation: currentGeneration }); - remote.state = 'running'; remote.players = 0; - await acknowledge(job.deploymentId); - expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1); - expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' }); - expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released'); - expect(getDeployment(job.deploymentId)!.capacity.pools.every(pool => !pool.ownsSlot)).toBe(true); - await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id); - expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1); - }); + // Restarting the controller must preserve the current recovery command. + closeDb(); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(restart.id); + await acknowledge(job.deploymentId); + const rebuild = (await operationFor(job.deploymentId))!; + expect(rebuild).toMatchObject({ + kind: 'instance.rebuild', + state: 'pending', + generation: currentGeneration, + instanceId: original.instanceId, + payload: { + unhealthyRecovery: true, + nextGeneration: currentGeneration + 1, + files: original.payload.files, + env: original.payload.env, + rconPassword: original.payload.rconPassword, + libraryId: original.payload.libraryId, + image: original.payload.image + } + }); + expect(rebuild.id).not.toBe(restart.id); + expect(nativeInstance(job.name)?.generation).toBe(currentGeneration); + expect(rconStatusResult).not.toHaveBeenCalled(); + expect( + db() + .prepare( + 'SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?' + ) + .get(job.deploymentId) + ).toEqual({ recovery_step: 'rebuild', generation: currentGeneration }); + remote.state = 'running'; + remote.players = 0; + await acknowledge(job.deploymentId); + expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1); + expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' }); + expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released'); + expect( + getDeployment(job.deploymentId)!.capacity.pools.every((pool) => !pool.ownsSlot) + ).toBe(true); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id); + expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1); + } + ); - it.each(['create', 'rebuild'] as const)('finishes %s after a successful recovery restart without another rebuild', async action => { - const job = await launch(action); - remote.state = 'unhealthy'; await acknowledge(job.deploymentId); - const restart = (await operationFor(job.deploymentId))!; - expect(restart.kind).toBe('instance.restart'); - remote.state = 'running'; await acknowledge(job.deploymentId); await advance(); - expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' }); - expect((await operationFor(job.deploymentId))!.id).toBe(restart.id); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()) - .toEqual({ count: action === 'rebuild' ? 1 : 0 }); - }); + it.each(['create', 'rebuild'] as const)( + 'finishes %s after a successful recovery restart without another rebuild', + async (action) => { + const job = await launch(action); + remote.state = 'unhealthy'; + await acknowledge(job.deploymentId); + const restart = (await operationFor(job.deploymentId))!; + expect(restart.kind).toBe('instance.restart'); + remote.state = 'running'; + await acknowledge(job.deploymentId); + await advance(); + expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' }); + expect((await operationFor(job.deploymentId))!.id).toBe(restart.id); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'" + ) + .get() + ).toEqual({ count: action === 'rebuild' ? 1 : 0 }); + } + ); - it.each(['restart', 'rebuild'] as const)('retains an unknown recovery %s receipt without replaying or advancing recovery', async step => { - const job = await launch('create'); - remote.state = 'unhealthy'; await acknowledge(job.deploymentId); - if (step === 'rebuild') await acknowledge(job.deploymentId); - const operation = (await operationFor(job.deploymentId))!; - expect(operation.kind).toBe(`instance.${step}`); - const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Reply lost during recovery' }); - const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb(); - const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments(); - await advance(); await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get()) - .toEqual({ count: step === 'restart' ? 2 : 3 }); - expect((await detail(job.deploymentId)).state).not.toBe('failed'); - remote.state = 'running'; await acknowledge(job.deploymentId); - expect((await detail(job.deploymentId)).state).toBe('done'); - }); + it.each(['restart', 'rebuild'] as const)( + 'retains an unknown recovery %s receipt without replaying or advancing recovery', + async (step) => { + const job = await launch('create'); + remote.state = 'unhealthy'; + await acknowledge(job.deploymentId); + if (step === 'rebuild') await acknowledge(job.deploymentId); + const operation = (await operationFor(job.deploymentId))!; + expect(operation.kind).toBe(`instance.${step}`); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { + state: 'unknown', + error: 'Reply lost during recovery' + }); + const { closeDb, db } = await import('../../src/lib/server/db/index.js'); + closeDb(); + const { interruptDeployments } = + await import('../../src/lib/server/provisioning/history.js'); + interruptDeployments(); + await advance(); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(operation.id); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')" + ) + .get() + ).toEqual({ count: step === 'restart' ? 2 : 3 }); + expect((await detail(job.deploymentId)).state).not.toBe('failed'); + remote.state = 'running'; + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + } + ); - it.each(['starting', 'restarting', 'unknown'])('waits for %s container readiness without forcing recovery', async state => { - const job = await launch('create'); - const original = (await operationFor(job.deploymentId))!; - remote.state = state; await acknowledge(job.deploymentId); await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: `Waiting for container readiness: ${state}` }); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId)) - .toEqual({ recovery_step: null }); - }); + it.each(['starting', 'restarting', 'unknown'])( + 'waits for %s container readiness without forcing recovery', + async (state) => { + const job = await launch('create'); + const original = (await operationFor(job.deploymentId))!; + remote.state = state; + await acknowledge(job.deploymentId); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + expect(await detail(job.deploymentId)).toMatchObject({ + state: 'awaiting-health', + stage: `Waiting for container readiness: ${state}` + }); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?') + .get(job.deploymentId) + ).toEqual({ recovery_step: null }); + } + ); - it.each(['create', 'rebuild'] as const)('does not submit recovery if %s is cancelled during its health probe', async action => { - const job = await launch(action); - const original = (await operationFor(job.deploymentId))!; - const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, original.id, { state: 'completed' }); - const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); - const pending = advance(); - await vi.waitFor(() => expect(release).toBeTypeOf('function')); - expect(cancelProvisioningJob(job.deploymentId)).toBe(true); - release({ healthy: false, state: 'unhealthy' } as never); - await pending; await advance(); - expect((await detail(job.deploymentId)).state).toBe('cancelled'); - expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]); - }); + it.each(['create', 'rebuild'] as const)( + 'does not submit recovery if %s is cancelled during its health probe', + async (action) => { + const job = await launch(action); + const original = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, original.id, { state: 'completed' }); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + let release!: (value: never) => void; + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); + const pending = advance(); + await vi.waitFor(() => expect(release).toBeTypeOf('function')); + expect(cancelProvisioningJob(job.deploymentId)).toBe(true); + release({ healthy: false, state: 'unhealthy' } as never); + await pending; + await advance(); + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'") + .all() + ).toEqual([]); + } + ); - it.each(['create', 'rebuild'] as const)('reconciles a delivered recovery rebuild after its %s flow is cancelled', async action => { - const job = await launch(action); - remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await acknowledge(job.deploymentId); - const rebuild = (await operationFor(job.deploymentId))!; - expect(rebuild.kind).toBe('instance.rebuild'); - const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - pendingHostOperations(hostId); - expect(cancelProvisioningJob(job.deploymentId)).toBe(true); - reportHostOperation(hostId, rebuild.id, { state: 'completed' }); - await advance(); await advance(); - const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2); - expect((await detail(job.deploymentId)).state).toBe('cancelled'); - expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id); - }); + it.each(['create', 'rebuild'] as const)( + 'reconciles a delivered recovery rebuild after its %s flow is cancelled', + async (action) => { + const job = await launch(action); + remote.state = 'unhealthy'; + await acknowledge(job.deploymentId); + await acknowledge(job.deploymentId); + const rebuild = (await operationFor(job.deploymentId))!; + expect(rebuild.kind).toBe('instance.rebuild'); + const { pendingHostOperations, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + pendingHostOperations(hostId); + expect(cancelProvisioningJob(job.deploymentId)).toBe(true); + reportHostOperation(hostId, rebuild.id, { state: 'completed' }); + await advance(); + await advance(); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2); + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id); + } + ); it('fails clearly after an unhealthy recovery rebuild instead of looping', async () => { const job = await launch('create'); @@ -1819,151 +3262,277 @@ describe('unhealthy recovery within native create and rebuild operations', () => await acknowledge(job.deploymentId); const rebuild = (await operationFor(job.deploymentId))!; expect(rebuild.kind).toBe('instance.rebuild'); - await acknowledge(job.deploymentId); await advance(); - expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: expect.stringMatching(/unhealthy.*restart.*rebuild/i) }); + await acknowledge(job.deploymentId); + await advance(); + expect(await detail(job.deploymentId)).toMatchObject({ + state: 'failed', + error: expect.stringMatching(/unhealthy.*restart.*rebuild/i) + }); expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get()) - .toEqual({ count: 3 }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')" + ) + .get() + ).toEqual({ count: 3 }); }); - it.each(['create', 'rebuild'] as const)('holds unhealthy %s verification until the agent supports recovery', async action => { - const job = await launch(action); - const original = (await operationFor(job.deploymentId))!; - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - heartbeat(hostId, { ...inventory(), capabilities: [] }); - remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await advance(); - expect((await operationFor(job.deploymentId))!.id).toBe(original.id); - expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: expect.stringMatching(/agent/i) }); - heartbeat(hostId, inventory()); await advance(); - expect(await operationFor(job.deploymentId)).toMatchObject({ kind: 'instance.restart', payload: { unhealthyRecovery: true } }); - }); - - it.each([false, true])('rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)', async supported => { - const created = await launch('create'); await acknowledge(created.deploymentId); - const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] }); - const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); - const rebuild = queueNativeRebuild(created.name, 'test'); - if (!rebuild.ok) throw new Error(rebuild.message); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconStatusResult).mockClear(); - remote.state = 'unhealthy'; remote.players = 3; - await advance(); - if (!supported) { - expect(await operationFor(rebuild.deploymentId)).toBeNull(); - expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i); - heartbeat(hostId, inventory()); await advance(); + it.each(['create', 'rebuild'] as const)( + 'holds unhealthy %s verification until the agent supports recovery', + async (action) => { + const job = await launch(action); + const original = (await operationFor(job.deploymentId))!; + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + heartbeat(hostId, { ...inventory(), capabilities: [] }); + remote.state = 'unhealthy'; + await acknowledge(job.deploymentId); + await advance(); + expect((await operationFor(job.deploymentId))!.id).toBe(original.id); + expect(await detail(job.deploymentId)).toMatchObject({ + state: 'awaiting-health', + stage: expect.stringMatching(/agent/i) + }); + heartbeat(hostId, inventory()); + await advance(); + expect(await operationFor(job.deploymentId)).toMatchObject({ + kind: 'instance.restart', + payload: { unhealthyRecovery: true } + }); } - expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1, - payload: { unhealthyRecovery: true, nextGeneration: 2 } }); - expect(rconStatusResult).not.toHaveBeenCalled(); - }); + ); + + it.each([false, true])( + 'rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)', + async (supported) => { + const created = await launch('create'); + await acknowledge(created.deploymentId); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] }); + const { queueNativeRebuild } = + await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(created.name, 'test'); + if (!rebuild.ok) throw new Error(rebuild.message); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockClear(); + remote.state = 'unhealthy'; + remote.players = 3; + await advance(); + if (!supported) { + expect(await operationFor(rebuild.deploymentId)).toBeNull(); + expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i); + heartbeat(hostId, inventory()); + await advance(); + } + expect(await operationFor(rebuild.deploymentId)).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + payload: { unhealthyRecovery: true, nextGeneration: 2 } + }); + expect(rconStatusResult).not.toHaveBeenCalled(); + } + ); }); describe('native provisioning cancellation and rollback', () => { it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js'); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const restart = requestLifecycle(job.name, 'restart'); if (!restart.ok) throw new Error(restart.error); + const restart = requestLifecycle(job.name, 'restart'); + if (!restart.ok) throw new Error(restart.error); await advance(); const command = (await operationFor(restart.deploymentId))!; reportHostOperation(hostId, command.id, { state: 'completed' }); let release!: (value: never) => void; - vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + vi.mocked(nativeRequest).mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = advance(); await vi.waitFor(() => expect(release).toBeTypeOf('function')); expect(POST({ params: { id: restart.deploymentId } }).status).toBe(200); release({ healthy: true, state: 'running' } as never); await pending; expect(await detail(restart.deploymentId)).toMatchObject({ state: 'cancelled' }); - const replacement = requestLifecycle(job.name, 'restart'); if (!replacement.ok) throw new Error(replacement.error); + const replacement = requestLifecycle(job.name, 'restart'); + if (!replacement.ok) throw new Error(replacement.error); await advance(); - expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', state: 'pending' }); + expect(await operationFor(replacement.deploymentId)).toMatchObject({ + kind: 'instance.restart', + state: 'pending' + }); }); - it.each([false, true])('withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)', async delivered => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const first = requestLifecycle(job.name, 'restart'); if (!first.ok) throw new Error(first.error); - await advance(); const command = (await operationFor(first.deploymentId))!; - if (delivered) pendingHostOperations(hostId); - expect(cancelProvisioningJob(first.deploymentId)).toBe(true); - const next = requestLifecycle(job.name, 'restart'); if (!next.ok) throw new Error(next.error); - await advance(); - if (delivered) { - expect(await operationFor(next.deploymentId)).toBeNull(); - reportHostOperation(hostId, command.id, { state: 'completed' }); + it.each([false, true])( + 'withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)', + async (delivered) => { + const job = await queue(); await advance(); - } else { - expect((await operationFor(first.deploymentId))?.state).toBe('failed'); - expect(pendingHostOperations(hostId).some(op => op.id === command.id)).toBe(false); + await acknowledge(job.deploymentId); + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const { pendingHostOperations, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + const first = requestLifecycle(job.name, 'restart'); + if (!first.ok) throw new Error(first.error); + await advance(); + const command = (await operationFor(first.deploymentId))!; + if (delivered) pendingHostOperations(hostId); + expect(cancelProvisioningJob(first.deploymentId)).toBe(true); + const next = requestLifecycle(job.name, 'restart'); + if (!next.ok) throw new Error(next.error); + await advance(); + if (delivered) { + expect(await operationFor(next.deploymentId)).toBeNull(); + reportHostOperation(hostId, command.id, { state: 'completed' }); + await advance(); + } else { + expect((await operationFor(first.deploymentId))?.state).toBe('failed'); + expect(pendingHostOperations(hostId).some((op) => op.id === command.id)).toBe( + false + ); + } + expect(await operationFor(next.deploymentId)).not.toBeNull(); + expect((await detail(first.deploymentId)).state).toBe('cancelled'); } - expect(await operationFor(next.deploymentId)).not.toBeNull(); - expect((await detail(first.deploymentId)).state).toBe('cancelled'); - }); + ); it('rolls back a completed native activation even while its restart is stuck starting', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { db } = await import('../../src/lib/server/db/index.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { script, version } = createScript({ name: 'Broken beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('broken'), scopeType: 'game', scopeValue: 't5' }); - const betaId = Number(db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id,version.id).lastInsertRowid); - db().prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)').run(betaId,script.id,version.id); - db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId,job.name); - db().prepare('INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)').run(betaId,job.name,script.id); - queueFileChange({ sourceKind: 'beta', sourceId: String(betaId), server: job.name, kind: 'script', relPath: 'beta.gsc', content: Buffer.from('broken'), inventorySource: 'beta' }); - const restart = requestRestart(job.name,{kind:'beta',id:String(betaId)}); + const { script, version } = createScript({ + name: 'Broken beta', + kind: 'script', + targetRelPath: 'beta.gsc', + content: Buffer.from('broken'), + scopeType: 'game', + scopeValue: 't5' + }); + const betaId = Number( + db() + .prepare( + "INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')" + ) + .run(script.id, version.id).lastInsertRowid + ); + db() + .prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)') + .run(betaId, script.id, version.id); + db() + .prepare( + "INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')" + ) + .run(betaId, job.name); + db() + .prepare( + 'INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)' + ) + .run(betaId, job.name, script.id); + queueFileChange({ + sourceKind: 'beta', + sourceId: String(betaId), + server: job.name, + kind: 'script', + relPath: 'beta.gsc', + content: Buffer.from('broken'), + inventorySource: 'beta' + }); + const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) }); await advance(); - const staging = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {stage_operation_id:string}; - reportHostOperation(hostId,staging.stage_operation_id,{state:'completed'}); + const staging = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { stage_operation_id: string }; + reportHostOperation(hostId, staging.stage_operation_id, { state: 'completed' }); await advance(); const activation = (await operationFor(restart.id))!; - const files = JSON.parse((db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {files_json:string}).files_json); + const files = JSON.parse( + ( + db() + .prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { files_json: string } + ).files_json + ); remote.state = 'starting'; - reportHostOperation(hostId,activation.id,{state:'completed',result:{before:files.map((f:{id:number;path:string})=>({...f,contentBase64:Buffer.from('working').toString('base64')}))}}); + reportHostOperation(hostId, activation.id, { + state: 'completed', + result: { + before: files.map((f: { id: number; path: string }) => ({ + ...f, + contentBase64: Buffer.from('working').toString('base64') + })) + } + }); // Rollback must consume the receipt itself; the health worker has not caught up. const rollback = await rollbackBeta(betaId); expect((await detail(restart.id)).state).toBe('cancelled'); - const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string}; + const replacement = db() + .prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?') + .get(rollback.deploymentId) as { provisioning_id: string }; expect(replacement.provisioning_id).not.toBe(restart.id); - expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId)) - .toEqual({content:Buffer.from('working')}); + expect( + db() + .prepare( + "SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?" + ) + .get(rollback.deploymentId) + ).toEqual({ content: Buffer.from('working') }); await advance(); - const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string}; - reportHostOperation(hostId,stagedRollback.stage_operation_id,{state:'completed'}); + const stagedRollback = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(replacement.provisioning_id) as { stage_operation_id: string }; + reportHostOperation(hostId, stagedRollback.stage_operation_id, { state: 'completed' }); await advance(); - expect(await operationFor(replacement.provisioning_id)).toMatchObject({state:'pending'}); + expect(await operationFor(replacement.provisioning_id)).toMatchObject({ state: 'pending' }); }); }); describe('partial rebuild failure reconciliation', () => { const originalError = 'Replacement game library could not be prepared'; - const failure = (generation = 1) => ({ reconciled: true, originalError, rebuildFailure: { - phase: 'disrupting', generation, nextGeneration: generation + 1, - oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true - } }); + const failure = (generation = 1) => ({ + reconciled: true, + originalError, + rebuildFailure: { + phase: 'disrupting', + generation, + nextGeneration: generation + 1, + oldContainerMissing: true, + replacementContainerMissing: true, + replacementFilesRemoved: true + } + }); async function cancelledRebuild() { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const first = requestLifecycle(job.name, 'rebuild'); if (!first.ok) throw new Error(first.error); await advance(); const command = (await operationFor(first.deploymentId))!; - const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { pendingHostOperations, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); pendingHostOperations(hostId); reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError }); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(first.deploymentId)).toBe(true); const replacement = requestLifecycle(job.name, 'rebuild'); if (!replacement.ok) throw new Error(replacement.error); @@ -1974,245 +3543,480 @@ describe('partial rebuild failure reconciliation', () => { const { job, first, command, replacement } = await cancelledRebuild(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { getHostOperation, reportHostOperation, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js'); - const { nativeInstance, nativeRequest } = await import('../../src/lib/server/backends/native.js'); + const { getHostOperation, reportHostOperation, pendingHostOperations } = + await import('../../src/lib/server/hosts/index.js'); + const { nativeInstance, nativeRequest } = + await import('../../src/lib/server/backends/native.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { unsettledCancelledCommand } = await import('../../src/lib/server/provisioning/cancellation.js'); - const ports = db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all(); + const { unsettledCancelledCommand } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const ports = db() + .prepare( + 'SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol' + ) + .all(); await advance(); expect(await operationFor(replacement.deploymentId)).toBeNull(); - expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({ code: 'cancelled-command', - href: `/provisioning/${first.deploymentId}`, label: 'View cancelled operation', - message: expect.stringContaining(originalError) }); - expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain('unknown'); - reportHostOperation(hostId, command.id, { state: 'failed', - error: `Old container removed; replacement creation failed. ${originalError}`, result: failure() }); + expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({ + code: 'cancelled-command', + href: `/provisioning/${first.deploymentId}`, + label: 'View cancelled operation', + message: expect.stringContaining(originalError) + }); + expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain( + 'unknown' + ); + reportHostOperation(hostId, command.id, { + state: 'failed', + error: `Old container removed; replacement creation failed. ${originalError}`, + result: failure() + }); expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).not.toBeNull(); - expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId)) - .toEqual({ applied_at: null }); + expect( + db() + .prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?') + .get(first.deploymentId) + ).toEqual({ applied_at: null }); remote.state = 'missing'; vi.mocked(nativeRequest).mockImplementationOnce(async () => { - expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name)) - .toEqual({ container_state: 'missing', last_players: null }); - expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId)) - .toEqual({ applied_at: expect.any(String) }); + expect( + db() + .prepare('SELECT container_state,last_players FROM servers WHERE name=?') + .get(job.name) + ).toEqual({ container_state: 'missing', last_players: null }); + expect( + db() + .prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?') + .get(first.deploymentId) + ).toEqual({ applied_at: expect.any(String) }); expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).toBeNull(); return { exists: false, healthy: false, state: 'missing' } as never; }); - closeDb(); await advance(); + closeDb(); + await advance(); const next = (await operationFor(replacement.deploymentId))!; - expect(next).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: command.instanceId, - payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } }); + expect(next).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + instanceId: command.instanceId, + payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } + }); expect(next.id).not.toBe(command.id); - expect(nativeInstance(job.name)).toMatchObject({ id: command.instanceId, generation: 1, retired_at: null }); + expect(nativeInstance(job.name)).toMatchObject({ + id: command.instanceId, + generation: 1, + retired_at: null + }); expect(getServerKey(job.name)).toBe('saved-key'); - expect(db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual(ports); + expect( + db() + .prepare( + 'SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol' + ) + .all() + ).toEqual(ports); expect((await detail(first.deploymentId)).state).toBe('cancelled'); expect(getHostOperation(command.id)).toMatchObject({ state: 'failed', result: failure() }); - expect(pendingHostOperations(hostId).some(operation => operation.id === command.id)).toBe(false); + expect(pendingHostOperations(hostId).some((operation) => operation.id === command.id)).toBe( + false + ); await advance(); expect((await operationFor(replacement.deploymentId))!.id).toBe(next.id); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 2 }); - remote.state = 'running'; await acknowledge(replacement.deploymentId); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'" + ) + .get() + ).toEqual({ count: 2 }); + remote.state = 'running'; + await acknowledge(replacement.deploymentId); expect((await detail(replacement.deploymentId)).state).toBe('done'); expect(nativeInstance(job.name)?.generation).toBe(2); expect((await detail(first.deploymentId)).state).toBe('cancelled'); }); - it.each(['disrupting', 'creating'])('keeps an ambiguous %s rebuild blocked even when inspections currently see no containers', async phase => { - const { job, first, command, replacement } = await cancelledRebuild(); - const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError, - result: { checkpoint: { phase }, state: 'missing', oldContainerMissing: true, replacementContainerMissing: true } }); - remote.state = 'missing'; - const { nativeRequest, nativeInstance } = await import('../../src/lib/server/backends/native.js'); - vi.mocked(nativeRequest).mockClear(); - const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb(); - await advance(); await advance(); - expect(await operationFor(replacement.deploymentId)).toBeNull(); - expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled(); - expect((await operationFor(first.deploymentId))!.id).toBe(command.id); - expect(nativeInstance(job.name)?.generation).toBe(1); - expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId)) - .toEqual({ applied_at: null }); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 }); - }); + it.each(['disrupting', 'creating'])( + 'keeps an ambiguous %s rebuild blocked even when inspections currently see no containers', + async (phase) => { + const { job, first, command, replacement } = await cancelledRebuild(); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, command.id, { + state: 'unknown', + error: originalError, + result: { + checkpoint: { phase }, + state: 'missing', + oldContainerMissing: true, + replacementContainerMissing: true + } + }); + remote.state = 'missing'; + const { nativeRequest, nativeInstance } = + await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockClear(); + const { closeDb, db } = await import('../../src/lib/server/db/index.js'); + closeDb(); + await advance(); + await advance(); + expect(await operationFor(replacement.deploymentId)).toBeNull(); + expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled(); + expect((await operationFor(first.deploymentId))!.id).toBe(command.id); + expect(nativeInstance(job.name)?.generation).toBe(1); + expect( + db() + .prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?') + .get(first.deploymentId) + ).toEqual({ applied_at: null }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'" + ) + .get() + ).toEqual({ count: 1 }); + } + ); - it.each(['creation started', 'replacement files remain', 'wrong target generation', 'command target mismatch'])('does not release a replacement from invalid partial evidence: %s', async evidence => { + it.each([ + 'creation started', + 'replacement files remain', + 'wrong target generation', + 'command target mismatch' + ])('does not release a replacement from invalid partial evidence: %s', async (evidence) => { const { first, command, replacement } = await cancelledRebuild(); const result = failure(); if (evidence === 'creation started') result.rebuildFailure.phase = 'creating'; - if (evidence === 'replacement files remain') result.rebuildFailure.replacementFilesRemoved = false; + if (evidence === 'replacement files remain') + result.rebuildFailure.replacementFilesRemoved = false; if (evidence === 'wrong target generation') result.rebuildFailure.nextGeneration = 3; const { db } = await import('../../src/lib/server/db/index.js'); - if (evidence === 'command target mismatch') db().prepare("UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?") - .run(command.id); + if (evidence === 'command target mismatch') + db() + .prepare( + "UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?" + ) + .run(command.id); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); reportHostOperation(hostId, command.id, { state: 'failed', error: originalError, result }); - remote.state = 'missing'; await advance(); + remote.state = 'missing'; + await advance(); expect(await operationFor(replacement.deploymentId)).toBeNull(); - expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId)) - .toEqual({ applied_at: null }); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 }); + expect( + db() + .prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?') + .get(first.deploymentId) + ).toEqual({ applied_at: null }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'" + ) + .get() + ).toEqual({ count: 1 }); }); it('records an uncancelled partial rebuild failure without claiming replacement generation or successful health', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const first = requestLifecycle(job.name, 'rebuild'); if (!first.ok) throw new Error(first.error); await advance(); const command = (await operationFor(first.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, command.id, { state: 'failed', result: failure(), - error: `Old container removed; replacement creation failed. ${originalError}` }); + reportHostOperation(hostId, command.id, { + state: 'failed', + result: failure(), + error: `Old container removed; replacement creation failed. ${originalError}` + }); await advance(); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect((await detail(first.deploymentId)).state).toBe('failed'); expect(nativeInstance(job.name)).toMatchObject({ generation: 1, retired_at: null }); - expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name)) - .toEqual({ container_state: 'missing', last_players: null }); - expect(db().prepare('SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId)) - .toEqual({ applied_at: expect.any(String), verified_at: null }); - expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?').get(command.instanceId)).toEqual({ count: 2 }); + expect( + db() + .prepare('SELECT container_state,last_players FROM servers WHERE name=?') + .get(job.name) + ).toEqual({ container_state: 'missing', last_players: null }); + expect( + db() + .prepare( + 'SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?' + ) + .get(first.deploymentId) + ).toEqual({ applied_at: expect.any(String), verified_at: null }); + expect( + db() + .prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?') + .get(command.instanceId) + ).toEqual({ count: 2 }); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(job.name)).toBe('saved-key'); }); }); describe('cancelled native mutation bookkeeping', () => { - it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { db } = await import('../../src/lib/server/db/index.js'); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta, reviseBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); - const { reportHostOperation, getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const { script, version } = createScript({ name: 'Revised beta', kind: 'script', targetRelPath: 'revision.gsc', - content: Buffer.from('revision one'), scopeType: 'game', scopeValue: 't5' }); - const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: [job.name], waitForHealth: true, waitForMatch: false }); - async function activate(before: string | null) { - const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) }); + it.each([null, 'before beta'])( + 'keeps the original host receipt across beta revisions (original=%s)', + async (original) => { + const job = await queue(); await advance(); - const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?').get(restart.id) as { stage_operation_id: string }; - const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ id: number; path: string }>; - reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' }); - await advance(); - // A dispatched activation must settle before revision submission. - await expect(reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('premature update') }])).rejects.toThrow(); - const activation = (await operationFor(restart.id))!; - reportHostOperation(hostId, activation.id, { state: 'completed', result: { - before: files.map(file => ({ ...file, contentBase64: before === null ? null : Buffer.from(before).toString('base64') })) - } }); - await advance(); - expect((await detail(restart.id)).state).toBe('done'); + await acknowledge(job.deploymentId); + const { db } = await import('../../src/lib/server/db/index.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta, reviseBeta, rollbackBeta } = + await import('../../src/lib/server/fleet/beta.js'); + const { requestRestart } = + await import('../../src/lib/server/provisioning/requests.js'); + const { reportHostOperation, getHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + const { script, version } = createScript({ + name: 'Revised beta', + kind: 'script', + targetRelPath: 'revision.gsc', + content: Buffer.from('revision one'), + scopeType: 'game', + scopeValue: 't5' + }); + const { betaId } = await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: [job.name], + waitForHealth: true, + waitForMatch: false + }); + async function activate(before: string | null) { + const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) }); + await advance(); + const staged = db() + .prepare( + 'SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?' + ) + .get(restart.id) as { stage_operation_id: string }; + const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ + id: number; + path: string; + }>; + reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' }); + await advance(); + // A dispatched activation must settle before revision submission. + await expect( + reviseBeta(betaId, 1, [ + { scriptId: script.id, content: Buffer.from('premature update') } + ]) + ).rejects.toThrow(); + const activation = (await operationFor(restart.id))!; + reportHostOperation(hostId, activation.id, { + state: 'completed', + result: { + before: files.map((file) => ({ + ...file, + contentBase64: + before === null ? null : Buffer.from(before).toString('base64') + })) + } + }); + await advance(); + expect((await detail(restart.id)).state).toBe('done'); + } + await activate(original); + await reviseBeta(betaId, 1, [ + { scriptId: script.id, content: Buffer.from('revision two') } + ]); + await activate('revision one'); + const rollback = await rollbackBeta(betaId); + expect( + db() + .prepare( + "SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?" + ) + .get(rollback.deploymentId) + ).toEqual({ content: original === null ? null : Buffer.from(original) }); } - await activate(original); - await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]); - await activate('revision one'); - const rollback = await rollbackBeta(betaId); - expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId)) - .toEqual({ content: original === null ? null : Buffer.from(original) }); - }); + ); it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => { const job = await queue(); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelCreation, canCancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelProvisioningJob(job.deploymentId)).toBe(true); expect(canCancelCreation(job.deploymentId)).toBe(true); const cleanup = cancelCreation(job.deploymentId)!; expect((await detail(cleanup)).state).toBe('done'); }); - it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { db } = await import('../../src/lib/server/db/index.js'); - const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const result = action === 'rebuild' ? requestLifecycle(job.name,'rebuild') : destroyServer(job.name); - if (!result.ok || !result.deploymentId) throw new Error('Job was not queued'); - const id = result.deploymentId; - await advance(); const operation = (await operationFor(id))!; - pendingHostOperations(hostId); - expect(cancelProvisioningJob(id)).toBe(true); - expect(serverKeyPool('t5').available).toBe(0); - const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null; - if (replacement && !replacement.ok) throw new Error(replacement.error); - await advance(); - reportHostOperation(hostId,operation.id,{state:'completed'}); - await advance(); await advance(); - if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 }); - expect((await detail(id)).state).toBe('cancelled'); - expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0); - expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key'); - if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null}); - else { - expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); - expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined(); + it.each(['rebuild', 'destroy'] as const)( + 'reconciles a delivered %s receipt without resuming the cancelled job', + async (action) => { + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { db } = await import('../../src/lib/server/db/index.js'); + const { serverKeyPool, getServerKey } = + await import('../../src/lib/server/fleet/server-keys.js'); + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + const { destroyServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const { pendingHostOperations, reportHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + const result = + action === 'rebuild' + ? requestLifecycle(job.name, 'rebuild') + : destroyServer(job.name); + if (!result.ok || !result.deploymentId) throw new Error('Job was not queued'); + const id = result.deploymentId; + await advance(); + const operation = (await operationFor(id))!; + pendingHostOperations(hostId); + expect(cancelProvisioningJob(id)).toBe(true); + expect(serverKeyPool('t5').available).toBe(0); + const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null; + if (replacement && !replacement.ok) throw new Error(replacement.error); + await advance(); + reportHostOperation(hostId, operation.id, { state: 'completed' }); + await advance(); + await advance(); + if (replacement?.ok) + expect(await operationFor(replacement.deploymentId)).toMatchObject({ + kind: 'instance.restart', + generation: 2 + }); + expect((await detail(id)).state).toBe('cancelled'); + expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0); + expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key'); + if (action === 'rebuild') + expect( + db() + .prepare( + 'SELECT generation,retired_at FROM native_instances WHERE server=?' + ) + .get(job.name) + ).toEqual({ generation: 2, retired_at: null }); + else { + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); + expect( + db().prepare('SELECT * FROM servers WHERE name=?').get(job.name) + ).toBeUndefined(); + } } - }); + ); - it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const hosts = await import('../../src/lib/server/hosts/index.js'); - const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const destination = hosts.createHost({name:'Destination',portMin:29000,portMax:29005}); - hosts.enrollHost(hosts.createEnrollment(destination.id).token,inventory()); - updateHostLibraryCaches(destination.id,[{libraryVersionId:libraryId,status:'ready',manifestSha256:getLibraryVersion(libraryId)!.manifestSha256}]); - hosts.updateHost(hostId,{mode:'draining'}); - await processProvisioning(); - const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as {deployment_id:string;remove_operation_id:string}; - expect(migration.remove_operation_id).toBeTruthy(); - if(delivered) hosts.pendingHostOperations(hostId); - expect(cancelProvisioningJob(migration.deployment_id)).toBe(true); - expect(serverKeyPool('t5').available).toBe(0); - if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'}); - await processProvisioning(); await processProvisioning(); - expect((await detail(migration.deployment_id)).state).toBe('cancelled'); - expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]); - expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1); - expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2); - expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0); - expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key'); - }); + it.each([false, true])( + 'cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', + async (delivered) => { + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const { serverKeyPool, getServerKey } = + await import('../../src/lib/server/fleet/server-keys.js'); + const { getLibraryVersion, updateHostLibraryCaches } = + await import('../../src/lib/server/game-files/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { processProvisioning } = + await import('../../src/lib/server/provisioning/worker.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const destination = hosts.createHost({ + name: 'Destination', + portMin: 29000, + portMax: 29005 + }); + hosts.enrollHost(hosts.createEnrollment(destination.id).token, inventory()); + updateHostLibraryCaches(destination.id, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 + } + ]); + hosts.updateHost(hostId, { mode: 'draining' }); + await processProvisioning(); + const migration = db() + .prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations') + .get() as { deployment_id: string; remove_operation_id: string }; + expect(migration.remove_operation_id).toBeTruthy(); + if (delivered) hosts.pendingHostOperations(hostId); + expect(cancelProvisioningJob(migration.deployment_id)).toBe(true); + expect(serverKeyPool('t5').available).toBe(0); + if (delivered) + hosts.reportHostOperation(hostId, migration.remove_operation_id, { + state: 'completed' + }); + await processProvisioning(); + await processProvisioning(); + expect((await detail(migration.deployment_id)).state).toBe('cancelled'); + expect( + db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id) + ).toEqual([]); + expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1); + expect( + db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId) + ).toHaveLength(delivered ? 0 : 2); + expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0); + expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key'); + } + ); }); describe('additional concurrency admission', () => { - it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)('holds simultaneous startups with %s until recovery completes', async key => { - const first = await queue(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 }); - const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); - const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' }); - if (!second.ok) throw new Error(second.message); - await advance(); - const operations = [await operationFor(first.deploymentId), await operationFor(second.deploymentId)]; - expect(operations.filter(Boolean)).toHaveLength(1); - expect((await detail(second.deploymentId)).stage).toContain('capacity'); - saveFleetSettings({ ...fleetSettings(), [key]: 0 }); - await advance(); - expect(await operationFor(second.deploymentId)).not.toBeNull(); - }); + it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)( + 'holds simultaneous startups with %s until recovery completes', + async (key) => { + const first = await queue(); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); + saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 }); + const { queueCreateServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const second = queueCreateServer({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + hostId, + libraryId, + image, + plutoKey: 'second-key' + }); + if (!second.ok) throw new Error(second.message); + await advance(); + const operations = [ + await operationFor(first.deploymentId), + await operationFor(second.deploymentId) + ]; + expect(operations.filter(Boolean)).toHaveLength(1); + expect((await detail(second.deploymentId)).stage).toContain('capacity'); + saveFleetSettings({ ...fleetSettings(), [key]: 0 }); + await advance(); + expect(await operationFor(second.deploymentId)).not.toBeNull(); + } + ); }); describe('game key pool launch lifecycle', () => { it('reserves across maps atomically, rejects duplicate credentials, and reuses a cancelled launch key', async () => { await prepareLibrary(); - const { saveServerKey, serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { queueCreateServer, destroyServer, previewCreation } = await import('../../src/lib/server/provisioning/instances.js'); + const { saveServerKey, serverKeyPool, getServerKey } = + await import('../../src/lib/server/fleet/server-keys.js'); + const { queueCreateServer, destroyServer, previewCreation } = + await import('../../src/lib/server/provisioning/instances.js'); saveServerKey('t5', 'shared-t5-key', 'Any T5 map'); saveServerKey('t6', 'shared-t6-key'); - const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'moon', hostId, libraryId, image }; + const input = { + game: 't5' as const, + gameMode: 'zombie' as const, + mapSlug: 'moon', + hostId, + libraryId, + image + }; expect(previewCreation(input).hasSavedKey).toBe(true); const first = queueCreateServer(input); if (!first.ok) throw new Error(first.message); @@ -2222,7 +4026,9 @@ describe('game key pool launch lifecycle', () => { expect(serverKeyPool('t6')).toEqual({ total: 1, available: 1 }); expect(previewCreation({ ...input, mapSlug: 'theater' }).hasSavedKey).toBe(false); expect(queueCreateServer({ ...input, mapSlug: 'theater' }).ok).toBe(false); - expect(queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok).toBe(false); + expect( + queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok + ).toBe(false); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect(serverKeyPool('t5').available).toBe(0); @@ -2236,17 +4042,24 @@ describe('game key pool launch lifecycle', () => { }); it('keeps a key through stop and rebuild, then releases it only after confirmed removal', async () => { - const job = await queue(); await advance(); await acknowledge(job.deploymentId); - const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { serverKeyPool, getServerKey } = + await import('../../src/lib/server/fleet/server-keys.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); const stopped = requestLifecycle(job.name, 'stop'); if (!stopped.ok) throw new Error(stopped.error); - await advance(); remote.state = 'stopped'; await acknowledge(stopped.deploymentId); + await advance(); + remote.state = 'stopped'; + await acknowledge(stopped.deploymentId); expect(serverKeyPool('t5').available).toBe(0); const rebuilt = requestLifecycle(job.name, 'rebuild'); if (!rebuilt.ok) throw new Error(rebuilt.error); - await advance(); remote.state = 'running'; await acknowledge(rebuilt.deploymentId); + await advance(); + remote.state = 'running'; + await acknowledge(rebuilt.deploymentId); expect(getServerKey(job.name)).toBe('saved-key'); expect(serverKeyPool('t5').available).toBe(0); const removal = destroyServer(job.name); @@ -2264,9 +4077,11 @@ describe('game key pool launch lifecycle', () => { }); it('keeps the reservation for a submitted cancellation until the host confirms cleanup', async () => { - const job = await queue(); await advance(); + const job = await queue(); + await advance(); const { serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js'); - const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { cancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); const cleanup = cancelCreation(job.deploymentId)!; expect(cleanup).toEqual(expect.any(String)); expect(serverKeyPool('t5').available).toBe(0); diff --git a/tests/unit/native-recovery-cancellation.test.ts b/tests/unit/native-recovery-cancellation.test.ts index a2f6d2f..2705515 100644 --- a/tests/unit/native-recovery-cancellation.test.ts +++ b/tests/unit/native-recovery-cancellation.test.ts @@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) +})); let fixture: Fixture; const server = 't5_zombie_theater_1'; @@ -10,106 +13,214 @@ beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); async function recovering(step: 'restart' | 'rebuild') { const { host } = await prepareNativeHost([server]); const hosts = await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); - const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); - const { processNativeProvisioning, reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js'); - hosts.heartbeat(host.id, { ...hosts.getHost(host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] }); + const { beginDeployment, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); + const { cancelCreation, canCancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { processNativeProvisioning, reconcileNativeProvisioningReceipt } = + await import('../../src/lib/server/provisioning/native.js'); + hosts.heartbeat(host.id, { + ...hosts.getHost(host.id)!.inventory, + capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] + }); db().prepare('INSERT INTO servers(name) VALUES(?)').run(server); const instance = nativeInstance(server)!; hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'udp' }); hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'tcp' }); const deploymentId = beginDeployment(server, 'create'); - const original = hosts.queueHostOperation(host.id, 'instance.create', {}, { instanceId: instance.id, generation: 1 }); - hosts.reportHostOperation(host.id, original.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } }); - const recovery = hosts.queueHostOperation(host.id, `instance.${step}`, { unhealthyRecovery: true, ...(step === 'rebuild' ? { nextGeneration: 2 } : {}) }, - { instanceId: instance.id, generation: 1 }); - db().prepare('UPDATE native_provisioning SET operation_id=?,recovery_step=? WHERE deployment_id=?').run(recovery.id, step, deploymentId); - db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(deploymentId); + const original = hosts.queueHostOperation( + host.id, + 'instance.create', + {}, + { instanceId: instance.id, generation: 1 } + ); + hosts.reportHostOperation(host.id, original.id, { + state: 'completed', + result: { state: 'unhealthy', healthy: false } + }); + const recovery = hosts.queueHostOperation( + host.id, + `instance.${step}`, + { unhealthyRecovery: true, ...(step === 'rebuild' ? { nextGeneration: 2 } : {}) }, + { instanceId: instance.id, generation: 1 } + ); + db() + .prepare( + 'UPDATE native_provisioning SET operation_id=?,recovery_step=? WHERE deployment_id=?' + ) + .run(recovery.id, step, deploymentId); + db() + .prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?") + .run(deploymentId); const commandFor = (id: string) => { - const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(id) as { operation_id: string }; + const row = db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(id) as { operation_id: string }; return hosts.getHostOperation(row.operation_id)!; }; - return { host, instance, deploymentId, original, recovery, ...hosts, db, nativeInstance, cancelCreation, canCancelCreation, - getDeployment, reconcileNativeProvisioningReceipt, commandFor, advance: () => processNativeProvisioning(() => null) }; + return { + host, + instance, + deploymentId, + original, + recovery, + ...hosts, + db, + nativeInstance, + cancelCreation, + canCancelCreation, + getDeployment, + reconcileNativeProvisioningReceipt, + commandFor, + advance: () => processNativeProvisioning(() => null) + }; } describe('cancelling creation during unhealthy recovery', () => { it('keeps the original creation identity and suppresses recovery restart replay', async () => { const api = await recovering('restart'); - expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(api.recovery.id); + expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toContain( + api.recovery.id + ); const cleanup = api.cancelCreation(api.deploymentId)!; const removal = api.commandFor(cleanup); - expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1, payload: { createOperationId: api.original.id } }); + expect(removal).toMatchObject({ + kind: 'instance.cancel-create', + generation: 1, + payload: { createOperationId: api.original.id } + }); expect(removal.payload).toEqual({ createOperationId: api.original.id }); expect(api.pendingCreationCancellations(api.host.id)).toEqual([api.original.id]); - expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([removal.id]); - api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } }); + expect( + api + .pendingHostOperations(api.host.id) + .filter((operation) => operation.kind.startsWith('instance.')) + .map((operation) => operation.id) + ).toEqual([removal.id]); + api.reportHostOperation(api.host.id, removal.id, { + state: 'completed', + result: { exists: false } + }); await api.advance(); expect(api.getDeployment(cleanup)?.deployment.state).toBe('done'); expect(api.getHostOperation(api.recovery.id)?.state).toBe('failed'); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); }); - it.each(['pending', 'running', 'unknown', 'completed'] as const)('cleans up both rebuild generations when recovery is %s', async state => { - const api = await recovering('rebuild'); - if (state !== 'pending') { - api.pendingHostOperations(api.host.id); - api.reportHostOperation(api.host.id, api.recovery.id, { state, result: { state: 'unhealthy', healthy: false } }); + it.each(['pending', 'running', 'unknown', 'completed'] as const)( + 'cleans up both rebuild generations when recovery is %s', + async (state) => { + const api = await recovering('rebuild'); + if (state !== 'pending') { + api.pendingHostOperations(api.host.id); + api.reportHostOperation(api.host.id, api.recovery.id, { + state, + result: { state: 'unhealthy', healthy: false } + }); + } + if (state === 'completed') api.reconcileNativeProvisioningReceipt(api.deploymentId); + const registeredGeneration = state === 'completed' ? 2 : 1; + expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); + expect(api.canCancelCreation(api.deploymentId)).toBe(true); + const cleanup = api.cancelCreation(api.deploymentId)!; + const removal = api.commandFor(cleanup); + expect(removal).toMatchObject({ + kind: 'instance.cancel-create', + generation: 1, + payload: { + createOperationId: api.original.id, + recoveryRebuildOperationId: api.recovery.id, + recoveryGeneration: 2 + } + }); + expect(api.pendingCreationCancellations(api.host.id)).toEqual([ + api.original.id, + api.recovery.id + ]); + expect( + api + .db() + .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?') + .get(cleanup) + ).toEqual({ generation: registeredGeneration }); + expect( + api + .pendingHostOperations(api.host.id) + .filter((operation) => operation.kind.startsWith('instance.')) + .map((operation) => operation.id) + ).toEqual([removal.id]); + // A delivered rebuild may report success after cancellation, while the + // cleanup command already owns deletion of both possible containers. + api.reportHostOperation(api.host.id, api.recovery.id, { + state: 'completed', + result: { state: 'unhealthy', healthy: false } + }); + api.reconcileNativeProvisioningReceipt(api.deploymentId); + expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); + expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); + api.reportHostOperation(api.host.id, removal.id, { + state: 'completed', + result: { exists: false } + }); + await api.advance(); + expect(api.getDeployment(cleanup)?.deployment.state).toBe('done'); + expect(api.getDeployment(api.deploymentId)?.deployment).toMatchObject({ + state: 'cancelled', + server_deleted_at: expect.any(String) + }); + expect(api.nativeInstance(server)).toBeNull(); + expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); + expect( + api.db().prepare('SELECT name FROM servers WHERE name=?').get(server) + ).toBeUndefined(); + expect(api.pendingCreationCancellations(api.host.id)).toEqual([]); + expect(api.cancelCreation(api.deploymentId)).toBe(cleanup); } - if (state === 'completed') api.reconcileNativeProvisioningReceipt(api.deploymentId); - const registeredGeneration = state === 'completed' ? 2 : 1; - expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); - expect(api.canCancelCreation(api.deploymentId)).toBe(true); - const cleanup = api.cancelCreation(api.deploymentId)!; - const removal = api.commandFor(cleanup); - expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1, - payload: { createOperationId: api.original.id, recoveryRebuildOperationId: api.recovery.id, recoveryGeneration: 2 } }); - expect(api.pendingCreationCancellations(api.host.id)).toEqual([api.original.id, api.recovery.id]); - expect(api.db().prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?').get(cleanup)).toEqual({ generation: registeredGeneration }); - expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([removal.id]); - // A delivered rebuild may report success after cancellation, while the - // cleanup command already owns deletion of both possible containers. - api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } }); - api.reconcileNativeProvisioningReceipt(api.deploymentId); - expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); - expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); - api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } }); - await api.advance(); - expect(api.getDeployment(cleanup)?.deployment.state).toBe('done'); - expect(api.getDeployment(api.deploymentId)?.deployment).toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) }); - expect(api.nativeInstance(server)).toBeNull(); - expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); - expect(api.db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toBeUndefined(); - expect(api.pendingCreationCancellations(api.host.id)).toEqual([]); - expect(api.cancelCreation(api.deploymentId)).toBe(cleanup); - }); + ); it('does not adopt a changed generation without a completed recovery receipt', async () => { const api = await recovering('rebuild'); - api.db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(api.instance.id); + api.db() + .prepare('UPDATE native_instances SET generation=2 WHERE id=?') + .run(api.instance.id); expect(api.canCancelCreation(api.deploymentId)).toBe(false); expect(api.cancelCreation(api.deploymentId)).toBeNull(); api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed' }); - api.db().prepare('UPDATE native_instances SET generation=3 WHERE id=?').run(api.instance.id); + api.db() + .prepare('UPDATE native_instances SET generation=3 WHERE id=?') + .run(api.instance.id); expect(api.cancelCreation(api.deploymentId)).toBeNull(); }); it('holds extended cleanup until the host supports recovery cancellation', async () => { const api = await recovering('rebuild'); const cleanup = api.cancelCreation(api.deploymentId)!; - api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create'] }); + api.heartbeat(api.host.id, { + ...api.getHost(api.host.id)!.inventory, + capabilities: ['instance.cancel-create'] + }); expect(api.pendingHostOperations(api.host.id)).toEqual([]); await api.advance(); expect(api.getDeployment(cleanup)?.deployment.stage).toContain('Update the host agent'); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); - api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] }); - expect(api.pendingHostOperations(api.host.id).filter(operation => operation.kind.startsWith('instance.')).map(operation => operation.id)).toEqual([api.commandFor(cleanup).id]); + api.heartbeat(api.host.id, { + ...api.getHost(api.host.id)!.inventory, + capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] + }); + expect( + api + .pendingHostOperations(api.host.id) + .filter((operation) => operation.kind.startsWith('instance.')) + .map((operation) => operation.id) + ).toEqual([api.commandFor(cleanup).id]); }); }); diff --git a/tests/unit/native-registry-migration.test.ts b/tests/unit/native-registry-migration.test.ts index 6b72a44..5a40472 100644 --- a/tests/unit/native-registry-migration.test.ts +++ b/tests/unit/native-registry-migration.test.ts @@ -4,11 +4,19 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); it('retires only obsolete backend columns while preserving instances, history, keys and foreign keys', async () => { - const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8'); + const schema = await readFile( + new URL('../../src/lib/server/db/schema.sql', import.meta.url), + 'utf8' + ); const previous = new Database(fixture.dbPath); // Model the former required resource identity and an unrelated extension. previous.exec(schema); @@ -19,36 +27,67 @@ it('retires only obsolete backend columns while preserving instances, history, k ALTER TABLE servers ADD COLUMN komodo_name TEXT NOT NULL; ALTER TABLE servers ADD COLUMN komodo_server TEXT; ALTER TABLE servers ADD COLUMN operator_note TEXT;`); - previous.prepare(`INSERT INTO servers(name,komodo_name,komodo_server,game,game_port,operator_note) - VALUES('native-server','','old-host','t5',28960,'keep this')`).run(); - previous.prepare("INSERT INTO server_keys(server_id,pluto_key) VALUES('native-server','saved-key')").run(); - previous.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) - VALUES('creation','native-server','create','done','Ready','2026-01-01','fixture')`).run(); - previous.prepare(`INSERT INTO server_deployment_events(deployment_id,created_at,message) - VALUES('creation','2026-01-01','Original receipt')`).run(); + previous + .prepare( + `INSERT INTO servers(name,komodo_name,komodo_server,game,game_port,operator_note) + VALUES('native-server','','old-host','t5',28960,'keep this')` + ) + .run(); + previous + .prepare("INSERT INTO server_keys(server_id,pluto_key) VALUES('native-server','saved-key')") + .run(); + previous + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('creation','native-server','create','done','Ready','2026-01-01','fixture')` + ) + .run(); + previous + .prepare( + `INSERT INTO server_deployment_events(deployment_id,created_at,message) + VALUES('creation','2026-01-01','Original receipt')` + ) + .run(); const { migrateNativeInstances } = await import('../../src/lib/server/backends/schema.js'); migrateNativeInstances(previous); - previous.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('instance','native-server','host','library','image',28960,'2026-01-01')`).run(); + previous + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('instance','native-server','host','library','image',28960,'2026-01-01')` + ) + .run(); previous.close(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); for (let pass = 0; pass < 2; pass++) { const handle = db(); - const columns = (handle.prepare('PRAGMA table_info(servers)').all() as { name: string }[]).map(column => column.name); + const columns = ( + handle.prepare('PRAGMA table_info(servers)').all() as { name: string }[] + ).map((column) => column.name); expect(columns).not.toEqual(expect.arrayContaining(['komodo_type'])); expect(columns).not.toContain('komodo_name'); expect(columns).not.toContain('komodo_server'); expect(columns).toContain('operator_note'); - expect(handle.prepare('SELECT name,game_port,operator_note FROM servers WHERE name=?').get('native-server')) - .toEqual({ name: 'native-server', game_port: 28960, operator_note: 'keep this' }); - expect(handle.prepare('SELECT id,generation FROM native_instances').all()).toEqual([{ id: 'instance', generation: 1 }]); - expect(handle.prepare('SELECT pluto_key FROM server_keys').get()).toEqual({ pluto_key: 'saved-key' }); - expect(handle.prepare('SELECT message FROM server_deployment_events').get()).toEqual({ message: 'Original receipt' }); + expect( + handle + .prepare('SELECT name,game_port,operator_note FROM servers WHERE name=?') + .get('native-server') + ).toEqual({ name: 'native-server', game_port: 28960, operator_note: 'keep this' }); + expect(handle.prepare('SELECT id,generation FROM native_instances').all()).toEqual([ + { id: 'instance', generation: 1 } + ]); + expect(handle.prepare('SELECT pluto_key FROM server_keys').get()).toEqual({ + pluto_key: 'saved-key' + }); + expect(handle.prepare('SELECT message FROM server_deployment_events').get()).toEqual({ + message: 'Original receipt' + }); expect(handle.pragma('foreign_key_check')).toEqual([]); expect(handle.pragma('foreign_keys', { simple: true })).toBe(1); closeDb(); } // Native inserts must work without supplying the retired required field. - expect(() => db().prepare("INSERT INTO servers(name) VALUES('new-native-server')").run()).not.toThrow(); + expect(() => + db().prepare("INSERT INTO servers(name) VALUES('new-native-server')").run() + ).not.toThrow(); }); diff --git a/tests/unit/native-safety.test.ts b/tests/unit/native-safety.test.ts index 09ec41e..b1112a5 100644 --- a/tests/unit/native-safety.test.ts +++ b/tests/unit/native-safety.test.ts @@ -3,32 +3,64 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })), - rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) + rconStatusResult: vi.fn(async () => ({ + ok: true, + endpoint: { host: 'agent', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + })), + rconGetPlayerCount: vi.fn(async () => 0), + rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); -let fixture: Fixture; let hostId: string; let instanceId: string; +let fixture: Fixture; +let hostId: string; +let instanceId: string; const server = 't5_zombie_theater_1'; const healthy = { exists: true, healthy: true, state: 'running' }; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); + vi.resetModules(); + vi.clearAllMocks(); + fixture = await newFixture([]); - const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + const { createHost, createEnrollment, enrollHost } = + await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Safety test host' }).id; - enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, - storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }], capabilities: ['instance.safety-override'] }); + enrollHost(createEnrollment(hostId).token, { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100_000_000_000, + runtimeImages: ['plutainer:test'], + libraries: [{ id: 'published-game-version', state: 'ready' }], + capabilities: ['instance.safety-override'] + }); const { db } = await import('../../src/lib/server/db/index.js'); instanceId = randomUUID(); - db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`).run(instanceId, server, hostId, new Date().toISOString()); + db() + .prepare( + "INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')" + ) + .run(server); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)` + ) + .run(instanceId, server, hostId, new Date().toISOString()); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { if (kind !== 'instance.inspect') throw new Error(`Unexpected remote mutation: ${kind}`); return healthy as never; }); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); async function advance() { const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); @@ -36,34 +68,65 @@ async function advance() { } async function operationId(id: string) { const { db } = await import('../../src/lib/server/db/index.js'); - return (db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(id) as { operation_id: string | null }).operation_id; + return ( + db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(id) as { operation_id: string | null } + ).operation_id; } async function waitingRestart() { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const result = requestLifecycle(server, 'restart'); if (!result.ok) throw new Error(result.error); + const result = requestLifecycle(server, 'restart'); + if (!result.ok) throw new Error(result.error); return result.deploymentId; } async function suspendInspection() { const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); let release!: () => void; - vi.mocked(nativeRequest).mockImplementationOnce(async () => { await new Promise(resolve => { release = resolve; }); return healthy as never; }); + vi.mocked(nativeRequest).mockImplementationOnce(async () => { + await new Promise((resolve) => { + release = resolve; + }); + return healthy as never; + }); const running = advance(); await vi.waitFor(() => expect(release).toBeTypeOf('function')); return { running, release }; } async function stageCampaign() { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign-race','promote',?)").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')").run(server); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at) VALUES('campaign-race','promote',?)" + ) + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign-race',?,1,'pending')" + ) + .run(server); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); - queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('reviewed campaign bytes'), inventorySource: 'campaign:test' }); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'campaign-race', + server, + kind: 'script', + relPath: 'hud.gsc', + content: Buffer.from('reviewed campaign bytes'), + inventorySource: 'campaign:test' + }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const restart = requestRestart(server, { kind: 'campaign', id: 'campaign-race' }); await advance(); - const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; + const staged = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(restart.id) as { stage_operation_id: string }; expect(staged.stage_operation_id).toBeTruthy(); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed', result: { state: 'staged' } }); + reportHostOperation(hostId, staged.stage_operation_id, { + state: 'completed', + result: { state: 'staged' } + }); return restart.id; } @@ -71,92 +134,192 @@ describe('native disruption safety at dispatch', () => { it('lets fresh occupied telemetry veto an empty RCON result and resumes after that evidence expires', async () => { const id = await waitingRestart(); const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Still playing', guid: 'player-1' }], receivedAt: new Date().toISOString() }); + saveTelemetry({ + event: 'game_start', + server, + map: 'zombie_theater', + players: [{ name: 'Still playing', guid: 'player-1' }], + receivedAt: new Date().toISOString() + }); await advance(); expect(await operationId(id)).toBeNull(); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); - db().prepare('UPDATE server_telemetry SET received_at=? WHERE server=?').run('2000-01-01T00:00:00Z', server); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'" + ) + .get() + ).toEqual({ count: 0 }); + db() + .prepare('UPDATE server_telemetry SET received_at=? WHERE server=?') + .run('2000-01-01T00:00:00Z', server); await advance(); expect(await operationId(id)).toBeTypeOf('string'); }); it('holds destruction if a beta becomes active while remote inspection is pending', async () => { const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); - const removal = destroyServer(server); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued.'); + const removal = destroyServer(server); + if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued.'); const pending = await suspendInspection(); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { script, version } = createScript({ name: 'Race beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('main() {}'), scopeType: 'game', scopeValue: 't5' }); + const { script, version } = createScript({ + name: 'Race beta', + kind: 'script', + targetRelPath: 'beta.gsc', + content: Buffer.from('main() {}'), + scopeType: 'game', + scopeValue: 't5' + }); const { db } = await import('../../src/lib/server/db/index.js'); - const betaId = db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id, version.id).lastInsertRowid; - db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId, server); - pending.release(); await pending.running; + const betaId = db() + .prepare( + "INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')" + ) + .run(script.id, version.id).lastInsertRowid; + db() + .prepare( + "INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')" + ) + .run(betaId, server); + pending.release(); + await pending.running; expect(await operationId(removal.deploymentId)).toBeNull(); - expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server }); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.remove'").get()).toEqual({ count: 0 }); + expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ + name: server + }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.remove'" + ) + .get() + ).toEqual({ count: 0 }); }); - it.each(['cancel', 'change bytes'] as const)('does not activate staged files after a %s during remote inspection', async change => { - const id = await stageCampaign(); - const pending = await suspendInspection(); - const { cancelStagedFiles, queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); - if (change === 'cancel') cancelStagedFiles('campaign', 'campaign-race', server); - else queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign-race', server, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('different unreviewed bytes'), inventorySource: 'campaign:test' }); - pending.release(); await pending.running; - expect(await operationId(id)).toBeNull(); - expect(stagedFileChanges('campaign', 'campaign-race')[0].state).toBe(change === 'cancel' ? 'cancelled' : 'pending'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); - }); + it.each(['cancel', 'change bytes'] as const)( + 'does not activate staged files after a %s during remote inspection', + async (change) => { + const id = await stageCampaign(); + const pending = await suspendInspection(); + const { cancelStagedFiles, queueFileChange, stagedFileChanges } = + await import('../../src/lib/server/fleet/staged-files.js'); + if (change === 'cancel') cancelStagedFiles('campaign', 'campaign-race', server); + else + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'campaign-race', + server, + kind: 'script', + relPath: 'hud.gsc', + content: Buffer.from('different unreviewed bytes'), + inventorySource: 'campaign:test' + }); + pending.release(); + await pending.running; + expect(await operationId(id)).toBeNull(); + expect(stagedFileChanges('campaign', 'campaign-race')[0].state).toBe( + change === 'cancel' ? 'cancelled' : 'pending' + ); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'" + ) + .get() + ).toEqual({ count: 0 }); + } + ); it('does not dispatch an old generation if the instance changes during inspection', async () => { const id = await waitingRestart(); const pending = await suspendInspection(); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); - pending.release(); await pending.running; + pending.release(); + await pending.running; expect(await operationId(id)).toBeNull(); - expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); + expect( + db() + .prepare( + "SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'" + ) + .get() + ).toEqual({ count: 0 }); }); }); describe('provisioning admin override', () => { const actor = { id: 'admin-id', username: 'recovery-admin' }; async function confirm(id: string) { - const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js'); + const { enableSafetyOverride } = + await import('../../src/lib/server/provisioning/safety-override.js'); expect(enableSafetyOverride(id, actor)).toEqual({ ok: true }); } - it.each(['restart', 'stop', 'destroy'] as const)('dispatches %s with unavailable player evidence only after confirmation', async action => { - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); - const result = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action); - if (!result.ok || !result.deploymentId) throw new Error('Request failed'); - const id = result.deploymentId; - const status = await import('../../src/lib/server/fleet/status-evidence.js'); - vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() }); - await advance(); - expect(await operationId(id)).toBeNull(); - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - expect(getDeployment(id)).toMatchObject({ canOverrideSafety: true, safetyOverride: null }); - await confirm(id); - await confirm(id); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); // Approval and attribution survive controller restart. - await advance(); - const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - const commandId = (await operationId(id))!; - expect(getHostOperation(commandId)).toMatchObject({ kind: `instance.${action === 'destroy' ? 'remove' : action}`, payload: { safetyOverride: true } }); - expect(getDeployment(id)).toMatchObject({ canOverrideSafety: false, safetyOverride: { actorId: actor.id, actorName: actor.username } }); - expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'").get(id)).toEqual({ n: 1 }); - await advance(); - expect(await operationId(id)).toBe(commandId); - }); + it.each(['restart', 'stop', 'destroy'] as const)( + 'dispatches %s with unavailable player evidence only after confirmation', + async (action) => { + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + const { destroyServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const result = + action === 'destroy' ? destroyServer(server) : requestLifecycle(server, action); + if (!result.ok || !result.deploymentId) throw new Error('Request failed'); + const id = result.deploymentId; + const status = await import('../../src/lib/server/fleet/status-evidence.js'); + vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ + players: null, + source: null, + observedAt: new Date().toISOString() + }); + await advance(); + expect(await operationId(id)).toBeNull(); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + expect(getDeployment(id)).toMatchObject({ + canOverrideSafety: true, + safetyOverride: null + }); + await confirm(id); + await confirm(id); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); // Approval and attribution survive controller restart. + await advance(); + const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const commandId = (await operationId(id))!; + expect(getHostOperation(commandId)).toMatchObject({ + kind: `instance.${action === 'destroy' ? 'remove' : action}`, + payload: { safetyOverride: true } + }); + expect(getDeployment(id)).toMatchObject({ + canOverrideSafety: false, + safetyOverride: { actorId: actor.id, actorName: actor.username } + }); + expect( + db() + .prepare( + "SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='safety-override'" + ) + .get(id) + ).toEqual({ n: 1 }); + await advance(); + expect(await operationId(id)).toBe(commandId); + } + ); it('bypasses occupied telemetry while retaining health verification after the restart', async () => { const id = await waitingRestart(); const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - saveTelemetry({ event: 'game_start', server, map: 'zombie_theater', players: [{ name: 'Playing', guid: '1' }], receivedAt: new Date().toISOString() }); + saveTelemetry({ + event: 'game_start', + server, + map: 'zombie_theater', + players: [{ name: 'Playing', guid: '1' }], + receivedAt: new Date().toISOString() + }); await advance(); expect(await operationId(id)).toBeNull(); await confirm(id); @@ -168,20 +331,26 @@ describe('provisioning admin override', () => { await advance(); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); expect(getDeployment(id)?.deployment.state).toBe('awaiting-health'); - expect(getDeployment(id)?.progress?.checks).toContainEqual(expect.objectContaining({ key: 'container', result: 'waiting' })); + expect(getDeployment(id)?.progress?.checks).toContainEqual( + expect.objectContaining({ key: 'container', result: 'waiting' }) + ); }); it('retains recovery capacity limits and cancellation after approval', async () => { const id = await waitingRestart(); await confirm(id); const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 }); db().prepare("INSERT INTO servers(name) VALUES('other')").run(); - db().prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)").run(new Date().toISOString()); + db() + .prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('other',?)") + .run(new Date().toISOString()); await advance(); expect(await operationId(id)).toBeNull(); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(id)).toBe(true); db().prepare("DELETE FROM restart_rollout WHERE server='other'").run(); await advance(); @@ -191,16 +360,37 @@ describe('provisioning admin override', () => { it('rejects old agents, stale instances, unsupported actions and submitted or terminal operations', async () => { const id = await waitingRestart(); const { db } = await import('../../src/lib/server/db/index.js'); - const { enableSafetyOverride, safetyOverrideAvailability } = await import('../../src/lib/server/provisioning/safety-override.js'); - db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?").run(hostId); - expect(enableSafetyOverride(id, actor)).toMatchObject({ ok: false, error: expect.stringContaining('Update') }); - for (const state of ['running', 'awaiting-health', 'failed', 'interrupted', 'cancelled', 'done']) { + const { enableSafetyOverride, safetyOverrideAvailability } = + await import('../../src/lib/server/provisioning/safety-override.js'); + db() + .prepare( + "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json('[]')) WHERE id=?" + ) + .run(hostId); + expect(enableSafetyOverride(id, actor)).toMatchObject({ + ok: false, + error: expect.stringContaining('Update') + }); + for (const state of [ + 'running', + 'awaiting-health', + 'failed', + 'interrupted', + 'cancelled', + 'done' + ]) { db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, id); expect(safetyOverrideAvailability(id).applicable).toBe(false); } - db().prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?").run(id); + db() + .prepare("UPDATE server_deployments SET state='waiting',action='start' WHERE id=?") + .run(id); expect(safetyOverrideAvailability(id).applicable).toBe(false); - db().prepare("UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?").run(id); + db() + .prepare( + "UPDATE server_deployments SET action='restart',submitted_at='2026-10-05' WHERE id=?" + ) + .run(id); expect(safetyOverrideAvailability(id).applicable).toBe(false); db().prepare('UPDATE server_deployments SET submitted_at=NULL WHERE id=?').run(id); db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); @@ -212,37 +402,61 @@ describe('provisioning admin override', () => { const id = await waitingRestart(); const pending = await suspendInspection(); await confirm(id); - pending.release(); await pending.running; + pending.release(); + await pending.running; expect(await operationId(id)).toBeNull(); await advance(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ safetyOverride: true }); + expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ + safetyOverride: true + }); }); it('overrides a waiting campaign restart while keeping its staged changes attached', async () => { const id = await stageCampaign(); const status = await import('../../src/lib/server/fleet/status-evidence.js'); - vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ players: null, source: null, observedAt: new Date().toISOString() }); + vi.spyOn(status, 'probePlayerEvidence').mockResolvedValue({ + players: null, + source: null, + observedAt: new Date().toISOString() + }); await advance(); expect(await operationId(id)).toBeNull(); await confirm(id); await advance(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); - expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ stageId: id, safetyOverride: true }); + expect(getHostOperation((await operationId(id))!)?.payload).toMatchObject({ + stageId: id, + safetyOverride: true + }); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'").get()).toEqual({ state: 'applying' }); + expect( + db() + .prepare("SELECT state FROM staged_file_changes WHERE source_id='campaign-race'") + .get() + ).toEqual({ state: 'applying' }); }); it('requires an administrator and explicit confirmation at the API boundary', async () => { const id = await waitingRestart(); const { POST } = await import('../../src/routes/api/provisioning/[id]/override/+server.js'); - const request = (role: string | null, confirmed: unknown) => POST({ params: { id }, locals: { user: role ? { ...actor, role } : null }, - request: new Request(`http://localhost/api/provisioning/${id}/override`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ confirmed }) }) - } as Parameters[0]); - for (const role of [null, 'viewer', 'operator']) expect((await request(role, true)).status).toBe(403); - for (const value of [undefined, false, 'true', 1]) expect((await request('admin', value)).status).toBe(400); + const request = (role: string | null, confirmed: unknown) => + POST({ + params: { id }, + locals: { user: role ? { ...actor, role } : null }, + request: new Request(`http://localhost/api/provisioning/${id}/override`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ confirmed }) + }) + } as Parameters[0]); + for (const role of [null, 'viewer', 'operator']) + expect((await request(role, true)).status).toBe(403); + for (const value of [undefined, false, 'true', 1]) + expect((await request('admin', value)).status).toBe(400); expect((await request('admin', true)).status).toBe(202); - const { safetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js'); + const { safetyOverride } = + await import('../../src/lib/server/provisioning/safety-override.js'); expect(safetyOverride(id)?.actorId).toBe(actor.id); }); }); @@ -253,20 +467,34 @@ describe('native scheduler ownership', () => { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); db().prepare("INSERT INTO servers(name) VALUES('unplaced-registry')").run(); - expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({ ok: false, status: 503 }); + expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({ + ok: false, + status: 503 + }); expect(destroyServer('unplaced-registry')).toMatchObject({ ok: false, kind: 'conflict' }); - expect(db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all()).toEqual([]); + expect( + db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all() + ).toEqual([]); }); it('shares restart capacity until an agent receipt and fresh health release the first slot', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 }); - db().prepare("INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)").run(); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?`).run(instanceId); + db() + .prepare( + "INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)" + ) + .run(); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?` + ) + .run(instanceId); const first = await waitingRestart(); const second = requestLifecycle('second', 'restart'); if (!second.ok) throw new Error(second.error); @@ -280,22 +508,40 @@ describe('native scheduler ownership', () => { reportHostOperation(hostId, command!, { state: 'completed', result: healthy }); await advance(); await advance(); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({ state: 'done' }); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({ + state: 'done' + }); expect(await operationId(second.deploymentId)).toEqual(expect.any(String)); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toHaveLength(2); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all() + ).toHaveLength(2); }); it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => { const id = await stageCampaign(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'").run(); + db() + .prepare( + "UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'" + ) + .run(); await advance(); expect(await operationId(id)).toBeNull(); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'waiting' }); - db().prepare("UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'").run(); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ + state: 'waiting' + }); + db() + .prepare( + "UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'" + ) + .run(); await advance(); expect(await operationId(id)).toBeNull(); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' }); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ + state: 'cancelled' + }); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all() + ).toEqual([]); }); }); diff --git a/tests/unit/navigation-context.test.ts b/tests/unit/navigation-context.test.ts index 4935104..0867bce 100644 --- a/tests/unit/navigation-context.test.ts +++ b/tests/unit/navigation-context.test.ts @@ -3,9 +3,13 @@ import { contextHref, returnLabel, safeReturnHref } from '../../src/lib/navigati describe('navigation context', () => { it('retains nested origins, list scope, loaded results and anchors', () => { - const source = new URL('http://localhost/leaderboard/players/42?game=t5&offset=25&returnTo=%2Fleaderboard%3Fq%3DAce#history-title'); + const source = new URL( + 'http://localhost/leaderboard/players/42?game=t5&offset=25&returnTo=%2Fleaderboard%3Fq%3DAce#history-title' + ); const match = new URL(contextHref('/matches/123', source), source); - expect(safeReturnHref(match.searchParams.get('returnTo'), '/matches')).toBe(source.pathname + source.search + source.hash); + expect(safeReturnHref(match.searchParams.get('returnTo'), '/matches')).toBe( + source.pathname + source.search + source.hash + ); expect(returnLabel(match.searchParams.get('returnTo')!)).toBe('Back to player'); expect(returnLabel('/collections?view=history&launches=30')).toBe('Back to launch history'); }); @@ -18,13 +22,31 @@ describe('navigation context', () => { }); it('preserves the selected diagnostic tab on both sides of a contextual link', () => { - const url = new URL(contextHref('/servers/t5_zombie_theater_1?tab=logs&log=console', '/maintenance/incident?returnTo=%2Fmaintenance%3Fview%3Dactivity'), 'http://localhost'); + const url = new URL( + contextHref( + '/servers/t5_zombie_theater_1?tab=logs&log=console', + '/maintenance/incident?returnTo=%2Fmaintenance%3Fview%3Dactivity' + ), + 'http://localhost' + ); expect(url.searchParams.get('tab')).toBe('logs'); expect(url.searchParams.get('log')).toBe('console'); - expect(url.searchParams.get('returnTo')).toBe('/maintenance/incident?returnTo=%2Fmaintenance%3Fview%3Dactivity'); + expect(url.searchParams.get('returnTo')).toBe( + '/maintenance/incident?returnTo=%2Fmaintenance%3Fview%3Dactivity' + ); }); - it.each(['//evil.test', 'https://evil.test/', '/api/auth/logout', '/login', '/hosts/../../api/hosts', '/\\evil.test', '/hosts\n/42', '/unknown', '/' + 'x'.repeat(12000)])('rejects unsafe or unsupported return paths: %s', value => { + it.each([ + '//evil.test', + 'https://evil.test/', + '/api/auth/logout', + '/login', + '/hosts/../../api/hosts', + '/\\evil.test', + '/hosts\n/42', + '/unknown', + '/' + 'x'.repeat(12000) + ])('rejects unsafe or unsupported return paths: %s', (value) => { expect(safeReturnHref(value, '/betas')).toBe('/betas'); }); }); diff --git a/tests/unit/navigation.test.ts b/tests/unit/navigation.test.ts index 8bed1a5..dd3d752 100644 --- a/tests/unit/navigation.test.ts +++ b/tests/unit/navigation.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { navigationGroups, navigationItemActive, navigationLocation, visibleNavigation } from '../../src/lib/navigation.js'; +import { + navigationGroups, + navigationItemActive, + navigationLocation, + visibleNavigation +} from '../../src/lib/navigation.js'; describe('application navigation', () => { it.each([ @@ -19,26 +24,53 @@ describe('application navigation', () => { ['/server-keys', 'admin', '/server-keys'] ])('gives %s exactly one canonical home', (path, group, href) => { expect(navigationLocation(path, 'admin')?.group.id).toBe(group); - expect(navigationGroups.flatMap(group => group.items).filter(item => navigationItemActive(item, path)).map(item => item.href)).toEqual([href]); + expect( + navigationGroups + .flatMap((group) => group.items) + .filter((item) => navigationItemActive(item, path)) + .map((item) => item.href) + ).toEqual([href]); }); it('keeps Operations reachable from Activity and redirects legacy overview links', async () => { - expect(navigationGroups.find(group => group.id === 'activity')?.items.map(item => item.label)).toEqual(['Activity log', 'Provisioning', 'Maintenance']); + expect( + navigationGroups + .find((group) => group.id === 'activity') + ?.items.map((item) => item.label) + ).toEqual(['Activity log', 'Provisioning', 'Maintenance']); const { load } = await import('../../src/routes/operations/+page.server.js'); expect(load).toThrow(expect.objectContaining({ status: 308, location: '/activity' })); }); it('keeps administration and its shortcuts out of operator and viewer navigation', () => { for (const role of ['operator', 'viewer'] as const) { - expect(visibleNavigation(role).map(group => group.id)).not.toContain('admin'); + expect(visibleNavigation(role).map((group) => group.id)).not.toContain('admin'); expect(navigationLocation('/server-keys', role)).toBeNull(); - expect(visibleNavigation(role).flatMap(group => group.items).filter(item => ['h', 'k'].includes(item.shortcut ?? ''))).toEqual([]); + expect( + visibleNavigation(role) + .flatMap((group) => group.items) + .filter((item) => ['h', 'k'].includes(item.shortcut ?? '')) + ).toEqual([]); } }); it('groups file releases together and retains Campaigns as its own destination', () => { - expect(navigationGroups.map(group => group.label)).toEqual(['Fleet', 'Releases', 'Activity', 'Players', 'Admin']); - expect(navigationGroups.find(group => group.id === 'releases')?.items.map(item => item.label)).toEqual(['Scripts', 'Configs', 'Plugins', 'Game files', 'Betas', 'Campaigns']); - expect(navigationGroups.flatMap(group => group.items).some(item => /rollout/i.test(item.label))).toBe(false); + expect(navigationGroups.map((group) => group.label)).toEqual([ + 'Fleet', + 'Releases', + 'Activity', + 'Players', + 'Admin' + ]); + expect( + navigationGroups + .find((group) => group.id === 'releases') + ?.items.map((item) => item.label) + ).toEqual(['Scripts', 'Configs', 'Plugins', 'Game files', 'Betas', 'Campaigns']); + expect( + navigationGroups + .flatMap((group) => group.items) + .some((item) => /rollout/i.test(item.label)) + ).toBe(false); }); }); diff --git a/tests/unit/operation-context.test.ts b/tests/unit/operation-context.test.ts index 4f18afa..e4d9d49 100644 --- a/tests/unit/operation-context.test.ts +++ b/tests/unit/operation-context.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { currentOperationSource, withOperationSource } from '../../src/lib/server/hosts/operation-context.js'; +import { + currentOperationSource, + withOperationSource +} from '../../src/lib/server/hosts/operation-context.js'; import { trackActivity, workerActivity } from '../../src/lib/server/pipeline/activity.js'; describe('host operation source context', () => { it('isolates overlapping asynchronous sources and restores the caller', async () => { let release!: () => void; - const barrier = new Promise(resolve => { release = resolve; }); + const barrier = new Promise((resolve) => { + release = resolve; + }); const first = withOperationSource('GET /servers/[name]', async () => { await barrier; return currentOperationSource(); @@ -16,7 +21,10 @@ describe('host operation source context', () => { return currentOperationSource(); }); expect(currentOperationSource()).toBeNull(); - expect(await Promise.all([first, second])).toEqual(['GET /servers/[name]', 'Status monitor']); + expect(await Promise.all([first, second])).toEqual([ + 'GET /servers/[name]', + 'Status monitor' + ]); expect(currentOperationSource()).toBeNull(); }); @@ -27,7 +35,11 @@ describe('host operation source context', () => { expect(currentOperationSource()).toBe('Status monitor'); }); expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh'); - expect(() => withOperationSource('Failing task', () => { throw new Error('failed'); })).toThrow('failed'); + expect(() => + withOperationSource('Failing task', () => { + throw new Error('failed'); + }) + ).toThrow('failed'); expect(currentOperationSource()).toBe('POST /api/servers/[name]/refresh'); }); expect(currentOperationSource()).toBeNull(); @@ -44,18 +56,21 @@ describe('host operation source context', () => { await withOperationSource('POST /api/servers/[name]/restart', () => trackActivity('context-test-request', 'Processing provisioning requests', async () => { expect(currentOperationSource()).toBe('POST /api/servers/[name]/restart'); - })); + }) + ); expect(currentOperationSource()).toBeNull(); expect(workerActivity()).toEqual([]); }); it('restores context and clears activity when asynchronous work rejects', async () => { await withOperationSource('Manual maintenance check', async () => { - await expect(trackActivity('context-test-error', 'Waiting for files.read', async () => { - await Promise.resolve(); - expect(currentOperationSource()).toBe('Manual maintenance check'); - throw new Error('unavailable'); - })).rejects.toThrow('unavailable'); + await expect( + trackActivity('context-test-error', 'Waiting for files.read', async () => { + await Promise.resolve(); + expect(currentOperationSource()).toBe('Manual maintenance check'); + throw new Error('unavailable'); + }) + ).rejects.toThrow('unavailable'); expect(currentOperationSource()).toBe('Manual maintenance check'); }); expect(currentOperationSource()).toBeNull(); diff --git a/tests/unit/operation-history.test.ts b/tests/unit/operation-history.test.ts index aee5152..abccfdf 100644 --- a/tests/unit/operation-history.test.ts +++ b/tests/unit/operation-history.test.ts @@ -3,25 +3,62 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { betaFlow, campaignFlow } from '../../src/lib/operation-flow.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture(); +}); +afterEach(async () => { + await fixture.cleanup(); +}); describe('persistent activity', () => { it('keeps beta failures and retries across reopening without duplicating polls', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const handle = db(); handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); - handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run(); - handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 0, NULL)").run(); - handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run(); - handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'deploying')").run(); - handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed' WHERE beta_id = 1").run(); - handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed', updated_at = 'later' WHERE beta_id = 1").run(); - handle.prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL WHERE beta_id = 1").run(); + handle + .prepare( + "INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')" + ) + .run(); + handle + .prepare( + "INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 0, NULL)" + ) + .run(); + handle + .prepare( + "INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')" + ) + .run(); + handle + .prepare( + "INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'deploying')" + ) + .run(); + handle + .prepare( + "UPDATE beta_servers SET stage = 'failed', error = 'write failed' WHERE beta_id = 1" + ) + .run(); + handle + .prepare( + "UPDATE beta_servers SET stage = 'failed', error = 'write failed', updated_at = 'later' WHERE beta_id = 1" + ) + .run(); + handle + .prepare( + "UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL WHERE beta_id = 1" + ) + .run(); handle.prepare("UPDATE betas SET state = 'rolled-back' WHERE id = 1").run(); const before = handle.prepare('SELECT * FROM beta_events ORDER BY id').all(); expect(before).toHaveLength(5); - expect(before).toEqual(expect.arrayContaining([expect.objectContaining({ message: 'server: Failed — write failed' })])); + expect(before).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: 'server: Failed — write failed' }) + ]) + ); closeDb(); expect(db().prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(before); }); @@ -30,15 +67,33 @@ describe('persistent activity', () => { const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); - handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')").run(); - handle.prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')").run(); - handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')").run(); - handle.prepare("UPDATE deployment_servers SET attempts = attempts + 1, checked_at = 'later'").run(); + handle + .prepare( + "INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')" + ) + .run(); + handle + .prepare( + "INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')" + ) + .run(); + handle + .prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')" + ) + .run(); + handle + .prepare("UPDATE deployment_servers SET attempts = attempts + 1, checked_at = 'later'") + .run(); expect(handle.prepare('SELECT * FROM campaign_events').all()).toHaveLength(3); handle.prepare("UPDATE deployment_servers SET restarted_at = '2026-09-17T00:01:00Z'").run(); handle.prepare("UPDATE deployment_servers SET restart_state = 'done'").run(); handle.prepare("UPDATE deployments SET finished_at = '2026-09-17T00:02:00Z'").run(); - const messages = (handle.prepare('SELECT message FROM campaign_events ORDER BY id').all() as { message: string }[]).map(e => e.message); + const messages = ( + handle.prepare('SELECT message FROM campaign_events ORDER BY id').all() as { + message: string; + }[] + ).map((e) => e.message); expect(messages).toHaveLength(6); expect(messages).toContain('server: Restart requested; verifying health'); expect(messages).toContain('server: Server complete'); @@ -48,26 +103,75 @@ describe('persistent activity', () => { }); describe('operation progress', () => { - const server = { stage: 'awaiting-empty', deployed_at: 'now', restarted_at: null, healthy_at: null, match_seen_at: null }; + const server = { + stage: 'awaiting-empty', + deployed_at: 'now', + restarted_at: null, + healthy_at: null, + match_seen_at: null + }; it('uses the slowest beta server and omits disabled gates', () => { - const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [server, { ...server, stage: 'ready' }]); - expect(flow.map(s => s.label)).toEqual(['Stage file changes', 'Wait for empty servers', 'Apply files and restart', 'Review and promote']); - expect(flow.map(s => s.state)).toEqual(['done', 'current', 'pending', 'pending']); + const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [ + server, + { ...server, stage: 'ready' } + ]); + expect(flow.map((s) => s.label)).toEqual([ + 'Stage file changes', + 'Wait for empty servers', + 'Apply files and restart', + 'Review and promote' + ]); + expect(flow.map((s) => s.state)).toEqual(['done', 'current', 'pending', 'pending']); }); it('does not mark a failed deployment as completed', () => { - expect(betaFlow({ state: 'running', wait_for_health: 1, wait_for_match: 1 }, [{ ...server, stage: 'failed', deployed_at: null }])[0].state).toBe('failed'); - expect(campaignFlow([{ restart_state: 'failed', files_changed: 0, restarted_at: null }], false)[0].state).toBe('failed'); + expect( + betaFlow({ state: 'running', wait_for_health: 1, wait_for_match: 1 }, [ + { ...server, stage: 'failed', deployed_at: null } + ])[0].state + ).toBe('failed'); + expect( + campaignFlow( + [{ restart_state: 'failed', files_changed: 0, restarted_at: null }], + false + )[0].state + ).toBe('failed'); }); it('requires health verification after campaign restart and preserves manual work', () => { - expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'done', 'current']); - expect(campaignFlow([{ restart_state: 'manual', files_changed: 1, restarted_at: null }], true).some(s => s.state === 'current')).toBe(false); - expect(campaignFlow([{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], true).every(s => s.state === 'done')).toBe(true); + expect( + campaignFlow( + [{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], + false + ).map((s) => s.state) + ).toEqual(['done', 'done', 'done', 'done', 'current']); + expect( + campaignFlow( + [{ restart_state: 'manual', files_changed: 1, restarted_at: null }], + true + ).some((s) => s.state === 'current') + ).toBe(false); + expect( + campaignFlow( + [{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], + true + ).every((s) => s.state === 'done') + ).toBe(true); }); it('shows snapshot confirmation before file application and restart', () => { - const servers = [{ server: 'test', restart_state: 'pending', files_changed: 1, restarted_at: null }]; + const servers = [ + { server: 'test', restart_state: 'pending', files_changed: 1, restarted_at: null } + ]; const files = [{ server: 'test', snapshot_pending: 1, apply_state: 'applying' }]; - expect(campaignFlow(servers, false, files)[1]).toEqual({ label: 'Capture rollback snapshots', state: 'current' }); - expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0 }])[2].state).toBe('current'); - expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0, apply_state: 'applied' }])[3].state).toBe('current'); + expect(campaignFlow(servers, false, files)[1]).toEqual({ + label: 'Capture rollback snapshots', + state: 'current' + }); + expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0 }])[2].state).toBe( + 'current' + ); + expect( + campaignFlow(servers, false, [ + { ...files[0], snapshot_pending: 0, apply_state: 'applied' } + ])[3].state + ).toBe('current'); }); }); diff --git a/tests/unit/player-profile-scope.test.ts b/tests/unit/player-profile-scope.test.ts index 65b84e4..5fa76b0 100644 --- a/tests/unit/player-profile-scope.test.ts +++ b/tests/unit/player-profile-scope.test.ts @@ -3,22 +3,40 @@ import type Database from 'better-sqlite3'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); afterEach(async () => fixture.cleanup()); -function addMatches(handle: Database.Database, guid: string, count: number, - { game = 't5', map = 'theater', kills = 10, name = guid } = {}) { - handle.prepare(`INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) - VALUES (?, ?, ?, '2026-09-01', '2026-09-02')`).run(game, guid, name); +function addMatches( + handle: Database.Database, + guid: string, + count: number, + { game = 't5', map = 'theater', kills = 10, name = guid } = {} +) { + handle + .prepare( + `INSERT OR IGNORE INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at) + VALUES (?, ?, ?, '2026-09-01', '2026-09-02')` + ) + .run(game, guid, name); for (let i = 0; i < count; i++) { - const inserted = handle.prepare(`INSERT INTO leaderboard_matches + const inserted = handle + .prepare( + `INSERT INTO leaderboard_matches (server, game, map, map_key, map_name, started_at, ended_at, last_received_at, status, highest_round, last_event, last_fingerprint) - VALUES ('removed-server', ?, ?, ?, ?, '2026-09-01', '2026-09-02', '2026-09-02', 'completed', 20, 'game_end', 'test')`) + VALUES ('removed-server', ?, ?, ?, ?, '2026-09-01', '2026-09-02', '2026-09-02', 'completed', 20, 'game_end', 'test')` + ) .run(game, map, `${game}:${map}`, map); - handle.prepare(`INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, headshots, + handle + .prepare( + `INSERT INTO leaderboard_match_players (match_id, guid, highest_round, kills, headshots, downs, rounds_survived, completed, has_stats, first_seen_at, last_seen_at) - VALUES (?, ?, 20, ?, 5, 1, 2, 1, 1, '2026-09-01', '2026-09-02')`).run(inserted.lastInsertRowid, guid, kills); + VALUES (?, ?, 20, ?, 5, 1, 2, 1, 1, '2026-09-01', '2026-09-02')` + ) + .run(inserted.lastInsertRowid, guid, kills); } } @@ -34,19 +52,28 @@ it('filters and paginates match history independently of lifetime stats, map rec expect(first.matchesTotal).toBe(30); expect(first.matches).toHaveLength(25); expect(second.matches).toHaveLength(5); - expect(new Set([...first.matches, ...second.matches].map(match => match.id)).size).toBe(30); - expect(first.matches.every(match => match.game === 't5' && match.mapName === 'theater')).toBe(true); + expect(new Set([...first.matches, ...second.matches].map((match) => match.id)).size).toBe(30); + expect(first.matches.every((match) => match.game === 't5' && match.mapName === 'theater')).toBe( + true + ); for (const profile of [first, second, getPlayerProfile('101', 0, { game: 't6' })!]) { expect(profile.stats).toEqual(lifetime.stats); expect(profile.maps).toEqual(lifetime.maps); expect(profile.records).toEqual(lifetime.records); } expect(getPlayerProfile('101', 999, { game: 't5' })!.offset).toBe(25); - expect(getPlayerProfile('101', 999, { game: 't6' })).toMatchObject({ offset: 0, matchesTotal: 3 }); - expect(getPlayerProfile('101', 999, { game: 't6', map: 't5:theater' })).toMatchObject({ - stats: lifetime.stats, matches: [], matchesTotal: 0, offset: 0 + expect(getPlayerProfile('101', 999, { game: 't6' })).toMatchObject({ + offset: 0, + matchesTotal: 3 }); - for (const offset of [-1, NaN, 1.5, Infinity]) expect(getPlayerProfile('101', offset, { game: 't5' })!.offset).toBe(0); + expect(getPlayerProfile('101', 999, { game: 't6', map: 't5:theater' })).toMatchObject({ + stats: lifetime.stats, + matches: [], + matchesTotal: 0, + offset: 0 + }); + for (const offset of [-1, NaN, 1.5, Infinity]) + expect(getPlayerProfile('101', offset, { game: 't5' })!.offset).toBe(0); }); it('finds the exact GUID after combined or scoped ranking even beyond the first hundred name and GUID collisions', async () => { @@ -57,14 +84,34 @@ it('finds the exact GUID after combined or scoped ranking even beyond the first for (let index = 0; index < 105; index++) { addMatches(handle, `101-${index}`, 1, { kills: 100, name: '101 namesake' }); } - const { getPlayerLeaderboardEntry, getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); - expect(getLeaderboard({ q: '101', metric: 'kills', limit: 100 }).entries.some(entry => entry.guid === '101')).toBe(false); - expect(getPlayerLeaderboardEntry('101', { metric: 'kills' })).toMatchObject({ guid: '101', rank: 106, - gamesPlayed: 2, kills: 30, records: { bestKills: { game: 't6', value: 20 } } }); - expect(getPlayerLeaderboardEntry('101', { metric: 'kills', game: 't5' })).toMatchObject({ guid: '101', rank: 106, - gamesPlayed: 1, kills: 10, records: { bestKills: { game: 't5', value: 10 } } }); - expect(getPlayerLeaderboardEntry('101', { metric: 'kills', game: 't6', map: 't6:town' })).toMatchObject({ - guid: '101', rank: 1, gamesPlayed: 1, kills: 20 + const { getPlayerLeaderboardEntry, getLeaderboard } = + await import('../../src/lib/server/fleet/leaderboard.js'); + expect( + getLeaderboard({ q: '101', metric: 'kills', limit: 100 }).entries.some( + (entry) => entry.guid === '101' + ) + ).toBe(false); + expect(getPlayerLeaderboardEntry('101', { metric: 'kills' })).toMatchObject({ + guid: '101', + rank: 106, + gamesPlayed: 2, + kills: 30, + records: { bestKills: { game: 't6', value: 20 } } + }); + expect(getPlayerLeaderboardEntry('101', { metric: 'kills', game: 't5' })).toMatchObject({ + guid: '101', + rank: 106, + gamesPlayed: 1, + kills: 10, + records: { bestKills: { game: 't5', value: 10 } } + }); + expect( + getPlayerLeaderboardEntry('101', { metric: 'kills', game: 't6', map: 't6:town' }) + ).toMatchObject({ + guid: '101', + rank: 1, + gamesPlayed: 1, + kills: 20 }); expect(getPlayerLeaderboardEntry('10', { metric: 'kills' })).toBeNull(); expect(getPlayerLeaderboardEntry("' OR 1=1 --", { metric: 'kills' })).toBeNull(); @@ -74,16 +121,32 @@ it('keeps lifetime profiles available when selected history is empty or an effic const { db } = await import('../../src/lib/server/db/index.js'); addMatches(db(), '101', 4); const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - const filtered = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?game=t6&map=t6%3Atown&offset=25') }); - expect(filtered).toMatchObject({ returnTo: '/leaderboard', + const filtered = await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101?game=t6&map=t6%3Atown&offset=25') + }); + expect(filtered).toMatchObject({ + returnTo: '/leaderboard', profile: { stats: { gamesPlayed: 4, kills: 40 }, matches: [], matchesTotal: 0, offset: 0 }, - scope: { game: 't6', map: 't6:town', metric: 'highestRound', entry: null } }); - const unqualified = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') }); - expect(unqualified).toMatchObject({ profile: { matchesTotal: 4, stats: { gamesPlayed: 4 } }, - scope: { game: '', map: '', metric: 'averageKills', entry: null } }); + scope: { game: 't6', map: 't6:town', metric: 'highestRound', entry: null } + }); + const unqualified = await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') + }); + expect(unqualified).toMatchObject({ + profile: { matchesTotal: 4, stats: { gamesPlayed: 4 } }, + scope: { game: '', map: '', metric: 'averageKills', entry: null } + }); addMatches(db(), '101', 1); - expect(await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') })) - .toMatchObject({ scope: { entry: { guid: '101', rank: 1, averageKills: 10, gamesPlayed: 5 } } }); + expect( + await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101?metric=averageKills') + }) + ).toMatchObject({ + scope: { entry: { guid: '101', rank: 1, averageKills: 10, gamesPlayed: 5 } } + }); }); it('loads canonical navigation context while keeping source search and pagination out of profile rank', async () => { @@ -91,28 +154,76 @@ it('loads canonical navigation context while keeping source search and paginatio addMatches(db(), '101', 1, { kills: 100 }); addMatches(db(), '202', 1, { kills: 200 }); const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - const url = new URL('http://localhost/leaderboard/players/101?game=t5&map=t5%3Atheater&metric=kills'); - url.searchParams.set('returnTo', '/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50&unknown=drop'); + const url = new URL( + 'http://localhost/leaderboard/players/101?game=t5&map=t5%3Atheater&metric=kills' + ); + url.searchParams.set( + 'returnTo', + '/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50&unknown=drop' + ); const loaded = await load({ params: { guid: '101' }, url }); - expect(loaded.returnTo).toBe('/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50'); - expect(loaded.scope).toMatchObject({ game: 't5', map: 't5:theater', metric: 'kills', entry: { guid: '101', rank: 2 } }); + expect(loaded.returnTo).toBe( + '/leaderboard?game=t5&map=t5%3Atheater&metric=kills&q=101&offset=50' + ); + expect(loaded.scope).toMatchObject({ + game: 't5', + map: 't5:theater', + metric: 'kills', + entry: { guid: '101', rank: 2 } + }); expect(loaded.profile.offset).toBe(0); url.searchParams.set('returnTo', 'https://external.invalid/leaderboard'); expect((await load({ params: { guid: '101' }, url })).returnTo).toBe('/leaderboard'); - const defaults = await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101') }); - expect(defaults).toMatchObject({ returnTo: '/leaderboard', scope: { game: '', map: '', metric: 'highestRound' } }); + const defaults = await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101') + }); + expect(defaults).toMatchObject({ + returnTo: '/leaderboard', + scope: { game: '', map: '', metric: 'highestRound' } + }); }); it('rejects invalid filter parameters and retains existing offset normalization and missing-player behavior', async () => { const { db } = await import('../../src/lib/server/db/index.js'); addMatches(db(), '101', 30); const { load } = await import('../../src/routes/leaderboard/players/[guid]/+page.server.js'); - for (const query of ['game=t7', 'game=', 'metric=', 'metric=unknown', `map=${'x'.repeat(1025)}`]) { - await expect(load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) })).rejects.toThrow(expect.objectContaining({ status: 400 })); + for (const query of [ + 'game=t7', + 'game=', + 'metric=', + 'metric=unknown', + `map=${'x'.repeat(1025)}` + ]) { + await expect( + load({ + params: { guid: '101' }, + url: new URL(`http://localhost/leaderboard/players/101?${query}`) + }) + ).rejects.toThrow(expect.objectContaining({ status: 400 })); } for (const query of ['offset=garbage', 'offset=-1', 'offset=1.5', 'offset=Infinity']) { - expect((await load({ params: { guid: '101' }, url: new URL(`http://localhost/leaderboard/players/101?${query}`) })).profile.offset).toBe(0); + expect( + ( + await load({ + params: { guid: '101' }, + url: new URL(`http://localhost/leaderboard/players/101?${query}`) + }) + ).profile.offset + ).toBe(0); } - expect((await load({ params: { guid: '101' }, url: new URL('http://localhost/leaderboard/players/101?offset=999') })).profile.offset).toBe(25); - await expect(load({ params: { guid: 'missing' }, url: new URL('http://localhost/leaderboard/players/missing?game=t5') })).rejects.toThrow(expect.objectContaining({ status: 404 })); + expect( + ( + await load({ + params: { guid: '101' }, + url: new URL('http://localhost/leaderboard/players/101?offset=999') + }) + ).profile.offset + ).toBe(25); + await expect( + load({ + params: { guid: 'missing' }, + url: new URL('http://localhost/leaderboard/players/missing?game=t5') + }) + ).rejects.toThrow(expect.objectContaining({ status: 404 })); }); diff --git a/tests/unit/plugin-versions.test.ts b/tests/unit/plugin-versions.test.ts index 516ee17..b423f4d 100644 --- a/tests/unit/plugin-versions.test.ts +++ b/tests/unit/plugin-versions.test.ts @@ -8,41 +8,65 @@ beforeEach(async () => { delete process.env.RCON_PASSWORD; }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); const plugin = { - name: 't6-gsc-utils', kind: 'plugin' as const, - targetRelPath: 't6-gsc-utils.dll', content: Buffer.from([0, 1, 2, 3]) + name: 't6-gsc-utils', + kind: 'plugin' as const, + targetRelPath: 't6-gsc-utils.dll', + content: Buffer.from([0, 1, 2, 3]) }; describe('plugin release versions', () => { it('keeps release tags independent of upload order and preserves binary identity', async () => { - const { createScript, uploadNewVersion, listVersions, getVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, uploadNewVersion, listVersions } = + await import('../../src/lib/server/fleet/scripts.js'); const { script, version } = createScript({ ...plugin, releaseVersion: ' v3.1.0 ' }); expect(version).toMatchObject({ version: 1, release_version: 'v3.1.0' }); - const older = uploadNewVersion(script.id, Buffer.from([4, 5]), 'Known good release', 'v3.0.4'); + const older = uploadNewVersion( + script.id, + Buffer.from([4, 5]), + 'Known good release', + 'v3.0.4' + ); const rebuild = uploadNewVersion(script.id, Buffer.from([6, 7]), 'Rebuilt', 'v3.0.4'); - expect(listVersions(script.id).map(v => [v.version, v.release_version])).toEqual([ - [3, 'v3.0.4'], [2, 'v3.0.4'], [1, 'v3.1.0'] + expect(listVersions(script.id).map((v) => [v.version, v.release_version])).toEqual([ + [3, 'v3.0.4'], + [2, 'v3.0.4'], + [1, 'v3.1.0'] ]); expect(older.sha256).not.toBe(rebuild.sha256); - expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id)).toEqual(plugin.content); + expect( + (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id) + ).toEqual(plugin.content); }); it('allows unlabeled uploads and non-semver release tags', async () => { - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, uploadNewVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { script, version } = createScript(plugin); expect(version.release_version).toBeNull(); - expect(uploadNewVersion(script.id, plugin.content, undefined, ' ').release_version).toBeNull(); - expect(uploadNewVersion(script.id, plugin.content, undefined, '2026.09-rc1').release_version).toBe('2026.09-rc1'); + expect( + uploadNewVersion(script.id, plugin.content, undefined, ' ').release_version + ).toBeNull(); + expect( + uploadNewVersion(script.id, plugin.content, undefined, '2026.09-rc1').release_version + ).toBe('2026.09-rc1'); }); it('rejects invalid labels before storing an artifact or new upload', async () => { - const { createScript, uploadNewVersion, listScripts, listVersions } = await import('../../src/lib/server/fleet/scripts.js'); - expect(() => createScript({ ...plugin, releaseVersion: 'x'.repeat(81) })).toThrow(/release version/); + const { createScript, uploadNewVersion, listScripts, listVersions } = + await import('../../src/lib/server/fleet/scripts.js'); + expect(() => createScript({ ...plugin, releaseVersion: 'x'.repeat(81) })).toThrow( + /release version/ + ); expect(listScripts()).toHaveLength(0); const { script } = createScript(plugin); - expect(() => uploadNewVersion(script.id, plugin.content, undefined, 'v3\n.1')).toThrow(/release version/); + expect(() => uploadNewVersion(script.id, plugin.content, undefined, 'v3\n.1')).toThrow( + /release version/ + ); expect(listVersions(script.id)).toHaveLength(1); }); @@ -53,9 +77,14 @@ describe('plugin release versions', () => { db().exec('ALTER TABLE script_versions DROP COLUMN release_version'); closeDb(); expect(getVersion(version.id)).toMatchObject({ - id: version.id, version: 1, release_version: null, sha256: version.sha256 + id: version.id, + version: 1, + release_version: null, + sha256: version.sha256 }); - expect((await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id)).toEqual(plugin.content); + expect( + (await import('../../src/lib/server/fleet/scripts.js')).getVersionContent(version.id) + ).toEqual(plugin.content); closeDb(); expect(getVersion(version.id)!.release_version).toBeNull(); }); @@ -69,7 +98,10 @@ describe('plugin release versions', () => { form.set('name', plugin.name); form.set('releaseVersion', 'v3.1.0'); form.set('content', new File([plugin.content], plugin.targetRelPath)); - const created = await plugins.POST({ url, request: new Request(url, { method: 'POST', body: form }) }); + const created = await plugins.POST({ + url, + request: new Request(url, { method: 'POST', body: form }) + }); expect(created.status).toBe(200); const { script } = await created.json(); const listed = await (await plugins.GET({ url })).json(); @@ -77,11 +109,20 @@ describe('plugin release versions', () => { const next = new FormData(); next.set('releaseVersion', 'v3.1.1'); next.set('content', new File([new Uint8Array([4, 5])], plugin.targetRelPath)); - const uploaded = await uploads.POST({ params: { id: String(script.id) }, request: new Request(url, { method: 'POST', body: next }) }); + const uploaded = await uploads.POST({ + params: { id: String(script.id) }, + request: new Request(url, { method: 'POST', body: next }) + }); expect(uploaded.status).toBe(200); - expect((await uploaded.json()).version).toMatchObject({ version: 2, release_version: 'v3.1.1' }); + expect((await uploaded.json()).version).toMatchObject({ + version: 2, + release_version: 'v3.1.1' + }); const result = await (await detail.GET({ params: { id: String(script.id) } })).json(); - expect(result.versions.map((v: { release_version: string }) => v.release_version)).toEqual(['v3.1.1', 'v3.1.0']); + expect(result.versions.map((v: { release_version: string }) => v.release_version)).toEqual([ + 'v3.1.1', + 'v3.1.0' + ]); expect(result.versions[0]).not.toHaveProperty('content'); }); }); diff --git a/tests/unit/plutonium.test.ts b/tests/unit/plutonium.test.ts index a4fa795..33a2b9d 100644 --- a/tests/unit/plutonium.test.ts +++ b/tests/unit/plutonium.test.ts @@ -1,20 +1,38 @@ -import { maintenanceTick } from '../helpers/provisioning.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js'; +import { + maintenanceTimeInZone, + nextMaintenanceAt, + parsePlutoniumManifest, + parsePlutoniumRevision, + PLUTONIUM_MANIFEST_URL +} from '../../src/lib/plutonium.js'; -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(), + rconStatusResult: vi.fn(), + rconPlutoniumVersion: vi.fn() +})); let fixture: Fixture; const servers = ['t5_zombie_theater_1', 't6_zombie_town_1']; beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); + vi.resetModules(); + vi.resetAllMocks(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-21T05:59:00Z')); fixture = await newFixture(servers); - vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); - vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); + vi.stubEnv('RCON_PASSWORD', 'test'); + vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockImplementation(async () => + Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }) + ) + ); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); @@ -24,86 +42,145 @@ beforeEach(async () => { const r = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0); vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); - vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, status: { players: 0, map: null, gametype: null, roster: [] } }); + vi.mocked(r.rconStatusResult).mockResolvedValue({ + ok: true, + endpoint: { host: 'test', port: 28960 }, + status: { players: 0, map: null, gametype: null, roster: [] } + }); +}); +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.useRealTimers(); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); }); const api = () => import('../../src/lib/server/fleet/plutonium.js'); describe('Plutonium versions', () => { it.each([ - ['create', 'queued'], ['rebuild', 'waiting'], ['start', 'running'], - ['restart', 'awaiting-health'], ['stop', 'waiting'], ['destroy', 'running'] - ])('suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards', async (action, state) => { - const m = await api(); - await m.checkPlutonium(); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString()); - closeDb(); - vi.mocked(nativeHealth).mockClear(); - vi.mocked(rconPlutoniumVersion).mockClear(); - vi.setSystemTime(Date.now() + 1_000); - await m.checkPlutonium(); - expect(nativeHealth).toHaveBeenCalledTimes(1); - expect(nativeHealth).toHaveBeenCalledWith(servers[1]); - expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1); - expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run(); - await m.checkPlutonium(); - expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); - expect(db().prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?').get(servers[0])) - .toEqual({ checked_at: new Date().toISOString() }); - }); + ['create', 'queued'], + ['rebuild', 'waiting'], + ['start', 'running'], + ['restart', 'awaiting-health'], + ['stop', 'waiting'], + ['destroy', 'running'] + ])( + 'suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards', + async (action, state) => { + const m = await api(); + await m.checkPlutonium(); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); + const { rconPlutoniumVersion } = + await import('../../src/lib/server/integrations/rcon.js'); + const saved = db() + .prepare('SELECT * FROM plutonium_versions WHERE server = ?') + .get(servers[0]); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')` + ) + .run(servers[0], action, state, new Date().toISOString()); + closeDb(); + vi.mocked(nativeHealth).mockClear(); + vi.mocked(rconPlutoniumVersion).mockClear(); + vi.setSystemTime(Date.now() + 1_000); + await m.checkPlutonium(); + expect(nativeHealth).toHaveBeenCalledTimes(1); + expect(nativeHealth).toHaveBeenCalledWith(servers[1]); + expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toEqual(saved); + db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run(); + await m.checkPlutonium(); + expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); + expect( + db() + .prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?') + .get(servers[0]) + ).toEqual({ checked_at: new Date().toISOString() }); + } + ); it('suppresses fleet probes while a failed restart retains recovery capacity', async () => { const m = await api(); await m.checkPlutonium(); const { db } = await import('../../src/lib/server/db/index.js'); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString()); - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') + const saved = db() + .prepare('SELECT * FROM plutonium_versions WHERE server = ?') + .get(servers[0]); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); + db() + .prepare( + 'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)' + ) .run(servers[0], new Date().toISOString(), 'failed-restart'); vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); await m.checkPlutonium(); expect(nativeHealth).not.toHaveBeenCalledWith(servers[0]); expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); - db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toEqual(saved); + db() + .prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?') + .run(new Date().toISOString(), servers[0]); vi.setSystemTime(Date.now() + 1_000); await m.checkPlutonium(); expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); }); - it.each(['health', 'version'] as const)('discards maintenance observations when provisioning is queued during the %s probe', async phase => { - const m = await api(); - await m.checkPlutonium(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); - const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString()); - vi.mocked(rconPlutoniumVersion).mockClear(); - if (phase === 'health') vi.mocked(nativeHealth).mockImplementationOnce(async () => { - queue(); return { ok: true, state: 'running' }; - }); - else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { - queue(); return { revision: null, error: 'Restarting' }; - }); - vi.setSystemTime(Date.now() + 1_000); - expect(await probeVersion(servers[0], { maintenance: true })).toBeNull(); - if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); - }); + it.each(['health', 'version'] as const)( + 'discards maintenance observations when provisioning is queued during the %s probe', + async (phase) => { + const m = await api(); + await m.checkPlutonium(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); + const { rconPlutoniumVersion } = + await import('../../src/lib/server/integrations/rcon.js'); + const saved = db() + .prepare('SELECT * FROM plutonium_versions WHERE server = ?') + .get(servers[0]); + const queue = () => + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); + vi.mocked(rconPlutoniumVersion).mockClear(); + if (phase === 'health') + vi.mocked(nativeHealth).mockImplementationOnce(async () => { + queue(); + return { ok: true, state: 'running' }; + }); + else + vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { + queue(); + return { revision: null, error: 'Restarting' }; + }); + vi.setSystemTime(Date.now() + 1_000); + expect(await probeVersion(servers[0], { maintenance: true })).toBeNull(); + if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toEqual(saved); + } + ); it('coalesces published-release checks without probing the fleet', async () => { - const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js'); + const { checkRelease, release } = + await import('../../src/lib/server/integrations/plutonium.js'); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const first = checkRelease(); @@ -119,54 +196,105 @@ describe('Plutonium versions', () => { }); it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); + const { probeVersion, runningVersions } = + await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const submittedAt = new Date(Date.now() - 1000).toISOString(); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild'); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')` + ) + .run(servers[0], submittedAt, submittedAt); + db() + .prepare( + 'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)' + ) + .run(servers[0], submittedAt, 'rebuild'); expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354); - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null }); + expect( + db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0]) + ).toEqual({ healthy_at: null }); // A successful revision check alone cannot advertise completed recovery. - expect(runningVersions().find(version => version.server === servers[0])?.revision).toBeNull(); - const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]); + expect( + runningVersions().find((version) => version.server === servers[0])?.revision + ).toBeNull(); + const saved = db() + .prepare('SELECT * FROM plutonium_versions WHERE server=?') + .get(servers[0]); vi.mocked(rconPlutoniumVersion).mockClear(); expect(await probeVersion(servers[0], { maintenance: true })).toBeNull(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toEqual(saved); - }); - it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)('does not bypass recovery for %s', async scenario => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const submittedAt = new Date(Date.now() - 1000).toISOString(); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`).run(scenario === 'another-server' ? servers[1] : servers[0], - scenario === 'finished' ? 'done' : 'awaiting-health', submittedAt, scenario === 'unsubmitted' ? null : submittedAt); - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') - .run(servers[0], submittedAt, scenario === 'another-owner' ? 'other-rebuild' : 'rebuild'); - expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull(); - expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null }); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]) + ).toEqual(saved); }); + it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)( + 'does not bypass recovery for %s', + async (scenario) => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = + await import('../../src/lib/server/integrations/rcon.js'); + const submittedAt = new Date(Date.now() - 1000).toISOString(); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')` + ) + .run( + scenario === 'another-server' ? servers[1] : servers[0], + scenario === 'finished' ? 'done' : 'awaiting-health', + submittedAt, + scenario === 'unsubmitted' ? null : submittedAt + ); + db() + .prepare( + 'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)' + ) + .run( + servers[0], + submittedAt, + scenario === 'another-owner' ? 'other-rebuild' : 'rebuild' + ); + expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull(); + expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect( + db() + .prepare('SELECT healthy_at FROM restart_rollout WHERE server=?') + .get(servers[0]) + ).toEqual({ healthy_at: null }); + } + ); it('discards version evidence if its verifying operation finishes during the probe', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const submittedAt = new Date(Date.now() - 1000).toISOString(); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild'); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')` + ) + .run(servers[0], submittedAt, submittedAt); + db() + .prepare( + 'INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)' + ) + .run(servers[0], submittedAt, 'rebuild'); vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run(); return { revision: 5354, error: null }; }); expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull(); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toBeUndefined(); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]) + ).toBeUndefined(); }); it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => { const m = await api(); - const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js'); + const { checkRelease, HOUR } = + await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); await m.checkPlutonium(); vi.setSystemTime(Date.now() + HOUR); @@ -179,13 +307,22 @@ describe('Plutonium versions', () => { expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length); }); it('uses the official production manifest without requesting the challenged changelog page', async () => { - const m = await api(); await m.checkPlutonium(); + const m = await api(); + await m.checkPlutonium(); expect(fetch).toHaveBeenCalledTimes(1); expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.any(Object)); expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: null }); - expect(parsePlutoniumManifest({ product: 'plutonium-core-prod', revision: 5354 })).toBe(5354); - for (const value of [null, {}, { ref: '1.0.221' }, { product: 'plutonium-core-dev', revision: 6000 }, - { product: 'plutonium-core-prod', revision: '5354' }, { product: 'plutonium-core-prod', revision: -1 }]) { + expect(parsePlutoniumManifest({ product: 'plutonium-core-prod', revision: 5354 })).toBe( + 5354 + ); + for (const value of [ + null, + {}, + { ref: '1.0.221' }, + { product: 'plutonium-core-dev', revision: 6000 }, + { product: 'plutonium-core-prod', revision: '5354' }, + { product: 'plutonium-core-prod', revision: -1 } + ]) { expect(() => parsePlutoniumManifest(value)).toThrow(); } }); @@ -194,98 +331,158 @@ describe('Plutonium versions', () => { expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull(); }); it('preserves the cached release with a visible error and clears failed running-version observations', async () => { - const m = await api(); await m.checkPlutonium(); + const m = await api(); + await m.checkPlutonium(); vi.mocked(fetch).mockRejectedValue(new Error('Offline')); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' }); await m.checkPlutonium(); expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' }); - expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true); + expect( + m.plutoniumStatus().servers.every((s) => s.revision === null && s.error === 'Timeout') + ).toBe(true); }); it.each([ () => new Response('Blocked', { status: 403 }), () => new Response('Browser challenge'), () => Response.json({ product: 'plutonium-core-dev', revision: 9999 }) - ])('keeps the cached revision when the manifest is unavailable or invalid', async response => { - const m = await api(); await m.checkPlutonium(); - vi.mocked(fetch).mockImplementation(async () => response()); - await m.checkPlutonium(); - expect(m.plutoniumStatus().release.revision).toBe(5354); - expect(m.plutoniumStatus().release.error).toBeTruthy(); - m.queueMaintenance([servers[0]]); - expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull(); - }); + ])( + 'keeps the cached revision when the manifest is unavailable or invalid', + async (response) => { + const m = await api(); + await m.checkPlutonium(); + vi.mocked(fetch).mockImplementation(async () => response()); + await m.checkPlutonium(); + expect(m.plutoniumStatus().release.revision).toBe(5354); + expect(m.plutoniumStatus().release.error).toBeTruthy(); + m.queueMaintenance([servers[0]]); + expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull(); + } + ); it('rejects a version observed at the startup submission timestamp, then accepts a later observation', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js'); + const { probeVersion, runningVersions, checkRelease } = + await import('../../src/lib/server/integrations/plutonium.js'); const submittedAt = new Date().toISOString(); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')` + ) + .run(servers[0], submittedAt, submittedAt); await checkRelease(); expect(await probeVersion(servers[0])).toBeNull(); - expect(runningVersions()[0]).toMatchObject({ revision: null, checked_at: submittedAt, update_status: 'unknown', error: expect.stringContaining('after startup') }); + expect(runningVersions()[0]).toMatchObject({ + revision: null, + checked_at: submittedAt, + update_status: 'unknown', + error: expect.stringContaining('after startup') + }); vi.setSystemTime(Date.now() + 1); expect(await probeVersion(servers[0])).toBe(5354); - expect(runningVersions()[0]).toMatchObject({ revision: 5354, update_status: 'current', error: null }); + expect(runningVersions()[0]).toMatchObject({ + revision: 5354, + update_status: 'current', + error: null + }); }); it('discards a running revision when a new lifecycle submission arrives during its probe', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); + const { probeVersion, runningVersions } = + await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { const now = new Date().toISOString(); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')` + ) + .run(servers[0], now, now); return { revision: 5354, error: null }; }); expect(await probeVersion(servers[0])).toBeNull(); expect(runningVersions()[0]).toMatchObject({ revision: null, update_status: 'unknown' }); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toBeUndefined(); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toBeUndefined(); }); - it.each(['startup', 'replacement'] as const)('preserves a newer successful revision when an old probe finishes after %s', async change => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js'); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - await checkRelease(); - db().prepare('UPDATE servers SET created_at = ? WHERE name = ?').run('2026-09-21T05:58:00.000Z', servers[0]); - let finishOld!: (value: { revision: number; error: null }) => void; - let started!: () => void; - const probing = new Promise(resolve => { started = resolve; }); - vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => { - started(); - return new Promise(resolve => { finishOld = resolve; }); - }); - const oldProbe = probeVersion(servers[0]); - await probing; - const now = new Date().toISOString(); - if (change === 'startup') { - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now); - } else { - // Both registrations have no lifecycle submission, so only their - // registry identity distinguishes the old and replacement instances. - db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); - db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run(servers[0], now); + it.each(['startup', 'replacement'] as const)( + 'preserves a newer successful revision when an old probe finishes after %s', + async (change) => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion, runningVersions, checkRelease } = + await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = + await import('../../src/lib/server/integrations/rcon.js'); + await checkRelease(); + db() + .prepare('UPDATE servers SET created_at = ? WHERE name = ?') + .run('2026-09-21T05:58:00.000Z', servers[0]); + let finishOld!: (value: { revision: number; error: null }) => void; + let started!: () => void; + const probing = new Promise((resolve) => { + started = resolve; + }); + vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => { + started(); + return new Promise((resolve) => { + finishOld = resolve; + }); + }); + const oldProbe = probeVersion(servers[0]); + await probing; + const now = new Date().toISOString(); + if (change === 'startup') { + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')` + ) + .run(servers[0], now, now); + } else { + // Both registrations have no lifecycle submission, so only their + // registry identity distinguishes the old and replacement instances. + db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); + db() + .prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)') + .run(servers[0], now); + } + vi.setSystemTime(Date.now() + 1); + expect(await probeVersion(servers[0])).toBe(5354); + const saved = db() + .prepare('SELECT * FROM plutonium_versions WHERE server = ?') + .get(servers[0]); + vi.setSystemTime(Date.now() + 1); + finishOld({ revision: 5300, error: null }); + expect(await oldProbe).toBeNull(); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toEqual(saved); + expect(runningVersions()[0]).toMatchObject({ + revision: 5354, + error: null, + update_status: 'current' + }); } - vi.setSystemTime(Date.now() + 1); - expect(await probeVersion(servers[0])).toBe(5354); - const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); - vi.setSystemTime(Date.now() + 1); - finishOld({ revision: 5300, error: null }); - expect(await oldProbe).toBeNull(); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); - expect(runningVersions()[0]).toMatchObject({ revision: 5354, error: null, update_status: 'current' }); - }); + ); it('discards a running revision when the server is stopped during its probe', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); + const { probeVersion, runningVersions } = + await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { - db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[0]); + db() + .prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?") + .run(servers[0]); return { revision: 5354, error: null }; }); expect(await probeVersion(servers[0])).toBeNull(); - expect(runningVersions()[0]).toMatchObject({ revision: null, error: 'Server is intentionally stopped.', update_status: 'stopped' }); + expect(runningVersions()[0]).toMatchObject({ + revision: null, + error: 'Server is intentionally stopped.', + update_status: 'stopped' + }); }); }); @@ -304,9 +501,16 @@ describe('maintenance timezones', () => { }); it('preserves legacy UTC schedules and displays their next occurrence in the user timezone', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('plutonium_schedule', JSON.stringify({ - enabled: true, time: '06:00', nextAt: '2026-09-21T06:00:00.000Z' - })); + db() + .prepare('INSERT INTO settings (key, value) VALUES (?, ?)') + .run( + 'plutonium_schedule', + JSON.stringify({ + enabled: true, + time: '06:00', + nextAt: '2026-09-21T06:00:00.000Z' + }) + ); const m = await api(); const schedule = m.plutoniumStatus().schedule; expect(schedule.timeZone).toBe('UTC'); @@ -317,79 +521,161 @@ describe('maintenance timezones', () => { describe('persistent maintenance', () => { it('waits to attach legacy maintenance while another operation owns the server', async () => { - const m = await api(); await m.checkPlutonium(); + const m = await api(); + await m.checkPlutonium(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`).run(servers[0], new Date().toISOString()); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`) + .run(servers[0], new Date().toISOString()); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', provisioning_id: null, detail: expect.stringContaining('another provisioning operation') }); - expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 1 }); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + state: 'queued', + provisioning_id: null, + detail: expect.stringContaining('another provisioning operation') + }); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ + count: 1 + }); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'") + .run(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy(); - expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 }); + expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ + count: 2 + }); }); it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const m = await api(); + await m.checkPlutonium(); + m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); const job = m.plutoniumStatus().jobs[0]; - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?") + .run(job.provisioning_id); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); + vi.mocked(nativeHealth).mockClear(); + vi.mocked(rconPlutoniumVersion).mockClear(); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') }); - expect(nativeHealth).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + state: 'queued', + detail: expect.stringContaining('before verifying') + }); + expect(nativeHealth).not.toHaveBeenCalled(); + expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'") + .run(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('done'); expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]); }); it('does not fail maintenance verification when another operation starts during its version probe', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const m = await api(); + await m.checkPlutonium(); + m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?") + .run(m.plutoniumStatus().jobs[0].provisioning_id); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); return { revision: null, error: 'Restarting' }; }); - const before = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); + const before = db() + .prepare('SELECT * FROM plutonium_versions WHERE server = ?') + .get(servers[0]); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') }); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(before); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'").run(); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + state: 'queued', + detail: expect.stringContaining('before verifying') + }); + expect( + db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]) + ).toEqual(before); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'") + .run(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('done'); }); it('mirrors its own failed operation even while another operation owns the server', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); + const m = await api(); + await m.checkPlutonium(); + m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare( + "UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?" + ) + .run(m.plutoniumStatus().jobs[0].provisioning_id); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' }); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + state: 'failed', + detail: 'Restart failed' + }); }); it('validates API requests without changing the schedule or queue', async () => { const { POST } = await import('../../src/routes/api/plutonium/+server.js'); - const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) - }) }); + const post = (body: unknown) => + POST({ + request: new Request('http://localhost/api/plutonium', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + }); expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400); expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400); expect((await post({ action: 'cancel', id: 100 })).status).toBe(409); expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(400); - expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' })).status).toBe(400); - expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'America/New_York' })).status).toBe(200); + expect( + (await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' })) + .status + ).toBe(400); + expect( + ( + await post({ + action: 'schedule', + enabled: true, + time: '06:00', + timeZone: 'America/New_York' + }) + ).status + ).toBe(200); const m = await api(); expect(m.plutoniumStatus().jobs).toEqual([]); expect(m.plutoniumStatus().schedule.enabled).toBe(true); - expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' }); + expect(m.plutoniumStatus().schedule).toMatchObject({ + timeZone: 'America/New_York', + nextAt: '2026-09-21T10:00:00.000Z' + }); }); }); @@ -397,52 +683,110 @@ describe('Maintenance policies and runtime updates', () => { async function outdated() { const rcon = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); - const m = await api(); await m.checkPlutonium(); return m; + const m = await api(); + await m.checkPlutonium(); + return m; } it('preserves the legacy recovery preference without enabling updates', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run(); const p = await import('../../src/lib/server/maintenance/policies.js'); - expect(p.maintenancePolicies()).toEqual({ paused: false, health: false, drift: false, updates: false }); - p.setMaintenancePolicy('health', true); p.pauseMaintenance(true); closeDb(); - expect(p.maintenancePolicies()).toEqual({ paused: true, health: true, drift: false, updates: false }); + expect(p.maintenancePolicies()).toEqual({ + paused: false, + health: false, + drift: false, + updates: false + }); + p.setMaintenancePolicy('health', true); + p.pauseMaintenance(true); + closeDb(); + expect(p.maintenancePolicies()).toEqual({ + paused: true, + health: true, + drift: false, + updates: false + }); expect(p.maintenanceAllowed('health')).toBe(false); }); it('only automatically updates verified outdated servers and requires opt-in', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?').run(servers[1]); + db() + .prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?') + .run(servers[1]); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs).toEqual([]); - const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); - await m.maintenanceTick(); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]); - expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown'); - }); - it.each(['queued', 'waiting', 'running', 'awaiting-health'])('skips automatic updates during %s provisioning and resumes after completion', async state => { - const m = await outdated(); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')`).run(servers[0], state, new Date().toISOString()); - const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); + const p = await import('../../src/lib/server/maintenance/policies.js'); + p.setMaintenancePolicy('updates', true); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' })]); - expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?').get(servers[0])).toEqual({ count: 1 }); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'").run(); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toHaveLength(2); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'automatic' }); + expect(m.plutoniumStatus().jobs).toEqual([ + expect.objectContaining({ + server: servers[0], + kind: 'update', + origin: 'automatic', + target_revision: 5354 + }) + ]); + expect( + m.plutoniumStatus().servers.find((s) => s.server === servers[1])?.update_status + ).toBe('unknown'); }); + it.each(['queued', 'waiting', 'running', 'awaiting-health'])( + 'skips automatic updates during %s provisioning and resumes after completion', + async (state) => { + const m = await outdated(); + const { db } = await import('../../src/lib/server/db/index.js'); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')` + ) + .run(servers[0], state, new Date().toISOString()); + const p = await import('../../src/lib/server/maintenance/policies.js'); + p.setMaintenancePolicy('updates', true); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toEqual([ + expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' }) + ]); + expect( + db() + .prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?') + .get(servers[0]) + ).toEqual({ count: 1 }); + db() + .prepare( + "UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'" + ) + .run(); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toHaveLength(2); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + server: servers[0], + kind: 'update', + origin: 'automatic' + }); + } + ); it('skips a busy server for the scheduled occurrence while other servers proceed', async () => { - const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); + const m = await outdated(); + m.saveMaintenanceSchedule(true, '06:00', 'UTC'); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]); + expect(m.plutoniumStatus().jobs).toEqual([ + expect.objectContaining({ server: servers[1], kind: 'schedule' }) + ]); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-22T06:00:00.000Z'); - db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run(); + db() + .prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'") + .run(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs).toHaveLength(1); vi.setSystemTime(new Date('2026-09-22T06:00:00Z')); @@ -451,39 +795,66 @@ describe('Maintenance policies and runtime updates', () => { expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'schedule' }); }); it('skips automatic and scheduled requests while a failed operation retains recovery capacity', async () => { - const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); + const m = await outdated(); + m.saveMaintenanceSchedule(true, '06:00', 'UTC'); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')`).run(servers[0], new Date().toISOString()); - db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')`) + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); + db() + .prepare( + `INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')` + ) .run(servers[0], new Date().toISOString()); expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(0); - vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]); - db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]); + vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); + await m.maintenanceTick(); + expect(m.plutoniumStatus().jobs).toEqual([ + expect.objectContaining({ server: servers[1], kind: 'schedule' }) + ]); + db() + .prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?') + .run(new Date().toISOString(), servers[0]); expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(1); }); it('preserves explicit update requests while another operation owns the server', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString()); + db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')` + ) + .run(servers[0], new Date().toISOString()); expect(m.queueUpdates([servers[0]])).toBe(1); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' }); + expect(m.plutoniumStatus().jobs[0]).toMatchObject({ + server: servers[0], + kind: 'update', + origin: 'manual' + }); }); it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)').run(servers[0], new Date().toISOString(), new Date().toISOString()); + db() + .prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)') + .run(servers[0], new Date().toISOString(), new Date().toISOString()); expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown'); vi.setSystemTime(Date.now() + 3_600_000); expect(() => m.queueUpdates()).toThrow('published revision'); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockClear(); - vi.mocked(nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); + vi.mocked(nativeHealth).mockResolvedValue({ + ok: false, + state: 'starting', + reason: 'Starting' + }); await m.checkPlutonium(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true); + expect(m.plutoniumStatus().servers.every((s) => s.update_status === 'unknown')).toBe(true); }); }); diff --git a/tests/unit/provisioning-cancellation.test.ts b/tests/unit/provisioning-cancellation.test.ts index 38aed39..77f6bdd 100644 --- a/tests/unit/provisioning-cancellation.test.ts +++ b/tests/unit/provisioning-cancellation.test.ts @@ -1,57 +1,117 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kickProvisioning: vi.fn(async () => {}) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kickProvisioning: vi.fn(async () => {}) +})); let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); describe('provisioning cancellation', () => { - it.each(['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'])('cancels %s in every active phase and keeps cancellation terminal', async action => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { finishDeployment, getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js'); - for (const state of ['queued', 'waiting', 'running', 'awaiting-health', 'interrupted']) { - const id = `${action}-${state}`; - db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) - VALUES(?,'server',?,?, 'In progress','2026-10-05',?,'test')`).run(id, action, state, state === 'waiting' ? null : '2026-10-05'); - expect(canCancelProvisioningJob(id)).toBe(true); - expect(getDeployment(id)?.canCancel).toBe(true); - expect(listDeployments().find(row => row.id === id)?.canCancel).toBe(1); - expect(cancelProvisioningJob(id)).toBe(true); - expect(cancelProvisioningJob(id)).toBe(true); - finishDeployment(id, { ok: true }); - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' }); - expect(getDeployment(id)?.canCancel).toBe(false); - expect(db().prepare("SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='outcome'").get(id)).toEqual({ n: 1 }); + it.each(['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'])( + 'cancels %s in every active phase and keeps cancellation terminal', + async (action) => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { cancelProvisioningJob, canCancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); + const { finishDeployment, getDeployment, listDeployments } = + await import('../../src/lib/server/provisioning/history.js'); + for (const state of [ + 'queued', + 'waiting', + 'running', + 'awaiting-health', + 'interrupted' + ]) { + const id = `${action}-${state}`; + db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) + VALUES(?,'server',?,?, 'In progress','2026-10-05',?,'test')` + ) + .run(id, action, state, state === 'waiting' ? null : '2026-10-05'); + expect(canCancelProvisioningJob(id)).toBe(true); + expect(getDeployment(id)?.canCancel).toBe(true); + expect(listDeployments().find((row) => row.id === id)?.canCancel).toBe(1); + expect(cancelProvisioningJob(id)).toBe(true); + expect(cancelProvisioningJob(id)).toBe(true); + finishDeployment(id, { ok: true }); + expect(getDeployment(id)?.deployment).toMatchObject({ + state: 'cancelled', + stage: 'Cancelled by operator' + }); + expect(getDeployment(id)?.canCancel).toBe(false); + expect( + db() + .prepare( + "SELECT COUNT(*) n FROM server_deployment_events WHERE deployment_id=? AND kind='outcome'" + ) + .get(id) + ).toEqual({ n: 1 }); + } } - }); + ); it('releases only the cancelled job’s recovery slot and cancels its pending file changes', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); db().prepare("INSERT INTO servers(name) VALUES('one'),('two')").run(); - const id = beginDeployment('one','restart'); - const other = beginDeployment('two','restart'); - db().prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('one','2026-10-05',?),('two','2026-10-05',?)").run(id, other); - db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'campaign','source','one')").run(id); - queueFileChange({ sourceKind: 'campaign', sourceId: 'source', server: 'one', kind: 'script', relPath: 'test.gsc', content: Buffer.from('queued'), inventorySource: 'test' }); + const id = beginDeployment('one', 'restart'); + const other = beginDeployment('two', 'restart'); + db() + .prepare( + "INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('one','2026-10-05',?),('two','2026-10-05',?)" + ) + .run(id, other); + db() + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'campaign','source','one')" + ) + .run(id); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'source', + server: 'one', + kind: 'script', + relPath: 'test.gsc', + content: Buffer.from('queued'), + inventorySource: 'test' + }); expect(cancelProvisioningJob(id)).toBe(true); - expect(db().prepare('SELECT operation_id FROM restart_rollout').all()).toEqual([{ operation_id: other }]); - expect(db().prepare('SELECT state FROM staged_file_changes').get()).toEqual({ state: 'cancelled' }); + expect(db().prepare('SELECT operation_id FROM restart_rollout').all()).toEqual([ + { operation_id: other } + ]); + expect(db().prepare('SELECT state FROM staged_file_changes').get()).toEqual({ + state: 'cancelled' + }); }); it('rejects missing, completed and deleted-server jobs through the cancellation API', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js'); - for (const state of ['done','warning','failed']) { - db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'server','restart',?,'Finished','2026-10-05','test')`).run(state,state); + for (const state of ['done', 'warning', 'failed']) { + db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'server','restart',?,'Finished','2026-10-05','test')` + ) + .run(state, state); expect(POST({ params: { id: state } }).status).toBe(409); } expect(POST({ params: { id: 'missing' } }).status).toBe(409); - db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by,server_deleted_at) VALUES('deleted','server','restart','running','Running','2026-10-05','test','2026-10-05')`).run(); + db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by,server_deleted_at) VALUES('deleted','server','restart','running','Running','2026-10-05','test','2026-10-05')` + ) + .run(); expect(POST({ params: { id: 'deleted' } }).status).toBe(409); }); }); diff --git a/tests/unit/provisioning-detail.test.ts b/tests/unit/provisioning-detail.test.ts index e9646d8..4846efc 100644 --- a/tests/unit/provisioning-detail.test.ts +++ b/tests/unit/provisioning-detail.test.ts @@ -2,22 +2,36 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); }); -afterEach(async () => { await fixture.cleanup(); vi.restoreAllMocks(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture(); +}); +afterEach(async () => { + await fixture.cleanup(); + vi.restoreAllMocks(); +}); describe('provisioning detail projection', () => { it('records identical structured and legacy stage text once while preserving changed observations', async () => { - const { beginDeployment, deploymentStage, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, deploymentStage, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); const id = beginDeployment('server', 'restart'); - const update = { phase: 'eligibility' as const, - blocker: { code: 'players', message: 'Waiting for 3 players', automatic: true } }; + const update = { + phase: 'eligibility' as const, + blocker: { code: 'players', message: 'Waiting for 3 players', automatic: true } + }; deploymentStage(id, update.blocker.message, update); deploymentStage(id, update.blocker.message, update); - expect(getDeployment(id)!.events.filter(event => event.message === update.blocker.message)).toHaveLength(1); - deploymentStage(id, 'Waiting for 2 players', { ...update, blocker: { ...update.blocker, message: 'Waiting for 2 players' } }); + expect( + getDeployment(id)!.events.filter((event) => event.message === update.blocker.message) + ).toHaveLength(1); + deploymentStage(id, 'Waiting for 2 players', { + ...update, + blocker: { ...update.blocker, message: 'Waiting for 2 players' } + }); const events = getDeployment(id)!.events; - expect(events.filter(event => event.message === 'Waiting for 2 players')).toHaveLength(1); - expect(events.filter(event => event.kind === 'blocker')).toHaveLength(1); + expect(events.filter((event) => event.message === 'Waiting for 2 players')).toHaveLength(1); + expect(events.filter((event) => event.kind === 'blocker')).toHaveLength(1); }); it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => { @@ -27,21 +41,32 @@ describe('provisioning detail projection', () => { const inspect = vi.spyOn(backend, 'nativeHealth'); const status = vi.spyOn(rcon, 'rconStatusResult'); const id = history.beginDeployment('t5_zombie_theater_1', 'create'); - for (let index = 0; index < 40; index++) history.recordDeploymentEvent(id, `Recorded activity ${index}`); + for (let index = 0; index < 40; index++) + history.recordDeploymentEvent(id, `Recorded activity ${index}`); history.finishDeployment(id, { ok: false, message: 'A recorded startup failure' }); - for (let index = 0; index < 8; index++) history.recordDeploymentEvent(id, `Later observation ${index}`); + for (let index = 0; index < 8; index++) + history.recordDeploymentEvent(id, `Later observation ${index}`); const full = history.getDeployment(id)!; const recent = history.getDeployment(id, { events: 'recent' })!; expect(full.events).toHaveLength(50); expect(recent.events).toEqual(full.events.slice(-5)); - expect(recent.activity).toMatchObject({ total: 50, latestId: full.events.at(-1)!.id, - failureExcerpt: { message: 'Failed: A recorded startup failure', kind: 'outcome' } }); + expect(recent.activity).toMatchObject({ + total: 50, + latestId: full.events.at(-1)!.id, + failureExcerpt: { message: 'Failed: A recorded startup failure', kind: 'outcome' } + }); expect(recent.progress?.phase).toBe('requested'); - expect(recent.progress?.checkpoints.map(point => point.key)).toEqual(['requested']); + expect(recent.progress?.checkpoints.map((point) => point.key)).toEqual(['requested']); const { GET } = await import('../../src/routes/api/provisioning/[id]/+server.js'); expect((await (await GET({ params: { id } })).json()).events).toHaveLength(50); - expect((await (await GET({ params: { id }, url: new URL('http://localhost/?events=recent') })).json()).events).toHaveLength(5); + expect( + ( + await ( + await GET({ params: { id }, url: new URL('http://localhost/?events=recent') }) + ).json() + ).events + ).toHaveLength(5); const { load } = await import('../../src/routes/provisioning/[id]/+page.server.js'); expect(load({ params: { id } })?.detail.events).toHaveLength(5); expect(inspect).not.toHaveBeenCalled(); @@ -51,61 +76,129 @@ describe('provisioning detail projection', () => { it('retains missing sources, normalizes malformed warnings, and leaves legacy evidence unrecorded', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - db().prepare(`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,warnings) - VALUES ('legacy','old-server','restart','done','Old custom stage','2026-01-01','ui',?)`).run('["Check files",7,null,{"bad":true}]'); - db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','campaign','missing','old-server')").run(); - db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','watchdog','incident','old-server')").run(); - db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) - VALUES ('incident','old-server','health','resolved','Recovered game service','2026-01-01','2026-01-01')`).run(); + db() + .prepare( + `INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,warnings) + VALUES ('legacy','old-server','restart','done','Old custom stage','2026-01-01','ui',?)` + ) + .run('["Check files",7,null,{"bad":true}]'); + db() + .prepare( + "INSERT INTO provisioning_sources VALUES ('legacy','campaign','missing','old-server')" + ) + .run(); + db() + .prepare( + "INSERT INTO provisioning_sources VALUES ('legacy','watchdog','incident','old-server')" + ) + .run(); + db() + .prepare( + `INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) + VALUES ('incident','old-server','health','resolved','Recovered game service','2026-01-01','2026-01-01')` + ) + .run(); const detail = getDeployment('legacy')!; expect(detail.progress).toBeNull(); expect(detail.events).toEqual([]); expect(detail.activity).toEqual({ total: 0, latestId: null, failureExcerpt: null }); - expect(detail.warningMessages).toEqual(['Check files', 'Some recorded warnings could not be read.']); - expect(detail.sources).toEqual([ - { kind: 'campaign', source_id: 'missing', server: 'old-server', label: 'Campaign unavailable', href: null }, - { kind: 'watchdog', source_id: 'incident', server: 'old-server', label: 'Recovered game service', href: '/maintenance/incident' } + expect(detail.warningMessages).toEqual([ + 'Check files', + 'Some recorded warnings could not be read.' + ]); + expect(detail.sources).toEqual([ + { + kind: 'campaign', + source_id: 'missing', + server: 'old-server', + label: 'Campaign unavailable', + href: null + }, + { + kind: 'watchdog', + source_id: 'incident', + server: 'old-server', + label: 'Recovered game service', + href: '/maintenance/incident' + } + ]); + db() + .prepare("UPDATE server_deployments SET warnings = 'broken json' WHERE id = 'legacy'") + .run(); + expect(getDeployment('legacy')!.warningMessages).toEqual([ + 'Recorded warnings could not be read.' ]); - db().prepare("UPDATE server_deployments SET warnings = 'broken json' WHERE id = 'legacy'").run(); - expect(getDeployment('legacy')!.warningMessages).toEqual(['Recorded warnings could not be read.']); }); - it.each(['create', 'rebuild', 'start', 'restart'] as const)('shows live capacity only for active %s operations, excluding historical work on the same server', async action => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment, getDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); - db().prepare("INSERT INTO servers (name) VALUES ('server')").run(); - const old = beginDeployment('server', action); - finishDeployment(old, { ok: false, message: 'Timeout' }); - const current = beginDeployment('server', action); - db().prepare("INSERT INTO restart_rollout (server,requested_at,operation_id) VALUES ('server','2026-01-01',?)").run(current); - const kinds = action === 'create' ? ['startup'] : action === 'rebuild' ? ['startup', 'restart'] : ['restart']; - expect(getDeployment(current)!.capacity.pools).toEqual(kinds.map(kind => expect.objectContaining({ - kind, used: 1, ownsSlot: true, holders: [{ id: current, server: 'server' }] - }))); - for (const state of ['done', 'warning', 'failed', 'interrupted', 'cancelled']) { - db().prepare('UPDATE server_deployments SET state = ? WHERE id = ?').run(state, old); - expect(getDeployment(old)!.capacity.pools, state).toEqual([]); + it.each(['create', 'rebuild', 'start', 'restart'] as const)( + 'shows live capacity only for active %s operations, excluding historical work on the same server', + async (action) => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { beginDeployment, getDeployment, finishDeployment } = + await import('../../src/lib/server/provisioning/history.js'); + db().prepare("INSERT INTO servers (name) VALUES ('server')").run(); + const old = beginDeployment('server', action); + finishDeployment(old, { ok: false, message: 'Timeout' }); + const current = beginDeployment('server', action); + db() + .prepare( + "INSERT INTO restart_rollout (server,requested_at,operation_id) VALUES ('server','2026-01-01',?)" + ) + .run(current); + const kinds = + action === 'create' + ? ['startup'] + : action === 'rebuild' + ? ['startup', 'restart'] + : ['restart']; + expect(getDeployment(current)!.capacity.pools).toEqual( + kinds.map((kind) => + expect.objectContaining({ + kind, + used: 1, + ownsSlot: true, + holders: [{ id: current, server: 'server' }] + }) + ) + ); + for (const state of ['done', 'warning', 'failed', 'interrupted', 'cancelled']) { + db() + .prepare('UPDATE server_deployments SET state = ? WHERE id = ?') + .run(state, old); + expect(getDeployment(old)!.capacity.pools, state).toEqual([]); + } + expect(getDeployment(old)!.progress!.recovery).toBeNull(); + db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(new Date().toISOString()); + expect(getDeployment(old)!.capacity.pools).toEqual([]); + for (const state of ['queued', 'waiting', 'running', 'awaiting-health']) { + db() + .prepare('UPDATE server_deployments SET state = ? WHERE id = ?') + .run(state, current); + expect( + getDeployment(current)!.capacity.pools.map((pool) => pool.kind), + state + ).toEqual(kinds); + } } - expect(getDeployment(old)!.progress!.recovery).toBeNull(); - db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(new Date().toISOString()); - expect(getDeployment(old)!.capacity.pools).toEqual([]); - for (const state of ['queued', 'waiting', 'running', 'awaiting-health']) { - db().prepare('UPDATE server_deployments SET state = ? WHERE id = ?').run(state, current); - expect(getDeployment(current)!.capacity.pools.map(pool => pool.kind), state).toEqual(kinds); - } - }); + ); it('labels submission order independently of startup reservations and retains queued recipe status', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); const { recordProgress } = await import('../../src/lib/server/provisioning/progress.js'); const first = beginDeployment('first', 'create'); const next = beginDeployment('next', 'create'); db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(next); recordProgress(next, { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null }); const detail = getDeployment(next)!; - expect(detail.capacity.pools[0]).toMatchObject({ kind: 'startup', used: 1, requestOrder: 1, ownsSlot: false, - holders: [{ id: first, server: 'first' }] }); + expect(detail.capacity.pools[0]).toMatchObject({ + kind: 'startup', + used: 1, + requestOrder: 1, + ownsSlot: false, + holders: [{ id: first, server: 'first' }] + }); expect(detail.progress).toMatchObject({ recipeStatus: 'queued', observedAt: null }); }); @@ -114,37 +207,71 @@ describe('provisioning detail projection', () => { const history = await import('../../src/lib/server/provisioning/history.js'); db().prepare("INSERT INTO servers (name,game_port) VALUES ('server',28960)").run(); const id = history.beginDeployment('server', 'create'); - db().prepare("INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')").run(id); - history.deploymentStage(id, 'Waiting for game evidence', { phase: 'verify-game', checks: [ - { key: 'game', label: 'Game status', result: 'waiting', source: null } - ] }); + db() + .prepare( + "INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')" + ) + .run(id); + history.deploymentStage(id, 'Waiting for game evidence', { + phase: 'verify-game', + checks: [{ key: 'game', label: 'Game status', result: 'waiting', source: null }] + }); history.finishDeployment(id, { ok: false, message: 'Observation interrupted' }); const failed = history.getDeployment(id)!; expect(failed.progress?.phase).toBe('verify-game'); expect(failed.capacity.pools).toEqual([]); expect(history.resumeVerification(id)).toBe(true); const resumed = history.getDeployment(id)!; - expect(resumed.deployment).toMatchObject({ id, state: 'awaiting-health', finished_at: null, error: null }); - expect(resumed.progress).toMatchObject({ phase: 'verify-container', checks: [], observedAt: null }); - expect(resumed.capacity.pools).toEqual([expect.objectContaining({ kind: 'startup', ownsSlot: true })]); - const outcomes = resumed.events.filter(event => event.kind === 'outcome'); + expect(resumed.deployment).toMatchObject({ + id, + state: 'awaiting-health', + finished_at: null, + error: null + }); + expect(resumed.progress).toMatchObject({ + phase: 'verify-container', + checks: [], + observedAt: null + }); + expect(resumed.capacity.pools).toEqual([ + expect.objectContaining({ kind: 'startup', ownsSlot: true }) + ]); + const outcomes = resumed.events.filter((event) => event.kind === 'outcome'); expect(outcomes).toHaveLength(1); - expect(JSON.parse(outcomes[0].details_json!)).toMatchObject({ state: 'failed', finishedAt: failed.deployment.finished_at, - error: 'Observation interrupted' }); - expect(resumed.events.some(event => event.kind === 'resumed')).toBe(true); + expect(JSON.parse(outcomes[0].details_json!)).toMatchObject({ + state: 'failed', + finishedAt: failed.deployment.finished_at, + error: 'Observation interrupted' + }); + expect(resumed.events.some((event) => event.kind === 'resumed')).toBe(true); }); it('preserves a legacy cleanup failure and keeps deleted cleanup capability authoritative', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js'); - db().prepare(`INSERT INTO server_deployments (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by,error) - VALUES ('cleanup','old-server','2026-01-02','destroy','interrupted',?,'2026-01-01','2026-01-02','ui','App restarted')`).run(REMOVING_SERVER); - expect(getDeployment('cleanup')).toMatchObject({ canResumeCleanup: true, deployment: { server_exists: 0 } }); + const { getDeployment, resumeCleanup, REMOVING_SERVER } = + await import('../../src/lib/server/provisioning/history.js'); + db() + .prepare( + `INSERT INTO server_deployments (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by,error) + VALUES ('cleanup','old-server','2026-01-02','destroy','interrupted',?,'2026-01-01','2026-01-02','ui','App restarted')` + ) + .run(REMOVING_SERVER); + expect(getDeployment('cleanup')).toMatchObject({ + canResumeCleanup: true, + deployment: { server_exists: 0 } + }); expect(resumeCleanup('cleanup')).toBe(true); const result = getDeployment('cleanup')!; - expect(result.deployment).toMatchObject({ state: 'waiting', server_deleted_at: '2026-01-02' }); - const outcome = result.events.find(event => event.kind === 'outcome')!; - expect(JSON.parse(outcome.details_json!)).toMatchObject({ state: 'interrupted', finishedAt: '2026-01-02', error: 'App restarted' }); + expect(result.deployment).toMatchObject({ + state: 'waiting', + server_deleted_at: '2026-01-02' + }); + const outcome = result.events.find((event) => event.kind === 'outcome')!; + expect(JSON.parse(outcome.details_json!)).toMatchObject({ + state: 'interrupted', + finishedAt: '2026-01-02', + error: 'App restarted' + }); expect(result.progress?.phase).toBe('cleanup'); }); }); @@ -152,11 +279,15 @@ describe('provisioning detail projection', () => { describe('provisioning activity pagination', () => { it('keeps polling payloads bounded for a retained history with thousands of events', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); const id = beginDeployment('server', 'restart'); - const insert = db().prepare('INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES (?,?,?)'); + const insert = db().prepare( + 'INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES (?,?,?)' + ); db().transaction(() => { - for (let index = 0; index < 5000; index++) insert.run(id, '2026-09-28T12:00:00Z', `Observation ${index} ${'x'.repeat(250)}`); + for (let index = 0; index < 5000; index++) + insert.run(id, '2026-09-28T12:00:00Z', `Observation ${index} ${'x'.repeat(250)}`); })(); const full = getDeployment(id)!; const recent = getDeployment(id, { events: 'recent' })!; @@ -168,47 +299,70 @@ describe('provisioning activity pagination', () => { if (process.env.PROVISIONING_BENCHMARK) { const start = performance.now(); for (let index = 0; index < 30; index++) getDeployment(id, { events: 'recent' }); - console.info({ events: full.events.length, fullBytes, recentBytes, meanRecentReadMs: (performance.now() - start) / 30 }); + console.info({ + events: full.events.length, + fullBytes, + recentBytes, + meanRecentReadMs: (performance.now() - start) / 30 + }); } }); - it.each(['newest', 'oldest'] as const)('keeps stable %s pages under concurrent appends and searches literal text', async order => { - const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js'); - const { deploymentActivity } = await import('../../src/lib/server/provisioning/activity.js'); - const id = beginDeployment('server', 'stop'); - const other = beginDeployment('other', 'stop'); - for (let index = 0; index < 67; index++) { - recordDeploymentEvent(id, index === 30 ? 'literal %_needle' : `Event ${index}`); - recordDeploymentEvent(other, 'Unrelated event'); + it.each(['newest', 'oldest'] as const)( + 'keeps stable %s pages under concurrent appends and searches literal text', + async (order) => { + const { beginDeployment, recordDeploymentEvent } = + await import('../../src/lib/server/provisioning/history.js'); + const { deploymentActivity } = + await import('../../src/lib/server/provisioning/activity.js'); + const id = beginDeployment('server', 'stop'); + const other = beginDeployment('other', 'stop'); + for (let index = 0; index < 67; index++) { + recordDeploymentEvent(id, index === 30 ? 'literal %_needle' : `Event ${index}`); + recordDeploymentEvent(other, 'Unrelated event'); + } + let page = deploymentActivity(id, { order }); + const watermark = page.watermark; + const seen = [...page.events]; + expect(page.events).toHaveLength(25); + recordDeploymentEvent(id, 'New event during reading'); + while (page.nextCursor !== null) { + page = deploymentActivity(id, { order, watermark, cursor: page.nextCursor }); + seen.push(...page.events); + } + expect(seen).toHaveLength(68); + expect(new Set(seen.map((event) => event.id)).size).toBe(68); + expect(seen.every((event) => event.id <= watermark)).toBe(true); + expect(page.total).toBe(68); + expect(page.totalEvents).toBe(69); + expect( + deploymentActivity(id, { query: '%_NEEDLE' }).events.map((event) => event.message) + ).toEqual(['literal %_needle']); + expect(deploymentActivity(id, { query: 'not found' }).total).toBe(0); } - let page = deploymentActivity(id, { order }); - const watermark = page.watermark; - const seen = [...page.events]; - expect(page.events).toHaveLength(25); - recordDeploymentEvent(id, 'New event during reading'); - while (page.nextCursor !== null) { - page = deploymentActivity(id, { order, watermark, cursor: page.nextCursor }); - seen.push(...page.events); - } - expect(seen).toHaveLength(68); - expect(new Set(seen.map(event => event.id)).size).toBe(68); - expect(seen.every(event => event.id <= watermark)).toBe(true); - expect(page.total).toBe(68); - expect(page.totalEvents).toBe(69); - expect(deploymentActivity(id, { query: '%_NEEDLE' }).events.map(event => event.message)).toEqual(['literal %_needle']); - expect(deploymentActivity(id, { query: 'not found' }).total).toBe(0); - }); + ); it('validates cursor/search requests and caps page size without changing default legacy responses', async () => { - const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, recordDeploymentEvent } = + await import('../../src/lib/server/provisioning/history.js'); const { GET } = await import('../../src/routes/api/provisioning/[id]/events/+server.js'); const id = beginDeployment('server', 'stop'); for (let index = 0; index < 130; index++) recordDeploymentEvent(id, `Event ${index}`); const response = GET({ params: { id }, url: new URL('http://localhost/?limit=999') }); expect((await response.json()).events).toHaveLength(100); - for (const query of ['cursor=-1', 'watermark=NaN', 'limit=0', 'order=invalid', 'cursor=9007199254740992']) { - expect(() => GET({ params: { id }, url: new URL(`http://localhost/?${query}`) })).toThrow(); + for (const query of [ + 'cursor=-1', + 'watermark=NaN', + 'limit=0', + 'order=invalid', + 'cursor=9007199254740992' + ]) { + expect(() => + GET({ params: { id }, url: new URL(`http://localhost/?${query}`) }) + ).toThrow(); } - expect(() => GET({ params: { id: 'missing' }, url: new URL('http://localhost/') })).toThrow(); + expect(() => + GET({ params: { id: 'missing' }, url: new URL('http://localhost/') }) + ).toThrow(); }); }); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index 205e53e..9827b20 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -16,7 +16,8 @@ afterEach(async () => { describe('provisioning reads with retained lifecycle history', () => { it('uses indexed per-server submissions and separate lifecycle indexes for fleet reads', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { latestGameSubmission, latestGameSubmissions } = await import('../../src/lib/server/fleet/lifecycle-evidence.js'); + const { latestGameSubmission, latestGameSubmissions } = + await import('../../src/lib/server/fleet/lifecycle-evidence.js'); const { observationReader } = await import('../../src/lib/server/fleet/status.js'); const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); @@ -35,15 +36,28 @@ describe('provisioning reads with retained lifecycle history', () => { insert.run(`history-${i}`, `server-${i % 16}`, 'restart', at, at, at, null); } for (const [i, action] of ['create', 'start', 'restart', 'rebuild'].entries()) { - insert.run(`current-${i}`, `server-${i}`, action, iso(-1000), iso(-1000), iso(-1000), null); + insert.run( + `current-${i}`, + `server-${i}`, + action, + iso(-1000), + iso(-1000), + iso(-1000), + null + ); } insert.run('stop', 'server-0', 'stop', iso(0), iso(0), iso(0), null); insert.run('destroy', 'server-1', 'destroy', iso(0), iso(0), iso(0), null); insert.run('deleted', 'server-2', 'restart', iso(0), iso(0), iso(0), iso(0)); insert.run('unsubmitted', 'server-3', 'rebuild', iso(0), null, null, null); - handle.prepare('INSERT INTO restart_rollout(server, requested_at, healthy_at) VALUES (?, ?, ?)') + handle + .prepare( + 'INSERT INTO restart_rollout(server, requested_at, healthy_at) VALUES (?, ?, ?)' + ) .run('server-0', iso(-500), iso(0)); - handle.prepare('INSERT INTO restart_rollout(server, requested_at) VALUES (?, ?)').run('server-1', iso(-2000)); + handle + .prepare('INSERT INTO restart_rollout(server, requested_at) VALUES (?, ?)') + .run('server-1', iso(-2000)); })(); const submissions = latestGameSubmissions(); expect(submissions.get('empty')).toBeNull(); @@ -55,34 +69,58 @@ describe('provisioning reads with retained lifecycle history', () => { for (const read of [observationReader, runningVersions]) { const prepare = vi.spyOn(handle, 'prepare'); read(); - const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('submitted_at')); + const queries = prepare.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.includes('submitted_at')); prepare.mockRestore(); expect(queries).toHaveLength(1); - const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[]; - const details = plan.map(row => row.detail).join('\n'); - expect(details).toContain('SEARCH server_deployments USING COVERING INDEX idx_server_deployments_submission'); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { + detail: string; + }[]; + const details = plan.map((row) => row.detail).join('\n'); + expect(details).toContain( + 'SEARCH server_deployments USING COVERING INDEX idx_server_deployments_submission' + ); expect(details).not.toMatch(/SCAN server_deployments\b/); } const prepare = vi.spyOn(handle, 'prepare'); fleetOperations(now); - const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments')); + const queries = prepare.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.includes('server_deployments')); prepare.mockRestore(); expect(queries).toHaveLength(1); const statement = handle.prepare(queries[0]); - expect(statement.columns().map(column => column.name)).toEqual(['id', 'server', 'action', 'state', 'stage', 'error', 'started_at']); - const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[]; - const details = plan.map(row => row.detail).join('\n'); - expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_state'); + expect(statement.columns().map((column) => column.name)).toEqual([ + 'id', + 'server', + 'action', + 'state', + 'stage', + 'error', + 'started_at' + ]); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { + detail: string; + }[]; + const details = plan.map((row) => row.detail).join('\n'); + expect(details).toContain( + 'SEARCH server_deployments USING INDEX idx_server_deployments_state' + ); expect(queries[0]).not.toContain('finished_at >='); expect(details).toContain('SEARCH server_deployments USING INTEGER PRIMARY KEY (rowid=?)'); - expect(details).toContain('SCAN server_deployments USING COVERING INDEX idx_server_deployments_server'); + expect(details).toContain( + 'SCAN server_deployments USING COVERING INDEX idx_server_deployments_server' + ); expect(details).not.toMatch(/SCAN server_deployments(?:\n|$)/); }); it('indexes the server and queue reads used by collection matching and health polling', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { latestGameSubmission } = await import('../../src/lib/server/fleet/lifecycle-evidence.js'); - const { hasActiveProvisioning } = await import('../../src/lib/server/fleet/provisioning.js'); + const { latestGameSubmission } = + await import('../../src/lib/server/fleet/lifecycle-evidence.js'); + const { hasActiveProvisioning } = + await import('../../src/lib/server/fleet/provisioning.js'); const { housePartyTargets } = await import('../../src/lib/server/fleet/house-party.js'); const { creationQueue } = await import('../../src/lib/server/provisioning/instances.js'); const handle = db(); @@ -91,9 +129,13 @@ describe('provisioning reads with retained lifecycle history', () => { (id,server,action,state,stage,started_at,triggered_by,server_deleted_at) VALUES(?,?,'create','done','Complete','2025-01-01','fixture','2025-01-02')`); for (let i = 0; i < 2_000; i++) insert.run(`history-${i}`, `retired-${i}`); - handle.prepare(`INSERT INTO server_deployments + handle + .prepare( + `INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,submitted_at) - VALUES('current','current','create','queued','Queued','2026-01-01','fixture','2026-01-01')`).run(); + VALUES('current','current','create','queued','Queued','2026-01-01','fixture','2026-01-01')` + ) + .run(); })(); // Inspect the SQL issued by the public reads, so the regression follows @@ -101,44 +143,73 @@ describe('provisioning reads with retained lifecycle history', () => { const expectIndexed = (read: () => unknown, bindings: string[] = []) => { const prepare = vi.spyOn(handle, 'prepare'); read(); - const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments')); + const queries = prepare.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.includes('server_deployments')); prepare.mockRestore(); expect(queries).toHaveLength(1); - const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(...bindings) as { detail: string }[]; - const details = plan.map(row => row.detail).join('\n'); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(...bindings) as { + detail: string; + }[]; + const details = plan.map((row) => row.detail).join('\n'); expect(details).toMatch(/SEARCH server_deployments USING (?:COVERING )?INDEX/); expect(details).not.toMatch(/SCAN server_deployments\b/); }; - expectIndexed(() => expect(latestGameSubmission('current')).toBe('2026-01-01'), ['current', 'current']); + expectIndexed( + () => expect(latestGameSubmission('current')).toBe('2026-01-01'), + ['current', 'current'] + ); expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']); expectIndexed(() => expect(housePartyTargets()).toEqual(['current'])); - expectIndexed(() => expect(creationQueue().queued.map(job => job.id)).toEqual(['current'])); + expectIndexed(() => + expect(creationQueue().queued.map((job) => job.id)).toEqual(['current']) + ); }); it('installs the indexes after legacy table rebuilds and keeps them on reopen', async () => { const legacy = new Database(fixture.dbPath); try { - legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, table => table - .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'") - .replace(' server_deleted_at TEXT,\n', ''))); + legacy.exec( + schema.replace( + /CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, + (table) => + table + .replace( + "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", + "'create', 'destroy'" + ) + .replace(' server_deleted_at TEXT,\n', '') + ) + ); legacy.exec(`INSERT INTO servers(name) VALUES('existing'); INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('old','existing','create','done','Complete','2025-01-01','fixture'); INSERT INTO server_deployment_events(deployment_id,created_at,message) VALUES('old','2025-01-01','Creation complete');`); - } finally { legacy.close(); } + } finally { + legacy.close(); + } const { db, closeDb } = await import('../../src/lib/server/db/index.js'); for (let open = 0; open < 2; open++) { const handle = db(); - expect(handle.prepare('SELECT rowid,id,state FROM server_deployments').all()) - .toEqual([{ rowid: 1, id: 'old', state: 'done' }]); - expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([{ message: 'Creation complete' }]); + expect(handle.prepare('SELECT rowid,id,state FROM server_deployments').all()).toEqual([ + { rowid: 1, id: 'old', state: 'done' } + ]); + expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([ + { message: 'Creation complete' } + ]); const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[]; - expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([ - 'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at', - 'idx_server_deployments_submission', 'idx_server_deployments_finished_at', - 'idx_server_deployments_server_state', 'idx_server_deployments_unsettled' - ])); + expect(indexes.map((index) => index.name)).toEqual( + expect.arrayContaining([ + 'idx_server_deployments_server', + 'idx_server_deployments_state', + 'idx_server_deployments_started_at', + 'idx_server_deployments_submission', + 'idx_server_deployments_finished_at', + 'idx_server_deployments_server_state', + 'idx_server_deployments_unsettled' + ]) + ); expect(handle.pragma('foreign_key_check')).toEqual([]); closeDb(); } diff --git a/tests/unit/provisioning-list.test.ts b/tests/unit/provisioning-list.test.ts index 826e9de..e4f2b57 100644 --- a/tests/unit/provisioning-list.test.ts +++ b/tests/unit/provisioning-list.test.ts @@ -1,10 +1,21 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { provisioningListSources, provisioningMatchesSource, provisioningSourceLabel, type ProvisioningListSource } from '../../src/lib/provisioning-list.js'; +import { + provisioningListSources, + provisioningMatchesSource, + provisioningSourceLabel, + type ProvisioningListSource +} from '../../src/lib/provisioning-list.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); @@ -13,9 +24,25 @@ async function setup() { const insert = handle.prepare(`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,server_deleted_at,observation_json) VALUES(?,?,?,?,?,?,'fixture',?,?)`); - const add = (id: string, state = 'done', action = 'create', server = 't5_zombie_theater_1', - deleted: string | null = null, stage = 'Recorded', at = '2026-01-01') => - insert.run(id, server, action, state, stage, at, deleted, JSON.stringify({ snapshot: 'x'.repeat(4096) })); + const add = ( + id: string, + state = 'done', + action = 'create', + server = 't5_zombie_theater_1', + deleted: string | null = null, + stage = 'Recorded', + at = '2026-01-01' + ) => + insert.run( + id, + server, + action, + state, + stage, + at, + deleted, + JSON.stringify({ snapshot: 'x'.repeat(4096) }) + ); return { ...history, handle, add }; } @@ -23,16 +50,35 @@ describe('bounded provisioning lists', () => { it('keeps recovery sources concise as incident summaries change, including after retention', async () => { const { add, handle, deploymentListPage } = await setup(); add('recovery', 'waiting', 'rebuild'); - handle.prepare("INSERT INTO provisioning_sources VALUES ('recovery','watchdog','incident','t5_zombie_theater_1')").run(); - handle.prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) - VALUES ('incident','t5_zombie_theater_1','health','recovering',?,'2026-01-01','2026-01-01')`) + handle + .prepare( + "INSERT INTO provisioning_sources VALUES ('recovery','watchdog','incident','t5_zombie_theater_1')" + ) + .run(); + handle + .prepare( + `INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) + VALUES ('incident','t5_zombie_theater_1','health','recovering',?,'2026-01-01','2026-01-01')` + ) .run('Rebuild queued with the latest promoted setup'); - for (const summary of ['Rebuild queued with the latest promoted setup', 'Recovered game service']) { - handle.prepare('UPDATE watchdog_incidents SET summary=? WHERE id=?').run(summary, 'incident'); + for (const summary of [ + 'Rebuild queued with the latest promoted setup', + 'Recovered game service' + ]) { + handle + .prepare('UPDATE watchdog_incidents SET summary=? WHERE id=?') + .run(summary, 'incident'); const row = deploymentListPage({ view: 'active' }).deployments[0]; expect(provisioningSourceLabel(row)).toBe('Recovery'); - expect(row.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', label: summary, href: '/maintenance/incident' }]); + expect(row.sources).toEqual([ + { + kind: 'watchdog', + source_id: 'incident', + label: summary, + href: '/maintenance/incident' + } + ]); } handle.prepare("DELETE FROM watchdog_incidents WHERE id='incident'").run(); const retained = deploymentListPage({ view: 'active' }).deployments[0]; @@ -40,58 +86,102 @@ describe('bounded provisioning lists', () => { expect(retained.sources?.[0].href).toBeNull(); }); - it.each(['active', 'history'] as const)('keeps %s source categories and client filters aligned with API filters', async view => { - const { add, handle, deploymentListPage } = await setup(); - const categories: [ProvisioningListSource, string, string][] = [ - ['campaign', 'campaign', 'Campaign'], ['beta', 'beta', 'Beta'], ['maintenance', 'maintenance', 'Maintenance'], - ['recovery', 'watchdog', 'Recovery'], ['autoscaling', 'autoscaling', 'Autoscaling'], ['rebalance', 'rebalance', 'Rebalance'] - ]; - const state = view === 'active' ? 'waiting' : 'done'; - for (const [source, kind] of categories) { - // Rebalance ownership is recorded only in triggered_by. - if (source !== 'rebalance') { - add(`${source}-linked`, state); - handle.prepare('INSERT INTO provisioning_sources VALUES (?,?,?,?)').run(`${source}-linked`, kind, 'expired', 't5_zombie_theater_1'); + it.each(['active', 'history'] as const)( + 'keeps %s source categories and client filters aligned with API filters', + async (view) => { + const { add, handle, deploymentListPage } = await setup(); + const categories: [ProvisioningListSource, string, string][] = [ + ['campaign', 'campaign', 'Campaign'], + ['beta', 'beta', 'Beta'], + ['maintenance', 'maintenance', 'Maintenance'], + ['recovery', 'watchdog', 'Recovery'], + ['autoscaling', 'autoscaling', 'Autoscaling'], + ['rebalance', 'rebalance', 'Rebalance'] + ]; + const state = view === 'active' ? 'waiting' : 'done'; + for (const [source, kind] of categories) { + // Rebalance ownership is recorded only in triggered_by. + if (source !== 'rebalance') { + add(`${source}-linked`, state); + handle + .prepare('INSERT INTO provisioning_sources VALUES (?,?,?,?)') + .run(`${source}-linked`, kind, 'expired', 't5_zombie_theater_1'); + } + add(`${source}-legacy`, state); + handle + .prepare('UPDATE server_deployments SET triggered_by=? WHERE id=?') + .run(`${kind}:expired`, `${source}-legacy`); } - add(`${source}-legacy`, state); - handle.prepare('UPDATE server_deployments SET triggered_by=? WHERE id=?').run(`${kind}:expired`, `${source}-legacy`); - } - add('manual', state); - handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'").run(); - add('system', state); - // A shared operation must be discoverable through each of its sources, even when initiated manually. - add('shared', state); - handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='shared'").run(); - handle.exec(`INSERT INTO provisioning_sources VALUES ('shared','campaign','expired','t5_zombie_theater_1'); + add('manual', state); + handle + .prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'") + .run(); + add('system', state); + // A shared operation must be discoverable through each of its sources, even when initiated manually. + add('shared', state); + handle + .prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='shared'") + .run(); + handle.exec(`INSERT INTO provisioning_sources VALUES ('shared','campaign','expired','t5_zombie_theater_1'); INSERT INTO provisioning_sources VALUES ('shared','watchdog','expired','t5_zombie_theater_1');`); - const rows = deploymentListPage({ view }).deployments; - expect(rows).toHaveLength(14); - for (const [source, , label] of categories) { - for (const suffix of source === 'rebalance' ? ['legacy'] : ['linked', 'legacy']) { - expect(provisioningSourceLabel(rows.find(row => row.id === `${source}-${suffix}`)!)).toBe(label); + const rows = deploymentListPage({ view }).deployments; + expect(rows).toHaveLength(14); + for (const [source, , label] of categories) { + for (const suffix of source === 'rebalance' ? ['legacy'] : ['linked', 'legacy']) { + expect( + provisioningSourceLabel( + rows.find((row) => row.id === `${source}-${suffix}`)! + ) + ).toBe(label); + } } - } - expect(provisioningSourceLabel(rows.find(row => row.id === 'manual')!)).toBe('Manual'); - expect(provisioningSourceLabel(rows.find(row => row.id === 'system')!)).toBe('System'); + expect(provisioningSourceLabel(rows.find((row) => row.id === 'manual')!)).toBe( + 'Manual' + ); + expect(provisioningSourceLabel(rows.find((row) => row.id === 'system')!)).toBe( + 'System' + ); - const { GET } = await import('../../src/routes/api/provisioning/+server.js'); - for (const source of provisioningListSources) { - const expected = source === 'manual' || source === 'system' ? [source] - : source === 'rebalance' ? ['rebalance-legacy'] - : [`${source}-linked`, `${source}-legacy`, ...(['campaign', 'recovery'].includes(source) ? ['shared'] : [])]; - expect(rows.filter(row => provisioningMatchesSource(row, source)).map(row => row.id).sort()).toEqual(expected.sort()); - const result = await GET({ url: new URL(`http://localhost/api/provisioning?view=${view}&source=${source}`) }).json(); - expect(result.total).toBe(expected.length); - expect(result.deployments.map((row: { id: string }) => row.id).sort()).toEqual(expected); - if (view === 'history') { - const page = deploymentListPage({ view, source, limit: 1, offset: expected.length - 1 }); - expect(page.total).toBe(expected.length); - expect(page.deployments).toHaveLength(1); - expect(provisioningMatchesSource(page.deployments[0], source)).toBe(true); + const { GET } = await import('../../src/routes/api/provisioning/+server.js'); + for (const source of provisioningListSources) { + const expected = + source === 'manual' || source === 'system' + ? [source] + : source === 'rebalance' + ? ['rebalance-legacy'] + : [ + `${source}-linked`, + `${source}-legacy`, + ...(['campaign', 'recovery'].includes(source) ? ['shared'] : []) + ]; + expect( + rows + .filter((row) => provisioningMatchesSource(row, source)) + .map((row) => row.id) + .sort() + ).toEqual(expected.sort()); + const result = await GET({ + url: new URL(`http://localhost/api/provisioning?view=${view}&source=${source}`) + }).json(); + expect(result.total).toBe(expected.length); + expect(result.deployments.map((row: { id: string }) => row.id).sort()).toEqual( + expected + ); + if (view === 'history') { + const page = deploymentListPage({ + view, + source, + limit: 1, + offset: expected.length - 1 + }); + expect(page.total).toBe(expected.length); + expect(page.deployments).toHaveLength(1); + expect(provisioningMatchesSource(page.deployments[0], source)).toBe(true); + } } } - }); + ); it('filters readable sources before pagination and retains expired source references without broken links', async () => { const { add, handle, deploymentListPage } = await setup(); @@ -100,120 +190,344 @@ describe('bounded provisioning lists', () => { INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,notes) VALUES(75,1,1,'ui','promoted','Gameplay checks'); INSERT INTO deployments(id,action,started_at,beta_id) VALUES('campaign','promote','2026-01-01',75);`); add('old-campaign', 'done', 'restart'); - handle.prepare("UPDATE server_deployments SET triggered_by='campaign:campaign' WHERE id='old-campaign'").run(); + handle + .prepare( + "UPDATE server_deployments SET triggered_by='campaign:campaign' WHERE id='old-campaign'" + ) + .run(); add('linked-beta', 'waiting', 'restart'); - handle.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('linked-beta','beta','75','t5_zombie_theater_1')").run(); + handle + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('linked-beta','beta','75','t5_zombie_theater_1')" + ) + .run(); add('expired', 'done', 'restart'); - handle.prepare("UPDATE server_deployments SET triggered_by='campaign:expired' WHERE id='expired'").run(); + handle + .prepare( + "UPDATE server_deployments SET triggered_by='campaign:expired' WHERE id='expired'" + ) + .run(); add('manual', 'done', 'restart'); handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'").run(); for (let i = 0; i < 30; i++) add(`system-${i}`, 'done', 'restart'); - const first = deploymentListPage({ view: 'history', source: 'campaign', action: 'restart', status: 'done', server: 'Kino', limit: 1 }); + const first = deploymentListPage({ + view: 'history', + source: 'campaign', + action: 'restart', + status: 'done', + server: 'Kino', + limit: 1 + }); expect(first.total).toBe(2); - expect(first.deployments[0]).toMatchObject({ id: 'expired', sources: [{ label: 'Campaign unavailable', href: null }] }); - const second = deploymentListPage({ view: 'history', source: 'campaign', limit: 1, offset: 1 }); - expect(second.deployments[0]).toMatchObject({ id: 'old-campaign', sources: [{ label: 'Promotion · Beta #75', href: '/campaigns/campaign' }] }); + expect(first.deployments[0]).toMatchObject({ + id: 'expired', + sources: [{ label: 'Campaign unavailable', href: null }] + }); + const second = deploymentListPage({ + view: 'history', + source: 'campaign', + limit: 1, + offset: 1 + }); + expect(second.deployments[0]).toMatchObject({ + id: 'old-campaign', + sources: [{ label: 'Promotion · Beta #75', href: '/campaigns/campaign' }] + }); const beta = deploymentListPage({ view: 'active', source: 'beta', status: 'pending' }); - expect(beta.deployments).toEqual([expect.objectContaining({ id: 'linked-beta', sources: [{ kind: 'beta', source_id: '75', label: 'Beta #75: Gameplay checks', href: '/betas/75' }] })]); + expect(beta.deployments).toEqual([ + expect.objectContaining({ + id: 'linked-beta', + sources: [ + { + kind: 'beta', + source_id: '75', + label: 'Beta #75: Gameplay checks', + href: '/betas/75' + } + ] + }) + ]); expect(provisioningSourceLabel(second.deployments[0])).toBe('Campaign'); expect(provisioningSourceLabel(beta.deployments[0])).toBe('Beta'); - expect(deploymentListPage({ view: 'history', source: 'manual' }).deployments.map(row => row.id)).toEqual(['manual']); + expect( + deploymentListPage({ view: 'history', source: 'manual' }).deployments.map( + (row) => row.id + ) + ).toEqual(['manual']); expect(deploymentListPage({ view: 'history', source: 'system' }).total).toBe(30); const { GET } = await import('../../src/routes/api/provisioning/+server.js'); - const result = await GET({ url: new URL('http://localhost/api/provisioning?view=active&status=pending&source=beta') }).json(); + const result = await GET({ + url: new URL('http://localhost/api/provisioning?view=active&status=pending&source=beta') + }).json(); expect(result.deployments.map((row: { id: string }) => row.id)).toEqual(['linked-beta']); }); it('counts unresolved outcomes while retaining resolved failures and routine cancellations in history', async () => { const { add, handle, deploymentListPage } = await setup(); - add('failed', 'failed'); add('resolved', 'failed'); add('cancelled', 'cancelled'); add('warning', 'warning'); - handle.prepare("INSERT INTO retention_controls(category,record_id,resolved_at) VALUES('provisioning','resolved','2026-01-02')").run(); + add('failed', 'failed'); + add('resolved', 'failed'); + add('cancelled', 'cancelled'); + add('warning', 'warning'); + handle + .prepare( + "INSERT INTO retention_controls(category,record_id,resolved_at) VALUES('provisioning','resolved','2026-01-02')" + ) + .run(); expect(deploymentListPage({ view: 'history' }).total).toBe(4); const attention = deploymentListPage({ view: 'history', status: 'attention' }); expect(attention.needsReview).toBe(2); - expect(attention.deployments.map(row => row.id)).toEqual(['warning', 'failed']); + expect(attention.deployments.map((row) => row.id)).toEqual(['warning', 'failed']); expect(deploymentListPage({ view: 'history', status: 'failed' }).total).toBe(2); - handle.prepare("UPDATE server_deployments SET finished_at='2026-01-03' WHERE id='resolved'").run(); + handle + .prepare("UPDATE server_deployments SET finished_at='2026-01-03' WHERE id='resolved'") + .run(); expect(deploymentListPage({ view: 'history', status: 'attention' }).needsReview).toBe(3); }); it('exposes compact waiting facts without transferring observation snapshots', async () => { const { add, handle, deploymentListPage } = await setup(); add('waiting', 'waiting', 'restart'); - handle.prepare("UPDATE server_deployments SET phase_key='eligibility',phase_started_at='2026-01-01',observation_json=? WHERE id='waiting'") - .run(JSON.stringify({ blocker: { code: 'restart-capacity' }, checks: ['private snapshot'], checkpoints: [] })); - expect(deploymentListPage({ view: 'active' }).deployments[0]).toMatchObject({ waitCode: 'restart-capacity', phase: 'eligibility', phaseStartedAt: '2026-01-01' }); - expect(JSON.stringify(deploymentListPage({ view: 'active' }))).not.toContain('private snapshot'); + handle + .prepare( + "UPDATE server_deployments SET phase_key='eligibility',phase_started_at='2026-01-01',observation_json=? WHERE id='waiting'" + ) + .run( + JSON.stringify({ + blocker: { code: 'restart-capacity' }, + checks: ['private snapshot'], + checkpoints: [] + }) + ); + expect(deploymentListPage({ view: 'active' }).deployments[0]).toMatchObject({ + waitCode: 'restart-capacity', + phase: 'eligibility', + phaseStartedAt: '2026-01-01' + }); + expect(JSON.stringify(deploymentListPage({ view: 'active' }))).not.toContain( + 'private snapshot' + ); }); it('puts verification and executing work ahead of preflight waits and capacity queues', async () => { const { add, handle, listDeployments, deploymentListPage } = await setup(); - const addProgress = (id: string, state: string, second: number, phase: string, - waitCode: string | null = null, stage = 'Recorded', action = 'rebuild') => { - add(id, state, action, 't5_zombie_theater_1', null, stage, `2026-01-01T00:00:${String(second).padStart(2, '0')}.000Z`); - handle.prepare('UPDATE server_deployments SET phase_key=?, observation_json=? WHERE id=?') + const addProgress = ( + id: string, + state: string, + second: number, + phase: string, + waitCode: string | null = null, + stage = 'Recorded', + action = 'rebuild' + ) => { + add( + id, + state, + action, + 't5_zombie_theater_1', + null, + stage, + `2026-01-01T00:00:${String(second).padStart(2, '0')}.000Z` + ); + handle + .prepare('UPDATE server_deployments SET phase_key=?, observation_json=? WHERE id=?') .run(phase, JSON.stringify({ blocker: waitCode ? { code: waitCode } : null }), id); }; // Request order is deliberately different from progress order. addProgress('queued', 'queued', 1, 'requested'); - addProgress('cleanup', 'waiting', 12, 'cleanup', null, 'Removing server files and registry', 'destroy'); + addProgress( + 'cleanup', + 'waiting', + 12, + 'cleanup', + null, + 'Removing server files and registry', + 'destroy' + ); addProgress('verify-newer', 'awaiting-health', 21, 'verify-map'); addProgress('capacity', 'running', 2, 'startup-queue', 'startup-capacity'); addProgress('operation-conflict', 'running', 6, 'container-action', 'operation-conflict'); - addProgress('agent-wait', 'running', 11, 'container-action', 'native-host', 'Agent operation pending'); - addProgress('host-wait', 'waiting', 5, 'eligibility', 'native-host', 'Host is unavailable; waiting for reconnection.'); - addProgress('staging', 'waiting', 7, 'eligibility', 'native-host', 'Staging and verifying files on the host.'); - addProgress('staging-wait', 'waiting', 8, 'eligibility', 'native-host', 'Waiting for verified remote file staging.'); - addProgress('legacy-capacity', 'waiting', 3, 'eligibility', 'native-host', 'Queued to rebuild; waiting for another server to recover'); - addProgress('verify-older', 'awaiting-health', 20, 'verify-container', 'native-host', 'Waiting for container readiness: starting'); + addProgress( + 'agent-wait', + 'running', + 11, + 'container-action', + 'native-host', + 'Agent operation pending' + ); + addProgress( + 'host-wait', + 'waiting', + 5, + 'eligibility', + 'native-host', + 'Host is unavailable; waiting for reconnection.' + ); + addProgress( + 'staging', + 'waiting', + 7, + 'eligibility', + 'native-host', + 'Staging and verifying files on the host.' + ); + addProgress( + 'staging-wait', + 'waiting', + 8, + 'eligibility', + 'native-host', + 'Waiting for verified remote file staging.' + ); + addProgress( + 'legacy-capacity', + 'waiting', + 3, + 'eligibility', + 'native-host', + 'Queued to rebuild; waiting for another server to recover' + ); + addProgress( + 'verify-older', + 'awaiting-health', + 20, + 'verify-container', + 'native-host', + 'Waiting for container readiness: starting' + ); addProgress('preparing-queued', 'queued', 9, 'prepare'); addProgress('executing', 'running', 10, 'prepare'); addProgress('eligibility', 'running', 4, 'eligibility', 'eligibility'); addProgress('interrupted', 'interrupted', 30, 'container-action', 'native-host'); addProgress('failed', 'failed', 31, 'verify-map', 'native-host'); - const expected = ['verify-older', 'verify-newer', 'staging', 'staging-wait', 'preparing-queued', 'executing', 'agent-wait', 'cleanup', - 'eligibility', 'host-wait', 'operation-conflict', 'queued', 'capacity', 'legacy-capacity']; - expect(listDeployments({ view: 'active' }).map(row => row.id)).toEqual(expected); + const expected = [ + 'verify-older', + 'verify-newer', + 'staging', + 'staging-wait', + 'preparing-queued', + 'executing', + 'agent-wait', + 'cleanup', + 'eligibility', + 'host-wait', + 'operation-conflict', + 'queued', + 'capacity', + 'legacy-capacity' + ]; + expect(listDeployments({ view: 'active' }).map((row) => row.id)).toEqual(expected); const { GET } = await import('../../src/routes/api/provisioning/+server.js'); - const active = await GET({ url: new URL('http://localhost/api/provisioning?view=active&limit=1') }).json(); + const active = await GET({ + url: new URL('http://localhost/api/provisioning?view=active&limit=1') + }).json(); expect(active.deployments.map((row: { id: string }) => row.id)).toEqual(expected); expect(active).toMatchObject({ total: expected.length, needsReview: 2 }); // Progress priority applies only to Active; retained outcomes and mixed reads stay newest first. - expect(deploymentListPage({ view: 'history', limit: 1 }).deployments.map(row => row.id)).toEqual(['failed']); - expect(deploymentListPage({ view: 'history', limit: 1, offset: 1 }).deployments.map(row => row.id)).toEqual(['interrupted']); - expect(listDeployments({ view: 'all', limit: 3 }).map(row => row.id)).toEqual(['failed', 'interrupted', 'verify-newer']); + expect( + deploymentListPage({ view: 'history', limit: 1 }).deployments.map((row) => row.id) + ).toEqual(['failed']); + expect( + deploymentListPage({ view: 'history', limit: 1, offset: 1 }).deployments.map( + (row) => row.id + ) + ).toEqual(['interrupted']); + expect(listDeployments({ view: 'all', limit: 3 }).map((row) => row.id)).toEqual([ + 'failed', + 'interrupted', + 'verify-newer' + ]); }); it('keeps oldest requests and insertion ties stable while active progress is polled', async () => { const { add, handle, listDeployments } = await setup(); - add('z-first', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking game status', '2026-01-02'); - add('a-second', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking selected engine map', '2026-01-02'); - add('old-inserted-last', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking container state', '2026-01-01'); - add('queued', 'queued', 'rebuild', 't5_zombie_theater_1', null, 'Request recorded', '2025-12-31'); - const ids = () => listDeployments({ view: 'active' }).map(row => row.id); + add( + 'z-first', + 'awaiting-health', + 'rebuild', + 't5_zombie_theater_1', + null, + 'Checking game status', + '2026-01-02' + ); + add( + 'a-second', + 'awaiting-health', + 'rebuild', + 't5_zombie_theater_1', + null, + 'Checking selected engine map', + '2026-01-02' + ); + add( + 'old-inserted-last', + 'awaiting-health', + 'rebuild', + 't5_zombie_theater_1', + null, + 'Checking container state', + '2026-01-01' + ); + add( + 'queued', + 'queued', + 'rebuild', + 't5_zombie_theater_1', + null, + 'Request recorded', + '2025-12-31' + ); + const ids = () => listDeployments({ view: 'active' }).map((row) => row.id); const expected = ['old-inserted-last', 'z-first', 'a-second', 'queued']; expect(ids()).toEqual(expected); expect(ids()).toEqual(expected); - handle.prepare(`UPDATE server_deployments SET phase_key='verify-container', phase_started_at='2026-01-03', - observed_at='2026-01-03', observation_json=? WHERE id='z-first'`) - .run(JSON.stringify({ blocker: { code: 'native-host', message: 'Waiting for container readiness: starting' } })); - handle.prepare("UPDATE server_deployments SET phase_key='verify-plutonium', phase_started_at='2026-01-04' WHERE id='a-second'").run(); + handle + .prepare( + `UPDATE server_deployments SET phase_key='verify-container', phase_started_at='2026-01-03', + observed_at='2026-01-03', observation_json=? WHERE id='z-first'` + ) + .run( + JSON.stringify({ + blocker: { + code: 'native-host', + message: 'Waiting for container readiness: starting' + } + }) + ); + handle + .prepare( + "UPDATE server_deployments SET phase_key='verify-plutonium', phase_started_at='2026-01-04' WHERE id='a-second'" + ) + .run(); expect(ids()).toEqual(expected); - handle.prepare("UPDATE server_deployments SET state='done', finished_at='2026-01-05' WHERE id='z-first'").run(); - handle.prepare("UPDATE server_deployments SET state='awaiting-health', phase_key='verify-game' WHERE id='queued'").run(); + handle + .prepare( + "UPDATE server_deployments SET state='done', finished_at='2026-01-05' WHERE id='z-first'" + ) + .run(); + handle + .prepare( + "UPDATE server_deployments SET state='awaiting-health', phase_key='verify-game' WHERE id='queued'" + ) + .run(); expect(ids()).toEqual(['queued', 'old-inserted-last', 'a-second']); - expect(listDeployments({ view: 'all' }).map(row => row.id)).toEqual(['a-second', 'z-first', 'old-inserted-last', 'queued']); + expect(listDeployments({ view: 'all' }).map((row) => row.id)).toEqual([ + 'a-second', + 'z-first', + 'old-inserted-last', + 'queued' + ]); }); it('polls every active row and counts review outcomes without transferring retained history', async () => { const { handle, add, REMOVING_SERVER } = await setup(); handle.transaction(() => { for (let i = 0; i < 6_000; i++) add(`history-${i}`, i % 2 ? 'done' : 'failed'); - for (const state of ['queued', 'awaiting-health', 'running', 'waiting']) add(state, state); + for (const state of ['queued', 'awaiting-health', 'running', 'waiting']) + add(state, state); for (let i = 0; i < 110; i++) add(`active-${i}`, 'waiting', 'destroy'); add('cleanup', 'waiting', 'destroy', 'cleanup', null, REMOVING_SERVER); add('deleted-active', 'running', 'create', 'retired', '2026-01-02'); @@ -221,25 +535,37 @@ describe('bounded provisioning lists', () => { add('interrupted', 'interrupted'); })(); const { GET } = await import('../../src/routes/api/provisioning/+server.js'); - const response = GET({ url: new URL('http://localhost/api/provisioning?view=active&limit=1') }); + const response = GET({ + url: new URL('http://localhost/api/provisioning?view=active&limit=1') + }); const body = await response.json(); expect(body.total).toBe(115); expect(body.deployments).toHaveLength(115); expect(body.needsReview).toBe(3_002); - expect(body.deployments.map((row: { id: string }) => row.id)).not.toContain('deleted-active'); - expect(body.deployments.find((row: { id: string }) => row.id === 'active-0').canCancelRemoval).toBe(1); - expect(body.deployments.find((row: { id: string }) => row.id === 'cleanup').canCancelRemoval).toBe(0); + expect(body.deployments.map((row: { id: string }) => row.id)).not.toContain( + 'deleted-active' + ); + expect( + body.deployments.find((row: { id: string }) => row.id === 'active-0').canCancelRemoval + ).toBe(1); + expect( + body.deployments.find((row: { id: string }) => row.id === 'cleanup').canCancelRemoval + ).toBe(0); expect(JSON.stringify(body)).not.toContain('observation_json'); expect(Buffer.byteLength(JSON.stringify(body))).toBeLessThan(60_000); expect(response.headers.get('cache-control')).toBe('no-store'); - const history = await GET({ url: new URL('http://localhost/api/provisioning?view=history&limit=100') }).json(); + const history = await GET({ + url: new URL('http://localhost/api/provisioning?view=history&limit=100') + }).json(); expect(history.deployments).toHaveLength(100); expect(history.total).toBe(6_003); expect(Buffer.byteLength(JSON.stringify(history))).toBeLessThan(50_000); // The legacy endpoint must not reintroduce the unbounded response. const { GET: legacyGET } = await import('../../src/routes/api/deployments/+server.js'); - expect(await legacyGET({ url: new URL('http://localhost/api/deployments') }).json()).toEqual({ ...body, limit: 25 }); + expect( + await legacyGET({ url: new URL('http://localhost/api/deployments') }).json() + ).toEqual({ ...body, limit: 25 }); }); it('paginates in a stable order and applies combined filters before the limit', async () => { @@ -258,16 +584,26 @@ describe('bounded provisioning lists', () => { expect(page.deployments[0].id).toBe('deleted-active'); const next = deploymentListPage({ view: 'history', limit: 100, offset: 100 }); expect(next.deployments).toHaveLength(35); - expect(new Set([...page.deployments, ...next.deployments].map(row => row.id)).size).toBe(135); + expect(new Set([...page.deployments, ...next.deployments].map((row) => row.id)).size).toBe( + 135 + ); expect(next.deployments.at(-1)?.id).toBe('old-but-inserted-last'); expect(page.deployments[0]).not.toHaveProperty('observation_json'); expect(page.deployments[0]).not.toHaveProperty('phase_key'); const { GET } = await import('../../src/routes/api/provisioning/+server.js'); - const filtered = await GET({ url: new URL('http://localhost/api/provisioning?view=history&action=restart&status=attention&server=KINO&limit=1') }).json(); + const filtered = await GET({ + url: new URL( + 'http://localhost/api/provisioning?view=history&action=restart&status=attention&server=KINO&limit=1' + ) + }).json(); expect(filtered.total).toBe(2); expect(filtered.needsReview).toBe(3); - expect(filtered.deployments).toEqual([expect.objectContaining({ id: 'warning', server_exists: 0 })]); - expect(deploymentListPage({ view: 'history', server: 'Lobby 1', action: 'restart' }).total).toBe(3); + expect(filtered.deployments).toEqual([ + expect.objectContaining({ id: 'warning', server_exists: 0 }) + ]); + expect( + deploymentListPage({ view: 'history', server: 'Lobby 1', action: 'restart' }).total + ).toBe(3); expect(deploymentListPage({ view: 'history', server: "%' OR 1=1 --" }).total).toBe(0); }); @@ -281,7 +617,9 @@ describe('bounded provisioning lists', () => { ['limit=-1&offset=1.9', 1, 1], ['limit=0&offset=Infinity', 25, 0] ] as const) { - const body = await GET({ url: new URL(`http://localhost/api/provisioning?view=history&${query}`) }).json(); + const body = await GET({ + url: new URL(`http://localhost/api/provisioning?view=history&${query}`) + }).json(); expect(body).toMatchObject({ limit, offset, total: 120 }); expect(body.deployments).toHaveLength(limit); } @@ -296,10 +634,16 @@ describe('bounded provisioning lists', () => { const queries = prepare.mock.calls.map(([sql]) => sql); prepare.mockRestore(); const explain = (sql: string, bindings: (string | number)[]) => - (handle.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...bindings) as { detail: string }[]).map(row => row.detail).join('\n'); - const active = queries.find(sql => sql.includes('AS canCancelRemoval') && !sql.includes('LIMIT'))!; - const history = queries.find(sql => sql.includes('AS canCancelRemoval') && sql.includes('LIMIT'))!; - const review = queries.find(sql => sql.includes('COUNT(*) AS needsReview'))!; + (handle.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...bindings) as { detail: string }[]) + .map((row) => row.detail) + .join('\n'); + const active = queries.find( + (sql) => sql.includes('AS canCancelRemoval') && !sql.includes('LIMIT') + )!; + const history = queries.find( + (sql) => sql.includes('AS canCancelRemoval') && sql.includes('LIMIT') + )!; + const review = queries.find((sql) => sql.includes('COUNT(*) AS needsReview'))!; expect(explain(active, [REMOVING_SERVER])).toContain('idx_server_deployments_state'); expect(explain(review, [])).toContain('idx_server_deployments_state'); const historyPlan = explain(history, [REMOVING_SERVER, 100, 0]); diff --git a/tests/unit/provisioning-presentation.test.ts b/tests/unit/provisioning-presentation.test.ts index 0c069f0..698f526 100644 --- a/tests/unit/provisioning-presentation.test.ts +++ b/tests/unit/provisioning-presentation.test.ts @@ -1,33 +1,88 @@ import { describe, expect, it } from 'vitest'; import { render } from 'svelte/server'; -import type { ProvisioningDetail, ProvisioningProgress } from '../../src/lib/provisioning-detail.js'; +import type { + ProvisioningDetail, + ProvisioningProgress +} from '../../src/lib/provisioning-detail.js'; import { provisioningActions } from '../../src/lib/provisioning.js'; -import { isProvisioningDetail, isProvisioningEventPage, provisioningGroups, provisioningOutcome, provisioningRequester, provisioningReturn } from '../../src/lib/ui/provisioning-presentation.js'; +import { + isProvisioningDetail, + isProvisioningEventPage, + provisioningGroups, + provisioningOutcome, + provisioningRequester, + provisioningReturn +} from '../../src/lib/ui/provisioning-presentation.js'; import ProvisioningProgressView from '../../src/lib/ui/components/ProvisioningProgress.svelte'; import ProvisioningActivity from '../../src/lib/ui/components/ProvisioningActivity.svelte'; const at = '2026-09-28T12:00:00Z'; function progress(patch: Partial = {}): ProvisioningProgress { - return { version: 1, phase: 'verify-map', phaseStartedAt: at, observedAt: at, blocker: null, checks: [], checkpoints: [], - command: null, recovery: null, recipeStatus: null, ...patch }; + return { + version: 1, + phase: 'verify-map', + phaseStartedAt: at, + observedAt: at, + blocker: null, + checks: [], + checkpoints: [], + command: null, + recovery: null, + recipeStatus: null, + ...patch + }; } function fixture(): ProvisioningDetail { return { - deployment: { id: 'operation-a', server: 't5_zombie_theater_1', action: 'create', state: 'awaiting-health', stage: 'Waiting for a map', - server_deleted_at: null, server_exists: 1, started_at: at, finished_at: null, triggered_by: 'ui', error: null, warnings: '[]' }, - generatedAt: at, events: [], activity: { total: 0, latestId: null, failureExcerpt: null }, sources: [], - recipe: null, blueprintName: null, canResumeCleanup: false, canResumeRebuild: false, canResumeVerification: false, canCancelRemoval: false, - collection: null, progress: progress(), capacity: { evaluatedAt: at, pools: [] }, warningMessages: [] + deployment: { + id: 'operation-a', + server: 't5_zombie_theater_1', + action: 'create', + state: 'awaiting-health', + stage: 'Waiting for a map', + server_deleted_at: null, + server_exists: 1, + started_at: at, + finished_at: null, + triggered_by: 'ui', + error: null, + warnings: '[]' + }, + generatedAt: at, + events: [], + activity: { total: 0, latestId: null, failureExcerpt: null }, + sources: [], + recipe: null, + blueprintName: null, + canResumeCleanup: false, + canResumeRebuild: false, + canResumeVerification: false, + canCancelRemoval: false, + collection: null, + progress: progress(), + capacity: { evaluatedAt: at, pools: [] }, + warningMessages: [] }; } function completedFixture(): ProvisioningDetail { const detail = fixture(); detail.deployment.state = 'done'; detail.deployment.finished_at = at; - detail.progress = progress({ checks: (['container', 'game', 'map', 'plutonium-release', 'plutonium-version'] as const).map(key => ({ - key, label: key, result: 'pass', source: 'agent', checkedAt: at, receivedAt: at, - expected: `Expected ${key}`, actual: `Observed ${key}`, message: null - })) }); + detail.progress = progress({ + checks: ( + ['container', 'game', 'map', 'plutonium-release', 'plutonium-version'] as const + ).map((key) => ({ + key, + label: key, + result: 'pass', + source: 'agent', + checkedAt: at, + receivedAt: at, + expected: `Expected ${key}`, + actual: `Observed ${key}`, + message: null + })) + }); return detail; } @@ -52,73 +107,122 @@ describe('provisioning outcome and evidence presentation', () => { expect(provisioningOutcome(detail).checksSummary).toBeNull(); }); - it.each(['fail', 'waiting', 'unknown'] as const)('keeps %s evidence visible even with a recorded completed outcome', result => { - const detail = completedFixture(); - detail.progress!.checks[2].result = result; - expect(provisioningOutcome(detail).checksSummary).toBeNull(); - const body = render(ProvisioningProgressView, { props: { detail } }).body; - const openGroups = body.match(/]*\bopen\b[^>]*>[\s\S]*?<\/details>/g); - expect(openGroups).toHaveLength(1); - expect(openGroups![0]).toContain('Observed map'); - }); - - it.each(['queued', 'waiting', 'running', 'awaiting-health', 'warning', 'failed', 'interrupted', 'cancelled'])( - 'keeps %s evidence expanded without claiming the operation passed all checks', state => { + it.each(['fail', 'waiting', 'unknown'] as const)( + 'keeps %s evidence visible even with a recorded completed outcome', + (result) => { const detail = completedFixture(); - detail.deployment.state = state; + detail.progress!.checks[2].result = result; expect(provisioningOutcome(detail).checksSummary).toBeNull(); const body = render(ProvisioningProgressView, { props: { detail } }).body; - expect(body.match(/]*\bopen\b/g)).toHaveLength(3); - }); + const openGroups = body.match(/]*\bopen\b[^>]*>[\s\S]*?<\/details>/g); + expect(openGroups).toHaveLength(1); + expect(openGroups![0]).toContain('Observed map'); + } + ); + + it.each([ + 'queued', + 'waiting', + 'running', + 'awaiting-health', + 'warning', + 'failed', + 'interrupted', + 'cancelled' + ])('keeps %s evidence expanded without claiming the operation passed all checks', (state) => { + const detail = completedFixture(); + detail.deployment.state = state; + expect(provisioningOutcome(detail).checksSummary).toBeNull(); + const body = render(ProvisioningProgressView, { props: { detail } }).body; + expect(body.match(/]*\bopen\b/g)).toHaveLength(3); + }); it('shows recorded usernames and preserves unknown manual requesters', () => { expect(provisioningRequester({ triggered_by: 'ui', requested_by: 'alice' })).toBe('alice'); - expect(provisioningRequester({ triggered_by: 'rebalance:run', requested_by: 'bob' })).toBe('bob'); - expect(provisioningRequester({ triggered_by: 'ui', requested_by: null })).toBe('Not recorded'); + expect(provisioningRequester({ triggered_by: 'rebalance:run', requested_by: 'bob' })).toBe( + 'bob' + ); + expect(provisioningRequester({ triggered_by: 'ui', requested_by: null })).toBe( + 'Not recorded' + ); expect(provisioningRequester({ triggered_by: 'ui' })).toBe('Not recorded'); }); - it.each(['watchdog:incident', 'maintenance:job', 'autoscaling:policy', 'campaign:run', 'beta:75', 'rebalance:run', 'host-drain'])( - 'shows Maxis for system work from %s', triggered_by => { - expect(provisioningRequester({ triggered_by, requested_by: null })).toBe('Maxis'); - expect(provisioningRequester({ triggered_by })).toBe('Maxis'); - }); + it.each([ + 'watchdog:incident', + 'maintenance:job', + 'autoscaling:policy', + 'campaign:run', + 'beta:75', + 'rebalance:run', + 'host-drain' + ])('shows Maxis for system work from %s', (triggered_by) => { + expect(provisioningRequester({ triggered_by, requested_by: null })).toBe('Maxis'); + expect(provisioningRequester({ triggered_by })).toBe('Maxis'); + }); it('distinguishes a failed creation retry from readiness verification and its linked recovery', () => { const detail = fixture(); detail.deployment.state = 'failed'; detail.canRetryWithRebuild = true; - expect(provisioningOutcome(detail)).toMatchObject({ active: false, headline: 'Create failed' }); - expect(provisioningOutcome(detail).description).toContain('new attempt for the same lobby and port'); + expect(provisioningOutcome(detail)).toMatchObject({ + active: false, + headline: 'Create failed' + }); + expect(provisioningOutcome(detail).description).toContain( + 'new attempt for the same lobby and port' + ); detail.canRetryWithRebuild = false; - detail.progress = progress({ recovery: { state: 'transferred', operationId: 'recovery-a', at, message: 'Rebuild requested' } }); + detail.progress = progress({ + recovery: { + state: 'transferred', + operationId: 'recovery-a', + at, + message: 'Rebuild requested' + } + }); expect(provisioningOutcome(detail).headline).toBe('Create failed'); expect(provisioningOutcome(detail).description).toContain('linked operation'); detail.progress = progress(); detail.canResumeVerification = true; - expect(provisioningOutcome(detail).description).toContain('remaining restart and rebuild recovery steps'); + expect(provisioningOutcome(detail).description).toContain( + 'remaining restart and rebuild recovery steps' + ); }); it('distinguishes cancellation from confirmed cleanup and validates its link data', () => { const detail = fixture(); detail.deployment.state = 'cancelled'; detail.canCancelCreation = false; - detail.cancellation = { cleanupId: 'cleanup-a', state: 'running', stage: 'Waiting for host', error: null }; + detail.cancellation = { + cleanupId: 'cleanup-a', + state: 'running', + stage: 'Waiting for host', + error: null + }; expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); expect(provisioningOutcome(detail).description).toContain('ports remain reserved'); detail.cancellation.state = 'done'; expect(provisioningOutcome(detail).description).toContain('have been removed'); - expect(isProvisioningDetail({ ...detail, cancellation: { state: 'done' } }, detail.deployment.id)).toBe(false); - }); - it.each(provisioningActions)('describes cancelled %s as terminal, with no automatic execution claim', action => { - const detail = fixture(); - detail.deployment.action = action; - detail.deployment.state = 'cancelled'; - const outcome = provisioningOutcome(detail); - expect(outcome.active).toBe(false); - expect(outcome.headline).toContain('cancelled'); - expect(outcome.description).toContain('remaining work will not run'); - expect(outcome.description).not.toMatch(/verifying|automatically/); + expect( + isProvisioningDetail( + { ...detail, cancellation: { state: 'done' } }, + detail.deployment.id + ) + ).toBe(false); }); + it.each(provisioningActions)( + 'describes cancelled %s as terminal, with no automatic execution claim', + (action) => { + const detail = fixture(); + detail.deployment.action = action; + detail.deployment.state = 'cancelled'; + const outcome = provisioningOutcome(detail); + expect(outcome.active).toBe(false); + expect(outcome.headline).toContain('cancelled'); + expect(outcome.description).toContain('remaining work will not run'); + expect(outcome.description).not.toMatch(/verifying|automatically/); + } + ); it('keeps backend-authorized cleanup actionable and active after identity deletion', () => { const detail = fixture(); @@ -127,167 +231,419 @@ describe('provisioning outcome and evidence presentation', () => { detail.deployment.server_deleted_at = at; detail.deployment.server_exists = 0; detail.canResumeCleanup = true; - expect(provisioningOutcome(detail).description).toContain('Resume the remaining local cleanup'); + expect(provisioningOutcome(detail).description).toContain( + 'Resume the remaining local cleanup' + ); detail.deployment.state = 'waiting'; detail.canResumeCleanup = false; detail.progress = progress({ phase: 'cleanup', checks: [] }); expect(provisioningOutcome(detail).active).toBe(true); - expect(provisioningOutcome(detail).description).toContain('without repeating container removal'); + expect(provisioningOutcome(detail).description).toContain( + 'without repeating container removal' + ); expect(provisioningOutcome(detail).description).not.toContain('eligibility'); }); it('does not invent completed phases from a later phase or final outcome', () => { const detail = fixture(); - const steps = provisioningGroups(detail).flatMap(group => group.steps); - expect(steps.find(step => step.key === 'prepare')?.state).toBe('unknown'); - expect(steps.find(step => step.key === 'verify-map')?.state).toBe('current'); + const steps = provisioningGroups(detail).flatMap((group) => group.steps); + expect(steps.find((step) => step.key === 'prepare')?.state).toBe('unknown'); + expect(steps.find((step) => step.key === 'verify-map')?.state).toBe('current'); detail.deployment.state = 'done'; - expect(provisioningGroups(detail).flatMap(group => group.steps).every(step => step.state === 'unknown')).toBe(true); + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .every((step) => step.state === 'unknown') + ).toBe(true); detail.progress = null; expect(provisioningGroups(detail)).toEqual([]); expect(provisioningOutcome(detail).description).toBeNull(); - expect(render(ProvisioningProgressView, { props: { detail } }).body).toContain('Detailed checkpoints were not recorded'); + expect(render(ProvisioningProgressView, { props: { detail } }).body).toContain( + 'Detailed checkpoints were not recorded' + ); }); it('shows passed container and telemetry game checks while waiting on the engine map', () => { const detail = fixture(); - detail.progress = progress({ checks: [ - { key: 'container', label: 'Container readiness', result: 'pass', source: 'agent', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null }, - { key: 'game', label: 'Game status', result: 'pass', source: 'telemetry', checkedAt: at, receivedAt: at, expected: null, actual: null, message: 'Fresh telemetry accepted' }, - { key: 'map', label: 'Engine map', result: 'waiting', source: 'telemetry', checkedAt: at, receivedAt: at, expected: 'zombie_theater', actual: 'zombie_moon', message: null } - ] }); - const steps = provisioningGroups(detail).flatMap(group => group.steps); - expect(steps.find(step => step.key === 'verify-container')?.state).toBe('done'); - expect(steps.find(step => step.key === 'verify-game')?.state).toBe('done'); - expect(steps.find(step => step.key === 'verify-map')?.state).toBe('waiting'); - expect(steps.find(step => step.key === 'verify-game')?.label).not.toContain('RCON'); + detail.progress = progress({ + checks: [ + { + key: 'container', + label: 'Container readiness', + result: 'pass', + source: 'agent', + checkedAt: at, + receivedAt: null, + expected: 'healthy', + actual: 'healthy', + message: null + }, + { + key: 'game', + label: 'Game status', + result: 'pass', + source: 'telemetry', + checkedAt: at, + receivedAt: at, + expected: null, + actual: null, + message: 'Fresh telemetry accepted' + }, + { + key: 'map', + label: 'Engine map', + result: 'waiting', + source: 'telemetry', + checkedAt: at, + receivedAt: at, + expected: 'zombie_theater', + actual: 'zombie_moon', + message: null + } + ] + }); + const steps = provisioningGroups(detail).flatMap((group) => group.steps); + expect(steps.find((step) => step.key === 'verify-container')?.state).toBe('done'); + expect(steps.find((step) => step.key === 'verify-game')?.state).toBe('done'); + expect(steps.find((step) => step.key === 'verify-map')?.state).toBe('waiting'); + expect(steps.find((step) => step.key === 'verify-game')?.label).not.toContain('RCON'); detail.deployment.state = 'done'; expect(provisioningOutcome(detail).description).toBeNull(); }); it('newer unknown readiness evidence supersedes an old completion checkpoint', () => { const detail = fixture(); - detail.progress = progress({ checkpoints: [{ key: 'verify-game', label: 'Game passed', at, phase: 'verify-game' }], - checks: [{ key: 'game', label: 'Game status', result: 'unknown', source: null, checkedAt: '2026-09-28T12:01:00Z', receivedAt: null, expected: null, actual: null, message: 'No fresh evidence' }] }); - expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')).toMatchObject({ state: 'unknown', at: '2026-09-28T12:01:00Z' }); - }); - - it.each(['failed', 'interrupted'])('shows where an unhealthy container stopped a %s operation', state => { - const detail = fixture(); - detail.deployment.state = state; - detail.deployment.finished_at = at; - detail.progress = progress({ phase: 'verify-container', - checkpoints: [{ key: 'container-action', label: 'Host command completed', at, phase: 'container-action' }], - checks: [{ key: 'container', label: 'Container readiness', result: 'waiting', source: 'agent', checkedAt: at, receivedAt: null, - expected: 'healthy', actual: 'unhealthy', message: null }] }); - const groups = provisioningGroups(detail); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'container-action')?.state).toBe('done'); - expect(groups.find(group => group.key === 'verify')).toMatchObject({ state: 'failed', steps: [ - { key: 'verify-container', state: 'failed', at }, - { key: 'verify-game', state: 'cancelled', at: null }, - { key: 'verify-map', state: 'cancelled', at: null }, - { key: 'verify-plutonium', state: 'cancelled', at: null } - ] }); - const body = render(ProvisioningProgressView, { props: { detail } }).body; - const phases = body; - expect(phases.match(/Stopped here/g)).toHaveLength(2); - expect(phases.match(/Not run/g)).toHaveLength(3); - expect(phases).not.toContain('Waiting'); - expect(phases).toMatch(/
  • { - const detail = fixture(); - detail.deployment.state = state; - detail.progress = progress(); - expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-map')?.state).toBe('failed'); - detail.progress.checks = [{ key: 'map', label: 'Engine map', result: 'unknown', source: null, checkedAt: at, receivedAt: null, - expected: 'zombie_theater', actual: null, message: 'No fresh evidence' }]; - const steps = provisioningGroups(detail).flatMap(group => group.steps); - expect(steps.find(step => step.key === 'verify-map')?.state).toBe('failed'); - expect(steps.find(step => step.key === 'verify-plutonium')?.state).toBe('cancelled'); - expect(steps.find(step => step.key === 'prepare')?.state).toBe('unknown'); - }); - - it.each(['done', 'warning', 'cancelled', 'failed', 'interrupted'])('does not highlight stale waiting evidence as active on a %s operation', state => { - const detail = fixture(); - detail.progress = progress({ phase: 'verify-container', checks: [ - { key: 'container', label: 'Container readiness', result: 'waiting', source: 'agent', checkedAt: at, receivedAt: null, - expected: 'healthy', actual: 'starting', message: null } - ] }); - const activeBody = render(ProvisioningProgressView, { props: { detail } }).body; - expect(activeBody).toMatch(/
  • { - const detail = fixture(); - detail.deployment.action = action; - detail.progress = progress({ phase: 'verify-plutonium', + detail.progress = progress({ + checkpoints: [{ key: 'verify-game', label: 'Game passed', at, phase: 'verify-game' }], checks: [ - { key: 'plutonium-release', label: 'Production Plutonium release', result: 'pass', source: 'manifest', checkedAt: at, receivedAt: at, expected: null, actual: 5200, message: null }, - { key: 'plutonium-version', label: 'Running Plutonium revision', result: 'unknown', source: 'rcon', checkedAt: at, receivedAt: null, expected: 5200, actual: null, message: 'Waiting for a runtime revision' } - ] }); - expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); - for (const [result, state] of [['unknown', 'unknown'], ['waiting', 'waiting'], ['fail', 'failed'], ['pass', 'done']] as const) { - detail.progress.checks[1].result = result; - const steps = provisioningGroups(detail).find(group => group.key === 'verify')!.steps; - expect(steps.at(-1)).toMatchObject({ key: 'verify-plutonium', label: 'Verify Plutonium revision', state, at }); - } + { + key: 'game', + label: 'Game status', + result: 'unknown', + source: null, + checkedAt: '2026-09-28T12:01:00Z', + receivedAt: null, + expected: null, + actual: null, + message: 'No fresh evidence' + } + ] + }); + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .find((step) => step.key === 'verify-game') + ).toMatchObject({ state: 'unknown', at: '2026-09-28T12:01:00Z' }); }); - it.each(['start', 'restart', 'stop', 'destroy'] as const)('does not add Plutonium verification to %s', action => { - const detail = fixture(); - detail.deployment.action = action; - expect(provisioningGroups(detail).flatMap(group => group.steps).some(step => step.key === 'verify-plutonium')).toBe(false); - }); + it.each(['failed', 'interrupted'])( + 'shows where an unhealthy container stopped a %s operation', + (state) => { + const detail = fixture(); + detail.deployment.state = state; + detail.deployment.finished_at = at; + detail.progress = progress({ + phase: 'verify-container', + checkpoints: [ + { + key: 'container-action', + label: 'Host command completed', + at, + phase: 'container-action' + } + ], + checks: [ + { + key: 'container', + label: 'Container readiness', + result: 'waiting', + source: 'agent', + checkedAt: at, + receivedAt: null, + expected: 'healthy', + actual: 'unhealthy', + message: null + } + ] + }); + const groups = provisioningGroups(detail); + expect( + groups + .flatMap((group) => group.steps) + .find((step) => step.key === 'container-action')?.state + ).toBe('done'); + expect(groups.find((group) => group.key === 'verify')).toMatchObject({ + state: 'failed', + steps: [ + { key: 'verify-container', state: 'failed', at }, + { key: 'verify-game', state: 'cancelled', at: null }, + { key: 'verify-map', state: 'cancelled', at: null }, + { key: 'verify-plutonium', state: 'cancelled', at: null } + ] + }); + const body = render(ProvisioningProgressView, { props: { detail } }).body; + const phases = body; + expect(phases.match(/Stopped here/g)).toHaveLength(2); + expect(phases.match(/Not run/g)).toHaveLength(3); + expect(phases).not.toContain('Waiting'); + expect(phases).toMatch(/
  • { + const detail = fixture(); + detail.deployment.state = state; + detail.progress = progress(); + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .find((step) => step.key === 'verify-map')?.state + ).toBe('failed'); + detail.progress.checks = [ + { + key: 'map', + label: 'Engine map', + result: 'unknown', + source: null, + checkedAt: at, + receivedAt: null, + expected: 'zombie_theater', + actual: null, + message: 'No fresh evidence' + } + ]; + const steps = provisioningGroups(detail).flatMap((group) => group.steps); + expect(steps.find((step) => step.key === 'verify-map')?.state).toBe('failed'); + expect(steps.find((step) => step.key === 'verify-plutonium')?.state).toBe('cancelled'); + expect(steps.find((step) => step.key === 'prepare')?.state).toBe('unknown'); + } + ); + + it.each(['done', 'warning', 'cancelled', 'failed', 'interrupted'])( + 'does not highlight stale waiting evidence as active on a %s operation', + (state) => { + const detail = fixture(); + detail.progress = progress({ + phase: 'verify-container', + checks: [ + { + key: 'container', + label: 'Container readiness', + result: 'waiting', + source: 'agent', + checkedAt: at, + receivedAt: null, + expected: 'healthy', + actual: 'starting', + message: null + } + ] + }); + const activeBody = render(ProvisioningProgressView, { props: { detail } }).body; + expect(activeBody).toMatch(/
  • { + const detail = fixture(); + detail.deployment.action = action; + detail.progress = progress({ + phase: 'verify-plutonium', + checks: [ + { + key: 'plutonium-release', + label: 'Production Plutonium release', + result: 'pass', + source: 'manifest', + checkedAt: at, + receivedAt: at, + expected: null, + actual: 5200, + message: null + }, + { + key: 'plutonium-version', + label: 'Running Plutonium revision', + result: 'unknown', + source: 'rcon', + checkedAt: at, + receivedAt: null, + expected: 5200, + actual: null, + message: 'Waiting for a runtime revision' + } + ] + }); + expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); + for (const [result, state] of [ + ['unknown', 'unknown'], + ['waiting', 'waiting'], + ['fail', 'failed'], + ['pass', 'done'] + ] as const) { + detail.progress.checks[1].result = result; + const steps = provisioningGroups(detail).find( + (group) => group.key === 'verify' + )!.steps; + expect(steps.at(-1)).toMatchObject({ + key: 'verify-plutonium', + label: 'Verify Plutonium revision', + state, + at + }); + } + } + ); + + it.each(['start', 'restart', 'stop', 'destroy'] as const)( + 'does not add Plutonium verification to %s', + (action) => { + const detail = fixture(); + detail.deployment.action = action; + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .some((step) => step.key === 'verify-plutonium') + ).toBe(false); + } + ); it('does not claim historical creations verified Plutonium without evidence', () => { const detail = fixture(); detail.deployment.state = 'done'; - detail.progress = progress({ checkpoints: [{ key: 'verify-map', label: 'Selected map verified', at, phase: 'verify-map' }] }); - expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-plutonium')) - .toMatchObject({ state: 'unknown', at: null }); + detail.progress = progress({ + checkpoints: [ + { key: 'verify-map', label: 'Selected map verified', at, phase: 'verify-map' } + ] + }); + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .find((step) => step.key === 'verify-plutonium') + ).toMatchObject({ state: 'unknown', at: null }); }); it('preserves the failed request when later recovery checks pass', () => { const detail = fixture(); - detail.deployment.action = 'restart'; detail.deployment.state = 'failed'; - detail.progress = progress({ phase: 'verify-game', recovery: { state: 'released', at, message: 'Recovery verified and capacity released.' }, - checks: [{ key: 'game', label: 'Game status', result: 'pass', source: 'rcon', checkedAt: at, receivedAt: null, expected: null, actual: 0, message: null }] }); + detail.deployment.action = 'restart'; + detail.deployment.state = 'failed'; + detail.progress = progress({ + phase: 'verify-game', + recovery: { + state: 'released', + at, + message: 'Recovery verified and capacity released.' + }, + checks: [ + { + key: 'game', + label: 'Game status', + result: 'pass', + source: 'rcon', + checkedAt: at, + receivedAt: null, + expected: null, + actual: 0, + message: null + } + ] + }); expect(provisioningOutcome(detail).headline).toBe('Restart failed'); - expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')?.state).toBe('done'); + expect( + provisioningGroups(detail) + .flatMap((group) => group.steps) + .find((step) => step.key === 'verify-game')?.state + ).toBe('done'); }); it('keeps confirmed agent removal separate from registry cleanup', () => { - const detail = fixture(); detail.deployment.action = 'destroy'; detail.deployment.state = 'waiting'; - detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'container-action', label: 'Agent confirmed removal', at, phase: 'container-action' }] }); + const detail = fixture(); + detail.deployment.action = 'destroy'; + detail.deployment.state = 'waiting'; + detail.progress = progress({ + phase: 'cleanup', + checkpoints: [ + { + key: 'container-action', + label: 'Agent confirmed removal', + at, + phase: 'container-action' + } + ] + }); const groups = provisioningGroups(detail); - expect(groups.map(group => group.key)).toEqual(['request', 'command', 'cleanup']); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'container-action')?.state).toBe('done'); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'cleanup')?.state).toBe('waiting'); + expect(groups.map((group) => group.key)).toEqual(['request', 'command', 'cleanup']); + expect( + groups.flatMap((group) => group.steps).find((step) => step.key === 'container-action') + ?.state + ).toBe('done'); + expect( + groups.flatMap((group) => group.steps).find((step) => step.key === 'cleanup')?.state + ).toBe('waiting'); }); it('renders check values and their provenance once inside the corresponding step', () => { const detail = fixture(); - detail.progress = progress({ checks: [ - { key: 'game', label: 'Game status', result: 'pass', source: 'rcon', checkedAt: at, receivedAt: at, expected: null, actual: 0, message: null }, - { key: 'map', label: 'Engine map', result: 'waiting', source: 'telemetry', checkedAt: at, receivedAt: at, expected: 'zombie_theater', actual: 'zombie_moon', message: 'Waiting for selected map' } - ] }); - const steps = provisioningGroups(detail).flatMap(group => group.steps); - expect(steps.find(step => step.key === 'verify-game')?.checks).toEqual([detail.progress.checks[0]]); - expect(steps.find(step => step.key === 'verify-map')?.checks).toEqual([detail.progress.checks[1]]); + detail.progress = progress({ + checks: [ + { + key: 'game', + label: 'Game status', + result: 'pass', + source: 'rcon', + checkedAt: at, + receivedAt: at, + expected: null, + actual: 0, + message: null + }, + { + key: 'map', + label: 'Engine map', + result: 'waiting', + source: 'telemetry', + checkedAt: at, + receivedAt: at, + expected: 'zombie_theater', + actual: 'zombie_moon', + message: 'Waiting for selected map' + } + ] + }); + const steps = provisioningGroups(detail).flatMap((group) => group.steps); + expect(steps.find((step) => step.key === 'verify-game')?.checks).toEqual([ + detail.progress.checks[0] + ]); + expect(steps.find((step) => step.key === 'verify-map')?.checks).toEqual([ + detail.progress.checks[1] + ]); const body = render(ProvisioningProgressView, { props: { detail } }).body; - const mapStep = body.slice(body.indexOf('Verify selected engine map'), body.indexOf('Verify Plutonium revision')); - for (const value of ['Expected', 'Observed', 'zombie_theater', 'zombie_moon', 'Telemetry', 'Evidence received', 'Waiting for selected map']) { + const mapStep = body.slice( + body.indexOf('Verify selected engine map'), + body.indexOf('Verify Plutonium revision') + ); + for (const value of [ + 'Expected', + 'Observed', + 'zombie_theater', + 'zombie_moon', + 'Telemetry', + 'Evidence received', + 'Waiting for selected map' + ]) { expect(mapStep).toContain(value); } expect(body.match(/zombie_theater/g)).toHaveLength(1); @@ -298,12 +654,37 @@ describe('provisioning outcome and evidence presentation', () => { it('keeps the release manifest in the revision step without treating it as runtime verification', () => { const detail = fixture(); - detail.progress = progress({ phase: 'verify-plutonium', checks: [ - { key: 'plutonium-release', label: 'Production Plutonium release', result: 'pass', source: 'manifest', checkedAt: at, receivedAt: null, expected: null, actual: 5200, message: null } - ] }); - const step = provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-plutonium'); + detail.progress = progress({ + phase: 'verify-plutonium', + checks: [ + { + key: 'plutonium-release', + label: 'Production Plutonium release', + result: 'pass', + source: 'manifest', + checkedAt: at, + receivedAt: null, + expected: null, + actual: 5200, + message: null + } + ] + }); + const step = provisioningGroups(detail) + .flatMap((group) => group.steps) + .find((step) => step.key === 'verify-plutonium'); expect(step).toMatchObject({ state: 'current', checks: detail.progress.checks }); - detail.progress.checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', result: 'waiting', source: 'rcon', checkedAt: at, receivedAt: null, expected: 5200, actual: 5199, message: null }); + detail.progress.checks.push({ + key: 'plutonium-version', + label: 'Running Plutonium revision', + result: 'waiting', + source: 'rcon', + checkedAt: at, + receivedAt: null, + expected: 5200, + actual: 5199, + message: null + }); const body = render(ProvisioningProgressView, { props: { detail } }).body; expect(body).toContain('Production Plutonium release'); expect(body).toContain('Running Plutonium revision'); @@ -314,48 +695,129 @@ describe('provisioning outcome and evidence presentation', () => { it('folds staging, command receipts, recovery and agent diagnostics into their steps', () => { const detail = fixture(); detail.progress = progress({ - command: { state: 'uncertain', at, message: 'Awaiting acknowledgment of the original command' }, + command: { + state: 'uncertain', + at, + message: 'Awaiting acknowledgment of the original command' + }, checkpoints: [ - { key: 'prepare', label: 'Launch files selected and command payload prepared', at, phase: 'prepare' }, - { key: 'staging-completed', label: 'File staging completed', at, phase: 'verify-map' }, - { key: 'agent-accepted', label: 'Host acknowledged command', at, phase: 'verify-map' }, - { key: 'recovery-restart', label: 'Recovery restart completed', at, phase: 'verify-container' }, + { + key: 'prepare', + label: 'Launch files selected and command payload prepared', + at, + phase: 'prepare' + }, + { + key: 'staging-completed', + label: 'File staging completed', + at, + phase: 'verify-map' + }, + { + key: 'agent-accepted', + label: 'Host acknowledged command', + at, + phase: 'verify-map' + }, + { + key: 'recovery-restart', + label: 'Recovery restart completed', + at, + phase: 'verify-container' + }, { key: 'legacy-receipt', label: 'Legacy receipt without a phase', at, phase: null } ] }); - detail.agentOperation = { id: 'agent-command-a', hostId: 'host-a', state: 'unknown', createdAt: at, updatedAt: at, error: 'Agent disconnected' }; - const steps = provisioningGroups(detail).flatMap(group => group.steps); - expect(steps.find(step => step.key === 'prepare')).toMatchObject({ label: detail.progress.checkpoints[0].label, checkpoints: [detail.progress.checkpoints[1]] }); - expect(steps.find(step => step.key === 'container-action')).toMatchObject({ - command: detail.progress.command, agentOperation: detail.agentOperation, checkpoints: detail.progress.checkpoints.slice(2) + detail.agentOperation = { + id: 'agent-command-a', + hostId: 'host-a', + state: 'unknown', + createdAt: at, + updatedAt: at, + error: 'Agent disconnected' + }; + const steps = provisioningGroups(detail).flatMap((group) => group.steps); + expect(steps.find((step) => step.key === 'prepare')).toMatchObject({ + label: detail.progress.checkpoints[0].label, + checkpoints: [detail.progress.checkpoints[1]] }); - const body = render(ProvisioningProgressView, { props: { detail, hostHistoryHref: '/hosts/operations?server=test&returnTo=%2Fprovisioning%2Foperation-a' } }).body; - const commandStep = body.slice(body.indexOf('Create instance through host agent'), body.indexOf('Verify readiness')); - for (const value of ['Remote response uncertain', detail.progress.command!.message, 'Agent disconnected', 'View host operation history']) { + expect(steps.find((step) => step.key === 'container-action')).toMatchObject({ + command: detail.progress.command, + agentOperation: detail.agentOperation, + checkpoints: detail.progress.checkpoints.slice(2) + }); + const body = render(ProvisioningProgressView, { + props: { + detail, + hostHistoryHref: + '/hosts/operations?server=test&returnTo=%2Fprovisioning%2Foperation-a' + } + }).body; + const commandStep = body.slice( + body.indexOf('Create instance through host agent'), + body.indexOf('Verify readiness') + ); + for (const value of [ + 'Remote response uncertain', + detail.progress.command!.message, + 'Agent disconnected', + 'View host operation history' + ]) { expect(commandStep).toContain(value); expect(body.split(value)).toHaveLength(2); } expect(commandStep).toContain('returnTo=%2Fprovisioning%2Foperation-a'); expect(body).not.toContain('Recorded checkpoints'); - for (const checkpoint of detail.progress.checkpoints) expect(body.split(checkpoint.label)).toHaveLength(2); + for (const checkpoint of detail.progress.checkpoints) + expect(body.split(checkpoint.label)).toHaveLength(2); }); it('retains recorded eligibility and staging evidence for actions without launch preparation by default', () => { const detail = fixture(); detail.deployment.action = 'restart'; - detail.progress = progress({ phase: 'eligibility', checks: [ - { key: 'occupancy', label: 'Player safety', result: 'waiting', source: 'rcon', checkedAt: at, receivedAt: null, expected: 0, actual: 2, message: 'Players are still connected' } - ], checkpoints: [{ key: 'staging-completed', label: 'Files staged', at, phase: 'container-action' }] }); + detail.progress = progress({ + phase: 'eligibility', + checks: [ + { + key: 'occupancy', + label: 'Player safety', + result: 'waiting', + source: 'rcon', + checkedAt: at, + receivedAt: null, + expected: 0, + actual: 2, + message: 'Players are still connected' + } + ], + checkpoints: [ + { key: 'staging-completed', label: 'Files staged', at, phase: 'container-action' } + ] + }); const groups = provisioningGroups(detail); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'eligibility')).toMatchObject({ state: 'waiting', checks: detail.progress.checks }); - expect(groups.find(group => group.key === 'command')?.steps.map(step => step.key)).toEqual(['prepare', 'container-action']); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'prepare')?.checkpoints).toEqual(detail.progress.checkpoints); + expect( + groups.flatMap((group) => group.steps).find((step) => step.key === 'eligibility') + ).toMatchObject({ state: 'waiting', checks: detail.progress.checks }); + expect( + groups.find((group) => group.key === 'command')?.steps.map((step) => step.key) + ).toEqual(['prepare', 'container-action']); + expect( + groups.flatMap((group) => group.steps).find((step) => step.key === 'prepare') + ?.checkpoints + ).toEqual(detail.progress.checkpoints); }); it('preserves host agent evidence when a legacy operation has no progress snapshot', () => { const detail = fixture(); detail.progress = null; - detail.agentOperation = { id: 'legacy-agent-command', hostId: 'host-a', state: 'failed', createdAt: at, updatedAt: at, error: 'Command failed on host' }; + detail.agentOperation = { + id: 'legacy-agent-command', + hostId: 'host-a', + state: 'failed', + createdAt: at, + updatedAt: at, + error: 'Command failed on host' + }; const body = render(ProvisioningProgressView, { props: { detail } }).body; expect(body).toContain('Detailed checkpoints were not recorded'); expect(body).toContain('Command failed on host'); @@ -363,9 +825,13 @@ describe('provisioning outcome and evidence presentation', () => { expect(body).not.toContain('Verify readiness'); }); - it.each([0, 1, 5])('keeps %i raw activity events behind one closed disclosure', count => { + it.each([0, 1, 5])('keeps %i raw activity events behind one closed disclosure', (count) => { const detail = fixture(); - detail.events = Array.from({ length: count }, (_, id) => ({ id, created_at: at, message: 'Raw event message' })); + detail.events = Array.from({ length: count }, (_, id) => ({ + id, + created_at: at, + message: 'Raw event message' + })); detail.activity.total = count; const body = render(ProvisioningActivity, { props: { detail } }).body; expect(body).toContain(`${count} event${count === 1 ? '' : 's'}`); @@ -382,40 +848,82 @@ describe('provisioning response and return-context boundaries', () => { it('accepts optional retry eligibility only as an authoritative boolean', () => { const detail = fixture(); for (const canRetryWithRebuild of [undefined, false, true]) { - expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(true); + expect( + isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id) + ).toBe(true); } for (const canRetryWithRebuild of ['true', 1, null]) { - expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(false); + expect( + isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id) + ).toBe(false); } }); it('accepts persisted usernames and legacy details but rejects malformed requesters', () => { const detail = fixture(); for (const requested_by of [undefined, null, 'alice']) { - expect(isProvisioningDetail({ ...detail, deployment: { ...detail.deployment, requested_by } }, detail.deployment.id)).toBe(true); + expect( + isProvisioningDetail( + { ...detail, deployment: { ...detail.deployment, requested_by } }, + detail.deployment.id + ) + ).toBe(true); } - expect(isProvisioningDetail({ ...detail, deployment: { ...detail.deployment, requested_by: 42 } }, detail.deployment.id)).toBe(false); + expect( + isProvisioningDetail( + { ...detail, deployment: { ...detail.deployment, requested_by: 42 } }, + detail.deployment.id + ) + ).toBe(false); }); it('retains list filters without accepting an external or unrelated return destination', () => { - expect(provisioningReturn('/provisioning?view=history&action=restart&server=Lobby+1&limit=50&redirect=https://evil.test').href) - .toBe('/provisioning?view=history&action=restart&server=Lobby+1&limit=50'); - expect(provisioningReturn('/provisioning?view=history&limit=25&offset=100').href) - .toBe('/provisioning?view=history&limit=25&offset=100'); - expect(provisioningReturn('/provisioning?view=history&source=campaign&status=failed&offset=25').href) - .toBe('/provisioning?view=history&status=failed&source=campaign&offset=25'); - for (const value of ['https://evil.test', '//evil.test', '/\\evil.test', '/\n/evil.test', '/settings', '/provisioning/other']) { + expect( + provisioningReturn( + '/provisioning?view=history&action=restart&server=Lobby+1&limit=50&redirect=https://evil.test' + ).href + ).toBe('/provisioning?view=history&action=restart&server=Lobby+1&limit=50'); + expect(provisioningReturn('/provisioning?view=history&limit=25&offset=100').href).toBe( + '/provisioning?view=history&limit=25&offset=100' + ); + expect( + provisioningReturn('/provisioning?view=history&source=campaign&status=failed&offset=25') + .href + ).toBe('/provisioning?view=history&status=failed&source=campaign&offset=25'); + for (const value of [ + 'https://evil.test', + '//evil.test', + '/\\evil.test', + '/\n/evil.test', + '/settings', + '/provisioning/other' + ]) { expect(provisioningReturn(value).href).toBe('/provisioning'); } - expect(provisioningReturn('/collections/launches/launch-a').href).toBe('/collections/launches/launch-a'); + expect(provisioningReturn('/collections/launches/launch-a').href).toBe( + '/collections/launches/launch-a' + ); }); it('returns rebalance moves to their run while discarding unrelated navigation parameters', () => { - expect(provisioningReturn('/capacity?rebalance=1&rebalanceRun=run-a&redirect=https://evil.test')) - .toEqual({ href: '/capacity?rebalance=1&rebalanceRun=run-a', label: 'Back to rebalance' }); - expect(provisioningReturn('/capacity?rebalance=1')).toEqual({ href: '/capacity?rebalance=1', label: 'Back to rebalance' }); + expect( + provisioningReturn( + '/capacity?rebalance=1&rebalanceRun=run-a&redirect=https://evil.test' + ) + ).toEqual({ href: '/capacity?rebalance=1&rebalanceRun=run-a', label: 'Back to rebalance' }); + expect(provisioningReturn('/capacity?rebalance=1')).toEqual({ + href: '/capacity?rebalance=1', + label: 'Back to rebalance' + }); expect(provisioningReturn('/capacity?rebalanceRun=run-a').href).toBe('/provisioning'); const detail = fixture(); - detail.sources = [{ kind: 'rebalance', source_id: 'run-a', label: 'Rebalance', href: '/capacity?rebalance=1&rebalanceRun=run-a' }]; + detail.sources = [ + { + kind: 'rebalance', + source_id: 'run-a', + label: 'Rebalance', + href: '/capacity?rebalance=1&rebalanceRun=run-a' + } + ]; expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); detail.sources[0].href = '//evil.test'; expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(false); @@ -426,16 +934,54 @@ describe('provisioning response and return-context boundaries', () => { expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); expect(isProvisioningDetail(detail, 'another-id')).toBe(false); for (const patch of [ - { progress: undefined }, { progress: {} }, { progress: { ...detail.progress, checks: [null] } }, - { capacity: { evaluatedAt: at, pools: [{ kind: 'restart', holders: null }] } }, { activity: { total: 0 } }, - { warningMessages: {} }, { recipe: { files: [] } }, { sources: [{ kind: 'campaign', label: 'Campaign', source_id: 'a', href: 'javascript:alert(1)' }] } - ]) expect(isProvisioningDetail({ ...detail, ...patch }, detail.deployment.id)).toBe(false); - expect(isProvisioningDetail({ ...detail, progress: { ...detail.progress, command: { state: 'rejected', at, message: 'Rejected before submission' } } }, detail.deployment.id)).toBe(true); + { progress: undefined }, + { progress: {} }, + { progress: { ...detail.progress, checks: [null] } }, + { capacity: { evaluatedAt: at, pools: [{ kind: 'restart', holders: null }] } }, + { activity: { total: 0 } }, + { warningMessages: {} }, + { recipe: { files: [] } }, + { + sources: [ + { + kind: 'campaign', + label: 'Campaign', + source_id: 'a', + href: 'javascript:alert(1)' + } + ] + } + ]) + expect(isProvisioningDetail({ ...detail, ...patch }, detail.deployment.id)).toBe(false); + expect( + isProvisioningDetail( + { + ...detail, + progress: { + ...detail.progress, + command: { state: 'rejected', at, message: 'Rejected before submission' } + } + }, + detail.deployment.id + ) + ).toBe(true); }); it('rejects an invalid history event without dropping the existing history view', () => { - const page = { events: [{ id: 1, created_at: at, message: 'Queued' }], total: 1, totalEvents: 1, latestId: 1, watermark: 1, nextCursor: null }; + const page = { + events: [{ id: 1, created_at: at, message: 'Queued' }], + total: 1, + totalEvents: 1, + latestId: 1, + watermark: 1, + nextCursor: null + }; expect(isProvisioningEventPage(page)).toBe(true); - expect(isProvisioningEventPage({ ...page, events: [{ id: 1, created_at: 'invalid', message: 'Queued' }] })).toBe(false); + expect( + isProvisioningEventPage({ + ...page, + events: [{ id: 1, created_at: 'invalid', message: 'Queued' }] + }) + ).toBe(false); }); }); diff --git a/tests/unit/provisioning-progress.test.ts b/tests/unit/provisioning-progress.test.ts index c69b393..20ef53b 100644 --- a/tests/unit/provisioning-progress.test.ts +++ b/tests/unit/provisioning-progress.test.ts @@ -10,15 +10,28 @@ beforeEach(async () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-28T12:00:00Z')); }); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.useRealTimers(); +}); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('operation', 'server', 'create', 'waiting', 'Recorded stage', '2026-09-28T12:00:00Z', 'test')`).run(); - const { recordProgress, readProgress } = await import('../../src/lib/server/provisioning/progress.js'); - const read = () => readProgress(handle.prepare('SELECT observation_json FROM server_deployments WHERE id = ?').get('operation') as { observation_json: string }); + handle + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('operation', 'server', 'create', 'waiting', 'Recorded stage', '2026-09-28T12:00:00Z', 'test')` + ) + .run(); + const { recordProgress, readProgress } = + await import('../../src/lib/server/provisioning/progress.js'); + const read = () => + readProgress( + handle + .prepare('SELECT observation_json FROM server_deployments WHERE id = ?') + .get('operation') as { observation_json: string } + ); return { handle, recordProgress, read }; } @@ -26,102 +39,235 @@ describe('provisioning evidence persistence', () => { it('keeps phase duration stable and avoids semantic event spam during repeated occupancy checks', async () => { const { handle, recordProgress, read } = await setup(); for (let players = 4; players > 0; players--) { - recordProgress('operation', { phase: 'eligibility', blocker: { code: 'occupied', message: `${players} players remain`, automatic: true }, - checks: [{ key: 'occupancy', label: 'Players', source: 'telemetry', result: 'waiting', actual: players, - receivedAt: new Date().toISOString() }] }); + recordProgress('operation', { + phase: 'eligibility', + blocker: { + code: 'occupied', + message: `${players} players remain`, + automatic: true + }, + checks: [ + { + key: 'occupancy', + label: 'Players', + source: 'telemetry', + result: 'waiting', + actual: players, + receivedAt: new Date().toISOString() + } + ] + }); vi.setSystemTime(Date.now() + 15_000); } - expect(read()).toMatchObject({ phaseStartedAt: '2026-09-28T12:00:00.000Z', observedAt: '2026-09-28T12:00:45.000Z', - checks: [expect.objectContaining({ actual: 1, receivedAt: '2026-09-28T12:00:45.000Z' })] }); - expect(handle.prepare("SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'blocker'").get()).toEqual({ count: 1 }); - recordProgress('operation', { phase: 'container-action', blocker: null, - checkpoint: { key: 'eligibility', label: 'Eligibility checked', phase: 'eligibility' } }); + expect(read()).toMatchObject({ + phaseStartedAt: '2026-09-28T12:00:00.000Z', + observedAt: '2026-09-28T12:00:45.000Z', + checks: [expect.objectContaining({ actual: 1, receivedAt: '2026-09-28T12:00:45.000Z' })] + }); + expect( + handle + .prepare( + "SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'blocker'" + ) + .get() + ).toEqual({ count: 1 }); + recordProgress('operation', { + phase: 'container-action', + blocker: null, + checkpoint: { key: 'eligibility', label: 'Eligibility checked', phase: 'eligibility' } + }); expect(read()).toMatchObject({ phaseStartedAt: '2026-09-28T12:01:00.000Z', blocker: null }); }); it('replaces transient checks while retaining confirmed milestones and prior command uncertainty', async () => { const { handle, recordProgress, read } = await setup(); - recordProgress('operation', { phase: 'verify-game', command: { state: 'uncertain', message: 'Reply was lost' }, - checkpoint: { key: 'container-action', label: 'Submission attempted', phase: 'container-action' }, - checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'pass' }, - { key: 'game', label: 'Game', source: 'telemetry', result: 'pass' }] }); + recordProgress('operation', { + phase: 'verify-game', + command: { state: 'uncertain', message: 'Reply was lost' }, + checkpoint: { + key: 'container-action', + label: 'Submission attempted', + phase: 'container-action' + }, + checks: [ + { key: 'container', label: 'Container', source: 'agent', result: 'pass' }, + { key: 'game', label: 'Game', source: 'telemetry', result: 'pass' } + ] + }); vi.setSystemTime(Date.now() + 15_000); - recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'unknown' }] }); + recordProgress('operation', { + phase: 'verify-container', + checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'unknown' }] + }); expect(read()?.checks).toHaveLength(1); expect(read()?.checks[0].result).toBe('unknown'); expect(read()?.checkpoints).toHaveLength(1); expect(read()?.command?.state).toBe('uncertain'); - expect(handle.prepare("SELECT message FROM server_deployment_events WHERE kind = 'command'").get()).toEqual({ message: 'Reply was lost' }); + expect( + handle + .prepare("SELECT message FROM server_deployment_events WHERE kind = 'command'") + .get() + ).toEqual({ message: 'Reply was lost' }); }); it('preserves Plutonium release and runtime evidence when later progress is recorded', async () => { const { recordProgress, read } = await setup(); - recordProgress('operation', { phase: 'verify-plutonium', + recordProgress('operation', { + phase: 'verify-plutonium', checks: [ - { key: 'plutonium-release', label: 'Production Plutonium release', result: 'pass', source: 'manifest', actual: 5200 }, - { key: 'plutonium-version', label: 'Running Plutonium revision', result: 'pass', source: 'rcon', expected: 5200, actual: 5200 } - ], checkpoint: { key: 'verify-plutonium', label: 'Plutonium revision verified' } }); + { + key: 'plutonium-release', + label: 'Production Plutonium release', + result: 'pass', + source: 'manifest', + actual: 5200 + }, + { + key: 'plutonium-version', + label: 'Running Plutonium revision', + result: 'pass', + source: 'rcon', + expected: 5200, + actual: 5200 + } + ], + checkpoint: { key: 'verify-plutonium', label: 'Plutonium revision verified' } + }); vi.setSystemTime(Date.now() + 15_000); recordProgress('operation', { blocker: null }); - expect(read()).toMatchObject({ phase: 'verify-plutonium', phaseStartedAt: '2026-09-28T12:00:00.000Z', + expect(read()).toMatchObject({ + phase: 'verify-plutonium', + phaseStartedAt: '2026-09-28T12:00:00.000Z', checks: [ { key: 'plutonium-release', result: 'pass', source: 'manifest', actual: 5200 }, - { key: 'plutonium-version', result: 'pass', source: 'rcon', expected: 5200, actual: 5200 } - ], checkpoints: [{ key: 'verify-plutonium', phase: 'verify-plutonium' }] }); + { + key: 'plutonium-version', + result: 'pass', + source: 'rcon', + expected: 5200, + actual: 5200 + } + ], + checkpoints: [{ key: 'verify-plutonium', phase: 'verify-plutonium' }] + }); }); it('keeps operation-owned recovery evidence after slot/server removal and does not rewrite the failed outcome', async () => { const { handle, recordProgress, read } = await setup(); - handle.prepare("UPDATE server_deployments SET state = 'failed', finished_at = '2026-09-28T12:00:00Z' WHERE id = 'operation'").run(); - recordProgress('operation', { recovery: { state: 'held', message: 'Observing possible submission' } }); + handle + .prepare( + "UPDATE server_deployments SET state = 'failed', finished_at = '2026-09-28T12:00:00Z' WHERE id = 'operation'" + ) + .run(); + recordProgress('operation', { + recovery: { state: 'held', message: 'Observing possible submission' } + }); vi.setSystemTime(Date.now() + 30_000); - recordProgress('operation', { recovery: { state: 'released', message: 'Recovery verified' } }); - recordProgress('operation', { recovery: { state: 'released', message: 'Recovery verified' } }); - expect(read()?.recovery).toMatchObject({ state: 'released', at: '2026-09-28T12:00:30.000Z' }); - expect(handle.prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'operation'").get()) - .toEqual({ state: 'failed', finished_at: '2026-09-28T12:00:00Z' }); - expect(handle.prepare("SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'recovery'").get()).toEqual({ count: 2 }); + recordProgress('operation', { + recovery: { state: 'released', message: 'Recovery verified' } + }); + recordProgress('operation', { + recovery: { state: 'released', message: 'Recovery verified' } + }); + expect(read()?.recovery).toMatchObject({ + state: 'released', + at: '2026-09-28T12:00:30.000Z' + }); + expect( + handle + .prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'operation'") + .get() + ).toEqual({ state: 'failed', finished_at: '2026-09-28T12:00:00Z' }); + expect( + handle + .prepare( + "SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'recovery'" + ) + .get() + ).toEqual({ count: 2 }); }); it('rolls back state, snapshot, and milestone events together with its caller transaction', async () => { const { handle, recordProgress, read } = await setup(); - expect(() => handle.transaction(() => { - handle.prepare("UPDATE server_deployments SET state = 'running' WHERE id = 'operation'").run(); - recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Remote deletion confirmed' } }); - throw new Error('Interrupted transaction'); - })()).toThrow('Interrupted transaction'); + expect(() => + handle.transaction(() => { + handle + .prepare( + "UPDATE server_deployments SET state = 'running' WHERE id = 'operation'" + ) + .run(); + recordProgress('operation', { + phase: 'cleanup', + checkpoint: { key: 'container-action', label: 'Remote deletion confirmed' } + }); + throw new Error('Interrupted transaction'); + })() + ).toThrow('Interrupted transaction'); expect(read()).toBeNull(); - expect(handle.prepare("SELECT state FROM server_deployments WHERE id = 'operation'").get()).toEqual({ state: 'waiting' }); + expect( + handle.prepare("SELECT state FROM server_deployments WHERE id = 'operation'").get() + ).toEqual({ state: 'waiting' }); expect(handle.prepare('SELECT * FROM server_deployment_events').all()).toEqual([]); }); it('treats unsupported or malformed historical snapshots as unrecorded', async () => { const { readProgress } = await import('../../src/lib/server/provisioning/progress.js'); - for (const observation_json of [null, '', '[]', '{broken', '{"version":2}', '{"version":1,"checks":"pass"}']) { + for (const observation_json of [ + null, + '', + '[]', + '{broken', + '{"version":2}', + '{"version":1,"checks":"pass"}' + ]) { expect(readProgress({ observation_json })).toBeNull(); } }); it('adds nullable evidence columns without losing historical identity, row order, or events across reopen', async () => { - const schema = (await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8')) - .replace(',\n phase_key TEXT,\n phase_started_at TEXT,\n observed_at TEXT,\n observation_json TEXT', '') + const schema = ( + await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8') + ) + .replace( + ',\n phase_key TEXT,\n phase_started_at TEXT,\n observed_at TEXT,\n observation_json TEXT', + '' + ) .replace(',\n kind TEXT,\n phase_key TEXT,\n details_json TEXT', '') - .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'"); + .replace( + "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", + "'create', 'destroy'" + ); const old = new Database(fixture.dbPath); old.exec(schema); - old.prepare(`INSERT INTO server_deployments (rowid,id,server,server_deleted_at,action,state,stage,started_at,triggered_by) - VALUES (42,'legacy','server','2026-09-01','create','done','Recorded result','2026-08-01','test')`).run(); - old.prepare("INSERT INTO server_deployment_events (id,deployment_id,created_at,message) VALUES (19,'legacy','2026-08-01','Original evidence')").run(); + old.prepare( + `INSERT INTO server_deployments (rowid,id,server,server_deleted_at,action,state,stage,started_at,triggered_by) + VALUES (42,'legacy','server','2026-09-01','create','done','Recorded result','2026-08-01','test')` + ).run(); + old.prepare( + "INSERT INTO server_deployment_events (id,deployment_id,created_at,message) VALUES (19,'legacy','2026-08-01','Original evidence')" + ).run(); old.close(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT rowid, server_deleted_at, observation_json FROM server_deployments WHERE id = 'legacy'").get()) - .toEqual({ rowid: 42, server_deleted_at: '2026-09-01', observation_json: null }); - const { recordProgress, readProgress } = await import('../../src/lib/server/provisioning/progress.js'); - recordProgress('legacy', { recovery: { state: 'unknown', message: 'Historical slot release was not recorded' } }); + expect( + db() + .prepare( + "SELECT rowid, server_deleted_at, observation_json FROM server_deployments WHERE id = 'legacy'" + ) + .get() + ).toEqual({ rowid: 42, server_deleted_at: '2026-09-01', observation_json: null }); + const { recordProgress, readProgress } = + await import('../../src/lib/server/provisioning/progress.js'); + recordProgress('legacy', { + recovery: { state: 'unknown', message: 'Historical slot release was not recorded' } + }); closeDb(); - const saved = db().prepare("SELECT observation_json FROM server_deployments WHERE id = 'legacy'").get() as { observation_json: string }; + const saved = db() + .prepare("SELECT observation_json FROM server_deployments WHERE id = 'legacy'") + .get() as { observation_json: string }; expect(readProgress(saved)?.recovery?.state).toBe('unknown'); - expect(db().prepare('SELECT id, message FROM server_deployment_events WHERE id = 19').get()).toEqual({ id: 19, message: 'Original evidence' }); + expect( + db().prepare('SELECT id, message FROM server_deployment_events WHERE id = 19').get() + ).toEqual({ id: 19, message: 'Original evidence' }); expect(db().pragma('foreign_key_check')).toEqual([]); }); }); diff --git a/tests/unit/provisioning-queue.test.ts b/tests/unit/provisioning-queue.test.ts index abc2c86..eeb4140 100644 --- a/tests/unit/provisioning-queue.test.ts +++ b/tests/unit/provisioning-queue.test.ts @@ -1,23 +1,72 @@ import { expect, it } from 'vitest'; -import { provisioningQueueStatus, provisioningWarnings } from '../../src/lib/ui/provisioning-queue.js'; +import { + provisioningQueueStatus, + provisioningWarnings +} from '../../src/lib/ui/provisioning-queue.js'; it('distinguishes capacity, occupancy, recovery and verification without inventing errors', () => { const row = { action: 'restart' as const, state: 'waiting', stage: 'Waiting' }; - expect(provisioningQueueStatus({ ...row, waitCode: 'restart-capacity' })).toMatchObject({ label: 'Queued for restart', tone: 'neutral', group: 'queued' }); - expect(provisioningQueueStatus({ ...row, stage: 'Queued to restart; waiting for another server to recover' })).toMatchObject({ group: 'queued' }); - expect(provisioningQueueStatus({ ...row, stage: 'Waiting for 3 player(s) to leave.' })).toMatchObject({ label: 'Waiting for empty lobby', group: 'waiting' }); - expect(provisioningQueueStatus({ ...row, stage: 'Waiting for a fresh local player count.' })).toMatchObject({ label: 'Checking lobby occupancy' }); - expect(provisioningQueueStatus({ ...row, waitCode: 'operation-conflict' })).toMatchObject({ label: 'Waiting for another operation' }); - expect(provisioningQueueStatus({ ...row, state: 'awaiting-health', waitCode: 'native-host', stage: 'Waiting for container readiness: starting' })).toMatchObject({ label: 'Verifying health', group: 'verifying' }); - expect(provisioningQueueStatus({ ...row, state: 'running', waitCode: 'native-host', stage: 'Waiting for startup capacity.' })).toMatchObject({ label: 'Waiting for capacity', group: 'queued' }); - expect(provisioningQueueStatus({ ...row, phase: 'cleanup' })).toMatchObject({ label: 'Cleaning up', group: 'running' }); + expect(provisioningQueueStatus({ ...row, waitCode: 'restart-capacity' })).toMatchObject({ + label: 'Queued for restart', + tone: 'neutral', + group: 'queued' + }); + expect( + provisioningQueueStatus({ + ...row, + stage: 'Queued to restart; waiting for another server to recover' + }) + ).toMatchObject({ group: 'queued' }); + expect( + provisioningQueueStatus({ ...row, stage: 'Waiting for 3 player(s) to leave.' }) + ).toMatchObject({ label: 'Waiting for empty lobby', group: 'waiting' }); + expect( + provisioningQueueStatus({ ...row, stage: 'Waiting for a fresh local player count.' }) + ).toMatchObject({ label: 'Checking lobby occupancy' }); + expect(provisioningQueueStatus({ ...row, waitCode: 'operation-conflict' })).toMatchObject({ + label: 'Waiting for another operation' + }); + expect( + provisioningQueueStatus({ + ...row, + state: 'awaiting-health', + waitCode: 'native-host', + stage: 'Waiting for container readiness: starting' + }) + ).toMatchObject({ label: 'Verifying health', group: 'verifying' }); + expect( + provisioningQueueStatus({ + ...row, + state: 'running', + waitCode: 'native-host', + stage: 'Waiting for startup capacity.' + }) + ).toMatchObject({ label: 'Waiting for capacity', group: 'queued' }); + expect(provisioningQueueStatus({ ...row, phase: 'cleanup' })).toMatchObject({ + label: 'Cleaning up', + group: 'running' + }); }); it('keeps recorded terminal outcomes ahead of leftover progress and blockers', () => { - const row = { action: 'restart' as const, state: 'done', stage: 'Old waiting text', waitCode: 'restart-capacity', phase: 'cleanup' }; + const row = { + action: 'restart' as const, + state: 'done', + stage: 'Old waiting text', + waitCode: 'restart-capacity', + phase: 'cleanup' + }; expect(provisioningQueueStatus(row)).toMatchObject({ label: 'Complete', group: 'complete' }); - expect(provisioningQueueStatus({ ...row, state: 'failed' })).toMatchObject({ label: 'Failed', tone: 'danger', group: 'attention' }); - expect(provisioningQueueStatus({ ...row, state: 'cancelled' })).toMatchObject({ label: 'Cancelled', group: 'cancelled', tone: 'neutral' }); + expect(provisioningQueueStatus({ ...row, state: 'failed' })).toMatchObject({ + label: 'Failed', + tone: 'danger', + group: 'attention' + }); + expect(provisioningQueueStatus({ ...row, state: 'cancelled' })).toMatchObject({ + label: 'Cancelled', + group: 'cancelled', + tone: 'neutral' + }); }); it('only displays strings from recorded warning arrays', () => { diff --git a/tests/unit/provisioning-refresh.test.ts b/tests/unit/provisioning-refresh.test.ts index c87cba1..edaf8f2 100644 --- a/tests/unit/provisioning-refresh.test.ts +++ b/tests/unit/provisioning-refresh.test.ts @@ -1,46 +1,91 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createProvisioningRefresh, type ProvisioningRefreshState } from '../../src/lib/ui/provisioning-refresh.js'; +import { + createProvisioningRefresh, + type ProvisioningRefreshState +} from '../../src/lib/ui/provisioning-refresh.js'; type Detail = { id: string; revision: number }; const detail = (id: string, revision = 1): Detail => ({ id, revision }); const response = (id: string, revision = 1) => Response.json(detail(id, revision)); const controllers: { dispose: () => void }[] = []; -afterEach(() => { controllers.splice(0).forEach(controller => controller.dispose()); vi.useRealTimers(); }); +afterEach(() => { + controllers.splice(0).forEach((controller) => controller.dispose()); + vi.useRealTimers(); +}); function setup(fetcher: typeof fetch) { let state!: ProvisioningRefreshState; - const onChange = vi.fn(value => { state = value; }); - const controller = createProvisioningRefresh({ fetcher, onChange, timeoutMs: 100, - validate: (value, id): value is Detail => !!value && typeof value === 'object' && (value as Detail).id === id, - now: () => '2026-09-28T12:00:00Z' }); + const onChange = vi.fn((value) => { + state = value; + }); + const controller = createProvisioningRefresh({ + fetcher, + onChange, + timeoutMs: 100, + validate: (value, id): value is Detail => + !!value && typeof value === 'object' && (value as Detail).id === id, + now: () => '2026-09-28T12:00:00Z' + }); controllers.push(controller); return { ...controller, onChange, state: () => state }; } function deferred() { let resolve!: (value: Response) => void; let reject!: (error: Error) => void; - const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); return { promise, resolve, reject }; } describe('provisioning detail refresh ownership', () => { it('requests a rebuild retry and refreshes the original failed operation', async () => { - const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })).mockResolvedValueOnce(response('a', 2)); + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })) + .mockResolvedValueOnce(response('a', 2)); const page = setup(fetcher); page.setOperation('a', detail('a')); await page.act('retry-rebuild', 'Follow the linked recovery operation.'); - expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/retry-rebuild', expect.objectContaining({ method: 'POST' })); - expect(fetcher).toHaveBeenNthCalledWith(2, '/api/provisioning/a?events=recent', expect.anything()); - expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 2), notice: 'Follow the linked recovery operation.', busy: null }); + expect(fetcher).toHaveBeenNthCalledWith( + 1, + '/api/provisioning/a/retry-rebuild', + expect.objectContaining({ method: 'POST' }) + ); + expect(fetcher).toHaveBeenNthCalledWith( + 2, + '/api/provisioning/a?events=recent', + expect.anything() + ); + expect(page.state()).toMatchObject({ + id: 'a', + detail: detail('a', 2), + notice: 'Follow the linked recovery operation.', + busy: null + }); }); it('sends explicit override confirmation and refreshes the operation after acceptance', async () => { - const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2)); + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json({ ok: true })) + .mockResolvedValueOnce(response('a', 2)); const page = setup(fetcher); page.setOperation('a', detail('a')); await page.act('override', 'Override enabled.', { confirmed: true }); - expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/override', expect.objectContaining({ - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"confirmed":true}' - })); - expect(page.state()).toMatchObject({ detail: detail('a', 2), notice: 'Override enabled.', busy: null }); + expect(fetcher).toHaveBeenNthCalledWith( + 1, + '/api/provisioning/a/override', + expect.objectContaining({ + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{"confirmed":true}' + }) + ); + expect(page.state()).toMatchObject({ + detail: detail('a', 2), + notice: 'Override enabled.', + busy: null + }); }); it('uses the server snapshot without another initial request and coalesces background reads', async () => { const pending = deferred(); @@ -53,12 +98,22 @@ describe('provisioning detail refresh ownership', () => { expect(fetcher).toHaveBeenCalledTimes(1); pending.resolve(response('a', 2)); await Promise.all([first, second]); - expect(page.state()).toMatchObject({ detail: detail('a', 2), refreshing: false, refreshedAt: '2026-09-28T12:00:00Z' }); + expect(page.state()).toMatchObject({ + detail: detail('a', 2), + refreshing: false, + refreshedAt: '2026-09-28T12:00:00Z' + }); }); it('ignores obsolete success and failure across A → B → A, even when transport ignores abort', async () => { - const oldA = deferred(), oldB = deferred(), newA = deferred(); - const fetcher = vi.fn().mockReturnValueOnce(oldA.promise).mockReturnValueOnce(oldB.promise).mockReturnValueOnce(newA.promise); + const oldA = deferred(), + oldB = deferred(), + newA = deferred(); + const fetcher = vi + .fn() + .mockReturnValueOnce(oldA.promise) + .mockReturnValueOnce(oldB.promise) + .mockReturnValueOnce(newA.promise); const page = setup(fetcher); page.setOperation('a', detail('a')); const first = page.refresh(); @@ -71,12 +126,20 @@ describe('provisioning detail refresh ownership', () => { oldA.resolve(response('a', 0)); oldB.reject(new Error('Obsolete failure')); await Promise.all([first, second]); - expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 4), loadError: '', refreshing: false }); + expect(page.state()).toMatchObject({ + id: 'a', + detail: detail('a', 4), + loadError: '', + refreshing: false + }); }); it('invalidates a pending poll and reconciles an accepted mutation before old data can return', async () => { const old = deferred(); - const fetcher = vi.fn().mockReturnValueOnce(old.promise).mockResolvedValueOnce(Response.json({ ok: true })) + const fetcher = vi + .fn() + .mockReturnValueOnce(old.promise) + .mockResolvedValueOnce(Response.json({ ok: true })) .mockResolvedValueOnce(response('a', 3)); const page = setup(fetcher); page.setOperation('a', detail('a')); @@ -85,21 +148,42 @@ describe('provisioning detail refresh ownership', () => { old.resolve(response('a', 2)); await polling; expect(fetcher).toHaveBeenCalledTimes(3); - expect(page.state()).toMatchObject({ detail: detail('a', 3), notice: 'Verification resumed.', busy: null, loadError: '', actionError: '' }); + expect(page.state()).toMatchObject({ + detail: detail('a', 3), + notice: 'Verification resumed.', + busy: null, + loadError: '', + actionError: '' + }); }); it('retains accepted-action feedback if reconciliation fails, and action errors survive successful polls', async () => { - const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })) + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json({ ok: true })) .mockRejectedValueOnce(new Error('Offline')) - .mockResolvedValueOnce(Response.json({ error: 'Removal already began.' }, { status: 409 })) - .mockResolvedValueOnce(response('a', 2)).mockResolvedValueOnce(response('a', 3)); + .mockResolvedValueOnce( + Response.json({ error: 'Removal already began.' }, { status: 409 }) + ) + .mockResolvedValueOnce(response('a', 2)) + .mockResolvedValueOnce(response('a', 3)); const page = setup(fetcher); page.setOperation('a', detail('a')); await page.act('resume', 'Resume accepted.'); - expect(page.state()).toMatchObject({ notice: 'Resume accepted.', loadError: 'Offline', actionError: '', detail: detail('a') }); + expect(page.state()).toMatchObject({ + notice: 'Resume accepted.', + loadError: 'Offline', + actionError: '', + detail: detail('a') + }); await page.act('cancel', 'Cancelled.'); await page.refresh(); - expect(page.state()).toMatchObject({ actionError: 'Removal already began.', notice: '', loadError: '', detail: detail('a', 3) }); + expect(page.state()).toMatchObject({ + actionError: 'Removal already began.', + notice: '', + loadError: '', + detail: detail('a', 3) + }); }); it('does not duplicate mutations or poll during them and ignores completion after navigation', async () => { @@ -120,15 +204,22 @@ describe('provisioning detail refresh ownership', () => { it('times out reads without discarding the last snapshot and suppresses updates after unmount', async () => { vi.useFakeTimers(); - const fetcher = vi.fn((_url, init) => new Promise((_, reject) => { - init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); - })) as typeof fetch; + const fetcher = vi.fn( + (_url, init) => + new Promise((_, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }) + ) as typeof fetch; const page = setup(fetcher); page.setOperation('a', detail('a')); const pending = page.refresh(); await vi.advanceTimersByTimeAsync(100); await pending; - expect(page.state()).toMatchObject({ detail: detail('a'), loadError: 'Operation refresh timed out. Try again.', refreshing: false }); + expect(page.state()).toMatchObject({ + detail: detail('a'), + loadError: 'Operation refresh timed out. Try again.', + refreshing: false + }); const next = page.refresh(); page.dispose(); const calls = page.onChange.mock.calls.length; diff --git a/tests/unit/provisioning-requester.test.ts b/tests/unit/provisioning-requester.test.ts index c5c74d1..5ed98ce 100644 --- a/tests/unit/provisioning-requester.test.ts +++ b/tests/unit/provisioning-requester.test.ts @@ -3,42 +3,70 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); it('persists the manual requester while preserving source filtering and historical records', async () => { const { withRequestUsername } = await import('../../src/lib/server/auth/request-context.js'); - const { beginDeployment, getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, getDeployment, listDeployments } = + await import('../../src/lib/server/provisioning/history.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const legacy = beginDeployment('legacy', 'restart'); const [manual, automatic, rebalance] = await withRequestUsername('alice', async () => [ - beginDeployment('manual', 'restart'), beginDeployment('automatic', 'restart', 'watchdog:incident'), + beginDeployment('manual', 'restart'), + beginDeployment('automatic', 'restart', 'watchdog:incident'), beginDeployment('rebalance', 'restart', 'rebalance:run') ]); closeDb(); - expect(getDeployment(manual)?.deployment).toMatchObject({ triggered_by: 'ui', requested_by: 'alice' }); - expect(getDeployment(automatic)?.deployment).toMatchObject({ triggered_by: 'watchdog:incident', requested_by: null }); - expect(getDeployment(rebalance)?.deployment).toMatchObject({ triggered_by: 'rebalance:run', requested_by: 'alice' }); - expect(getDeployment(legacy)?.deployment).toMatchObject({ triggered_by: 'ui', requested_by: null }); - expect(listDeployments({ source: 'manual' }).map(row => row.id)).toEqual(expect.arrayContaining([manual, legacy])); - expect(listDeployments({ source: 'manual' }).map(row => row.id)).not.toContain(automatic); + expect(getDeployment(manual)?.deployment).toMatchObject({ + triggered_by: 'ui', + requested_by: 'alice' + }); + expect(getDeployment(automatic)?.deployment).toMatchObject({ + triggered_by: 'watchdog:incident', + requested_by: null + }); + expect(getDeployment(rebalance)?.deployment).toMatchObject({ + triggered_by: 'rebalance:run', + requested_by: 'alice' + }); + expect(getDeployment(legacy)?.deployment).toMatchObject({ + triggered_by: 'ui', + requested_by: null + }); + expect(listDeployments({ source: 'manual' }).map((row) => row.id)).toEqual( + expect.arrayContaining([manual, legacy]) + ); + expect(listDeployments({ source: 'manual' }).map((row) => row.id)).not.toContain(automatic); const { GET } = await import('../../src/routes/api/provisioning/+server.js'); for (const view of ['active', 'history']) { if (view === 'history') db().prepare("UPDATE server_deployments SET state='done'").run(); - const result = await GET({ url: new URL(`http://localhost/api/provisioning?view=${view}`) }).json(); - expect(result.deployments).toEqual(expect.arrayContaining([ - expect.objectContaining({ id: manual, requested_by: 'alice' }), - expect.objectContaining({ id: automatic, requested_by: null }), - expect.objectContaining({ id: rebalance, requested_by: 'alice' }), - expect.objectContaining({ id: legacy, requested_by: null }) - ])); + const result = await GET({ + url: new URL(`http://localhost/api/provisioning?view=${view}`) + }).json(); + expect(result.deployments).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: manual, requested_by: 'alice' }), + expect.objectContaining({ id: automatic, requested_by: null }), + expect.objectContaining({ id: rebalance, requested_by: 'alice' }), + expect.objectContaining({ id: legacy, requested_by: null }) + ]) + ); } }); it('isolates overlapping users and stops attributing deferred work after the request returns', async () => { - const { withRequestUsername, currentRequestUsername } = await import('../../src/lib/server/auth/request-context.js'); + const { withRequestUsername, currentRequestUsername } = + await import('../../src/lib/server/auth/request-context.js'); let release!: () => void; - const gate = new Promise(resolve => { release = resolve; }); + const gate = new Promise((resolve) => { + release = resolve; + }); let deferred!: Promise; const first = withRequestUsername('alice', async () => { deferred = gate.then(() => currentRequestUsername()); @@ -57,24 +85,41 @@ it('isolates overlapping users and stops attributing deferred work after the req it('records the operator adopting watchdog work but retains the first requester on repeated manual requests', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const { withRequestUsername } = await import('../../src/lib/server/auth/request-context.js'); db().prepare("INSERT INTO servers(name) VALUES('server')").run(); await prepareNativeHost(['server']); const id = beginDeployment('server', 'restart', 'watchdog:incident'); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); - db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) - VALUES('incident','server','health','recovering','Unhealthy','{}',?,?)`).run(new Date().toISOString(), new Date().toISOString()); - db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'watchdog','incident','server')").run(id); - expect(await withRequestUsername('alice', async () => requestLifecycle('server', 'restart'))).toMatchObject({ ok: true, deploymentId: id }); - expect(await withRequestUsername('bob', async () => requestLifecycle('server', 'restart'))).toMatchObject({ ok: true, deploymentId: id }); - expect(getDeployment(id)?.deployment).toMatchObject({ requested_by: 'alice', triggered_by: 'ui' }); + db() + .prepare( + `INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at) + VALUES('incident','server','health','recovering','Unhealthy','{}',?,?)` + ) + .run(new Date().toISOString(), new Date().toISOString()); + db() + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'watchdog','incident','server')" + ) + .run(id); + expect( + await withRequestUsername('alice', async () => requestLifecycle('server', 'restart')) + ).toMatchObject({ ok: true, deploymentId: id }); + expect( + await withRequestUsername('bob', async () => requestLifecycle('server', 'restart')) + ).toMatchObject({ ok: true, deploymentId: id }); + expect(getDeployment(id)?.deployment).toMatchObject({ + requested_by: 'alice', + triggered_by: 'ui' + }); }); it('adds the requester column to existing databases without inventing historical usernames', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, getDeployment } = + await import('../../src/lib/server/provisioning/history.js'); const id = beginDeployment('legacy', 'restart'); db().exec('ALTER TABLE server_deployments DROP COLUMN requested_by'); closeDb(); diff --git a/tests/unit/provisioning-response.test.ts b/tests/unit/provisioning-response.test.ts index 92f6ffa..3999dfe 100644 --- a/tests/unit/provisioning-response.test.ts +++ b/tests/unit/provisioning-response.test.ts @@ -1,17 +1,27 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; -import { queueCreateServer, destroyServer, requestLifecycle } from '../../src/lib/server/provisioning/index.js'; +import { + queueCreateServer, + destroyServer, + requestLifecycle +} from '../../src/lib/server/provisioning/index.js'; import { kick, kickProvisioning } from '../../src/lib/server/pipeline/worker.js'; import { POST as create } from '../../src/routes/api/servers/+server.js'; import { DELETE as destroy } from '../../src/routes/api/servers/[name]/+server.js'; import { POST as lifecycle } from '../../src/routes/api/provisioning/+server.js'; vi.mock('../../src/lib/server/provisioning/index.js', () => ({ - queueCreateServer: vi.fn(), destroyServer: vi.fn(), requestLifecycle: vi.fn() + queueCreateServer: vi.fn(), + destroyServer: vi.fn(), + requestLifecycle: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ - kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) +})); +vi.mock('../../src/lib/server/fleet/snapshot.js', () => ({ + fleetSnapshot: vi.fn(), + serverSnapshot: vi.fn() })); -vi.mock('../../src/lib/server/fleet/snapshot.js', () => ({ fleetSnapshot: vi.fn(), serverSnapshot: vi.fn() })); vi.mock('../../src/lib/server/provisioning/history.js', () => ({ deploymentListPage: vi.fn() })); beforeEach(() => { @@ -20,31 +30,63 @@ beforeEach(() => { }); afterEach(() => vi.useRealTimers()); -const request = (body: unknown) => new Request('http://localhost/api/servers', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) -}); +const request = (body: unknown) => + new Request('http://localhost/api/servers', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); const routes = [ - { name: 'create', queue: queueCreateServer, invoke: () => create({ request: request({ game: 't5', gameMode: 'zombie', mapSlug: 'theater' }) }) }, - { name: 'destroy', queue: destroyServer, invoke: () => destroy({ params: { name: 'server' } }) }, - { name: 'restart', queue: requestLifecycle, invoke: () => lifecycle({ request: request({ server: 'server', action: 'restart' }) }) } + { + name: 'create', + queue: queueCreateServer, + invoke: () => + create({ request: request({ game: 't5', gameMode: 'zombie', mapSlug: 'theater' }) }) + }, + { + name: 'destroy', + queue: destroyServer, + invoke: () => destroy({ params: { name: 'server' } }) + }, + { + name: 'restart', + queue: requestLifecycle, + invoke: () => lifecycle({ request: request({ server: 'server', action: 'restart' }) }) + } ]; -it.each(routes)('returns the queued $name job before starting provisioning or unrelated observers', async ({ queue, invoke }) => { - vi.mocked(queue).mockReturnValue({ ok: true, name: 'server', port: 28960, deploymentId: 'job', warnings: [] }); - const response = await invoke(); - expect(response.status).toBe(202); - expect(await response.json()).toMatchObject({ ok: true, deploymentId: 'job' }); - expect(queue).toHaveBeenCalledTimes(1); - expect(kickProvisioning).not.toHaveBeenCalled(); - expect(kick).not.toHaveBeenCalled(); +it.each(routes)( + 'returns the queued $name job before starting provisioning or unrelated observers', + async ({ queue, invoke }) => { + vi.mocked(queue).mockReturnValue({ + ok: true, + name: 'server', + port: 28960, + deploymentId: 'job', + warnings: [] + }); + const response = await invoke(); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ ok: true, deploymentId: 'job' }); + expect(queue).toHaveBeenCalledTimes(1); + expect(kickProvisioning).not.toHaveBeenCalled(); + expect(kick).not.toHaveBeenCalled(); - await vi.runAllTimersAsync(); - expect(kickProvisioning).toHaveBeenCalledTimes(1); - expect(kick).not.toHaveBeenCalled(); -}); + await vi.runAllTimersAsync(); + expect(kickProvisioning).toHaveBeenCalledTimes(1); + expect(kick).not.toHaveBeenCalled(); + } +); it.each(routes)('does not wake workers when $name admission fails', async ({ queue, invoke }) => { - vi.mocked(queue).mockReturnValue({ ok: false, kind: 'conflict', message: 'Busy', error: 'Busy', status: 409, warnings: [] }); + vi.mocked(queue).mockReturnValue({ + ok: false, + kind: 'conflict', + message: 'Busy', + error: 'Busy', + status: 409, + warnings: [] + }); const response = await invoke(); expect(response.status).toBe(409); expect(await response.json()).toMatchObject({ error: 'Busy' }); diff --git a/tests/unit/rcon-diagnostics.test.ts b/tests/unit/rcon-diagnostics.test.ts index 2752a5e..da602e6 100644 --- a/tests/unit/rcon-diagnostics.test.ts +++ b/tests/unit/rcon-diagnostics.test.ts @@ -2,13 +2,24 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); async function setup() { const database = await import('../../src/lib/server/db/index.js'); - database.db().prepare(`INSERT INTO servers(name,game_port,last_error,last_reply_snippet) - VALUES ('server',28960,'bad-reply:unrecognized status','private reply 192.0.2.123:28960')`).run(); + database + .db() + .prepare( + `INSERT INTO servers(name,game_port,last_error,last_reply_snippet) + VALUES ('server',28960,'bad-reply:unrecognized status','private reply 192.0.2.123:28960')` + ) + .run(); return database; } @@ -22,7 +33,8 @@ it('excludes both snippet fields from list/detail/page snapshots and initial and const { statusEntry } = await import('../../src/lib/server/pipeline/worker.js'); const { emit } = await import('../../src/lib/server/pipeline/bus.js'); const safe = (text: string) => { - for (const value of ['replySnippet', 'last_reply_snippet', 'private reply', '192.0.2.123']) expect(text).not.toContain(value); + for (const value of ['replySnippet', 'last_reply_snippet', 'private reply', '192.0.2.123']) + expect(text).not.toContain(value); }; safe(await list().text()); safe(await detail({ params: { name: 'server' } }).text()); @@ -36,7 +48,9 @@ it('excludes both snippet fields from list/detail/page snapshots and initial and const live = decoder.decode((await reader.read()).value); expect(live).toContain('event: server-status'); safe(live); - } finally { await reader.cancel(); } + } finally { + await reader.cancel(); + } }); it('serves saved failure diagnostics without caching or probing and omits successful or missing evidence', async () => { @@ -59,15 +73,29 @@ it('sanitizes manually probed failures before persistence and clears them on suc const { db } = await setup(); const rcon = await import('../../src/lib/server/integrations/rcon.js'); const probe = vi.spyOn(rcon, 'rconStatusResult'); - probe.mockResolvedValue({ ok: false, endpoint: null, reason: 'bad-reply', detail: 'unrecognized status', - rawReply: 'unrecognized player 192.0.2.123:28960' }); + probe.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'bad-reply', + detail: 'unrecognized status', + rawReply: 'unrecognized player 192.0.2.123:28960' + }); const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); await POST({ params: { name: 'server' } } as never); - expect(db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get()) - .toEqual({ last_reply_snippet: 'unrecognized player [redacted]' }); - probe.mockResolvedValue({ ok: true, endpoint: { host: 'agent', port: 1 }, status: { players: 0, map: null, gametype: null, roster: [] } }); - expect(await (await POST({ params: { name: 'server' } } as never)).json()).not.toHaveProperty('rawReply'); - expect(db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get()).toEqual({ last_reply_snippet: null }); + expect( + db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get() + ).toEqual({ last_reply_snippet: 'unrecognized player [redacted]' }); + probe.mockResolvedValue({ + ok: true, + endpoint: { host: 'agent', port: 1 }, + status: { players: 0, map: null, gametype: null, roster: [] } + }); + expect(await (await POST({ params: { name: 'server' } } as never)).json()).not.toHaveProperty( + 'rawReply' + ); + expect( + db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get() + ).toEqual({ last_reply_snippet: null }); }); it('discards legacy plain text and hex snippets and unsafe error details once during upgrade', async () => { @@ -76,9 +104,20 @@ it('discards legacy plain text and hex snippets and unsafe error details once du UPDATE servers SET last_error='bad-reply:print reply not recognized: "player 192.0.2.123:28960"'; INSERT INTO servers(name,last_reply_snippet) VALUES ('legacy-hex','0000 31 39 32 2e 30 2e 32 2e 31 32 33 3a 32 38 39 36 192.0.2.123:2896');`); closeDb(); - expect(db().prepare('SELECT last_reply_snippet FROM servers').all()).toEqual([{ last_reply_snippet: null }, { last_reply_snippet: null }]); - expect(db().prepare("SELECT last_error FROM servers WHERE name='server'").get()).toEqual({ last_error: 'bad-reply:unparseable print reply' }); - db().prepare("UPDATE servers SET last_reply_snippet='new sanitized failure' WHERE name='server'").run(); + expect(db().prepare('SELECT last_reply_snippet FROM servers').all()).toEqual([ + { last_reply_snippet: null }, + { last_reply_snippet: null } + ]); + expect(db().prepare("SELECT last_error FROM servers WHERE name='server'").get()).toEqual({ + last_error: 'bad-reply:unparseable print reply' + }); + db() + .prepare( + "UPDATE servers SET last_reply_snippet='new sanitized failure' WHERE name='server'" + ) + .run(); closeDb(); - expect(db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get()).toEqual({ last_reply_snippet: 'new sanitized failure' }); + expect( + db().prepare("SELECT last_reply_snippet FROM servers WHERE name='server'").get() + ).toEqual({ last_reply_snippet: 'new sanitized failure' }); }); diff --git a/tests/unit/rcon-status.test.ts b/tests/unit/rcon-status.test.ts index 4f7e6a1..bf5bd67 100644 --- a/tests/unit/rcon-status.test.ts +++ b/tests/unit/rcon-status.test.ts @@ -4,7 +4,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { rconConfigured } from '../../src/lib/server/config.js'; import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js'; import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js'; -import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js'; +import { + rconGetPlayerCount, + rconPlutoniumVersion, + rconStatusResult +} from '../../src/lib/server/integrations/rcon.js'; import { nativeInstance, nativeRequest } from '../../src/lib/server/backends/native.js'; vi.mock('node:dgram', () => ({ createSocket: vi.fn() })); @@ -12,10 +16,18 @@ vi.mock('../../src/lib/server/config.js', () => ({ config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } }, rconConfigured: vi.fn() })); -vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() })); -vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() })); +vi.mock('../../src/lib/server/fleet/servers.js', () => ({ + getServer: vi.fn(), + setServerField: vi.fn() +})); +vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ + latestGameSubmission: vi.fn() +})); -vi.mock('../../src/lib/server/backends/native.js', () => ({ nativeInstance: vi.fn(), nativeRequest: vi.fn() })); +vi.mock('../../src/lib/server/backends/native.js', () => ({ + nativeInstance: vi.fn(), + nativeRequest: vi.fn() +})); class FakeSocket extends EventEmitter { send = vi.fn(); @@ -24,14 +36,21 @@ class FakeSocket extends EventEmitter { let sockets: FakeSocket[]; let server: ServerRow; -const packet = (body: string) => Buffer.concat([Buffer.from([255, 255, 255, 255]), Buffer.from(body)]); -const reply = (index = 0) => sockets[index].emit('message', packet('statusResponse\n\\mapname\\mp_kino\n10 45 "joe"')); +const packet = (body: string) => + Buffer.concat([Buffer.from([255, 255, 255, 255]), Buffer.from(body)]); +const reply = (index = 0) => + sockets[index].emit('message', packet('statusResponse\n\\mapname\\mp_kino\n10 45 "joe"')); beforeEach(() => { vi.resetAllMocks(); vi.useFakeTimers(); sockets = []; - server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow; + server = { + name: 'test-server', + created_at: '2026-10-01T00:00:00Z', + game_port: 28960, + rcon_host: null + } as ServerRow; vi.mocked(getServer).mockImplementation(() => ({ ...server })); vi.mocked(latestGameSubmission).mockReturnValue(null); vi.mocked(rconConfigured).mockReturnValue(true); @@ -57,7 +76,11 @@ describe('RCON status transport', () => { if (result.ok) throw new Error('Expected an unrecognized reply'); expect(result.detail).not.toContain('player'); expect(result.detail).not.toContain('192.0.2'); - const hex = result.rawReply!.split('\n').map(line => line.slice(6, 53).trim()).join(' ').replaceAll(' ', ''); + const hex = result + .rawReply!.split('\n') + .map((line) => line.slice(6, 53).trim()) + .join(' ') + .replaceAll(' ', ''); const decoded = Buffer.from(hex, 'hex').toString(); expect(decoded).toContain('[redacted]'); expect(decoded).not.toContain('192.0.2'); @@ -70,18 +93,38 @@ describe('RCON status transport', () => { ['invalid-reply', 'bad-reply', 'Agent RCON reply is missing'] ])('preserves the native agent %s diagnosis', async (agentReason, reason, detail) => { server.native_instance_id = 'instance'; - vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never); + vi.mocked(nativeInstance).mockReturnValue({ + id: 'instance', + generation: 1, + host_id: 'host', + port: 28960 + } as never); vi.mocked(nativeRequest).mockResolvedValue({ ok: false, reason: agentReason, text: '' }); - expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason, detail: expect.stringContaining(detail) }); + expect(await rconStatusResult(server.name)).toMatchObject({ + ok: false, + reason, + detail: expect.stringContaining(detail) + }); expect(nativeRequest).toHaveBeenCalledOnce(); expect(sockets).toHaveLength(0); }); it('uses the successful native reply after agent-side recovery', async () => { server.native_instance_id = 'instance'; - vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never); - vi.mocked(nativeRequest).mockResolvedValue({ ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' }); - expect(await rconStatusResult(server.name)).toMatchObject({ ok: true, status: { players: 0, map: 'mp_kino' } }); + vi.mocked(nativeInstance).mockReturnValue({ + id: 'instance', + generation: 1, + host_id: 'host', + port: 28960 + } as never); + vi.mocked(nativeRequest).mockResolvedValue({ + ok: true, + text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' + }); + expect(await rconStatusResult(server.name)).toMatchObject({ + ok: true, + status: { players: 0, map: 'mp_kino' } + }); }); it('shares overlapping health and occupancy probes and probes again after completion', async () => { @@ -128,7 +171,7 @@ describe('RCON status transport', () => { expect(sockets).toHaveLength(3); await vi.advanceTimersByTimeAsync(1_000); expect(await request).toMatchObject({ ok: false, reason: 'timeout' }); - expect(sockets.every(socket => socket.close.mock.calls.length === 1)).toBe(true); + expect(sockets.every((socket) => socket.close.mock.calls.length === 1)).toBe(true); const fresh = rconStatusResult(server.name); await vi.advanceTimersByTimeAsync(0); @@ -137,16 +180,19 @@ describe('RCON status transport', () => { expect(await fresh).toMatchObject({ ok: true }); }); - it.each(['event', 'callback'])('retries transient send errors reported through the %s', async source => { - const request = rconStatusResult(server.name); - await vi.advanceTimersByTimeAsync(0); - if (source === 'event') sockets[0].emit('error', new Error('UDP unavailable')); - else sockets[0].send.mock.calls[0][5](new Error('UDP unavailable')); - await vi.advanceTimersByTimeAsync(250); - expect(sockets).toHaveLength(2); - reply(1); - expect(await request).toMatchObject({ ok: true }); - }); + it.each(['event', 'callback'])( + 'retries transient send errors reported through the %s', + async (source) => { + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + if (source === 'event') sockets[0].emit('error', new Error('UDP unavailable')); + else sockets[0].send.mock.calls[0][5](new Error('UDP unavailable')); + await vi.advanceTimersByTimeAsync(250); + expect(sockets).toHaveLength(2); + reply(1); + expect(await request).toMatchObject({ ok: true }); + } + ); it.each([ ['missing prefix', Buffer.from('wrong listener')], @@ -162,7 +208,10 @@ describe('RCON status transport', () => { it('does not retry missing configuration or endpoint failures', async () => { vi.mocked(rconConfigured).mockReturnValue(false); - expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-config' }); + expect(await rconStatusResult(server.name)).toMatchObject({ + ok: false, + reason: 'no-config' + }); server.game_port = null; expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' }); vi.mocked(getServer).mockReturnValue(undefined); @@ -171,7 +220,9 @@ describe('RCON status transport', () => { }); it('cleans up a rejected in-flight probe so the next one can proceed', async () => { - vi.mocked(createSocket).mockImplementationOnce(() => { throw new Error('Socket initialization failed'); }); + vi.mocked(createSocket).mockImplementationOnce(() => { + throw new Error('Socket initialization failed'); + }); await expect(rconStatusResult(server.name)).rejects.toThrow('Socket initialization failed'); const request = rconStatusResult(server.name); await vi.advanceTimersByTimeAsync(0); @@ -179,22 +230,26 @@ describe('RCON status transport', () => { expect(await request).toMatchObject({ ok: true }); }); - it.each(['restart', 'registration', 'host', 'port', 'server'])('does not share probes across a changed %s', async change => { - const original = rconStatusResult(server.name); - await vi.advanceTimersByTimeAsync(0); - if (change === 'restart') vi.mocked(latestGameSubmission).mockReturnValue('2026-10-02T00:00:00Z'); - if (change === 'registration') server.created_at = '2026-10-02T00:00:00Z'; - if (change === 'host') server.rcon_host = 'new-host'; - if (change === 'port') server.game_port = 28961; - if (change === 'server') server.name = 'other-server'; - const current = rconStatusResult(server.name); - await vi.advanceTimersByTimeAsync(0); - expect(sockets).toHaveLength(2); - reply(); - sockets[1].emit('message', packet('statusResponse\n\\mapname\\mp_ascension\n')); - expect(await original).toMatchObject({ ok: true, status: { players: 1 } }); - expect(await current).toMatchObject({ ok: true, status: { players: 0 } }); - }); + it.each(['restart', 'registration', 'host', 'port', 'server'])( + 'does not share probes across a changed %s', + async (change) => { + const original = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + if (change === 'restart') + vi.mocked(latestGameSubmission).mockReturnValue('2026-10-02T00:00:00Z'); + if (change === 'registration') server.created_at = '2026-10-02T00:00:00Z'; + if (change === 'host') server.rcon_host = 'new-host'; + if (change === 'port') server.game_port = 28961; + if (change === 'server') server.name = 'other-server'; + const current = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + expect(sockets).toHaveLength(2); + reply(); + sockets[1].emit('message', packet('statusResponse\n\\mapname\\mp_ascension\n')); + expect(await original).toMatchObject({ ok: true, status: { players: 1 } }); + expect(await current).toMatchObject({ ok: true, status: { players: 0 } }); + } + ); it('leaves other RCON commands on their existing single-attempt behavior', async () => { const request = rconPlutoniumVersion(server.name); diff --git a/tests/unit/rcon.test.ts b/tests/unit/rcon.test.ts index 228af6d..9fbc31b 100644 --- a/tests/unit/rcon.test.ts +++ b/tests/unit/rcon.test.ts @@ -10,30 +10,56 @@ import { import { redactRconAddresses } from '../../src/lib/server/integrations/rcon-diagnostics.js'; describe('private RCON diagnostics', () => { - const addresses = ['192.0.2.123:28960', '198.51.100.9', '[2001:db8::1]:28960', - '2001:db8::abcd', '[::ffff:192.0.2.4]:28960', '[fe80::1%eth0]:1234', '::1']; + const addresses = [ + '192.0.2.123:28960', + '198.51.100.9', + '[2001:db8::1]:28960', + '2001:db8::abcd', + '[::ffff:192.0.2.4]:28960', + '[fe80::1%eth0]:1234', + '::1' + ]; - it.each(addresses)('redacts %s from both plain text and decoded hex diagnostics', address => { + it.each(addresses)('redacts %s from both plain text and decoded hex diagnostics', (address) => { const text = `unrecognized status\nplayer address ${address} trailing text`; - expect(redactRconAddresses(text)).toBe('unrecognized status\nplayer address [redacted] trailing text'); + expect(redactRconAddresses(text)).toBe( + 'unrecognized status\nplayer address [redacted] trailing text' + ); const dump = formatRawReply(Buffer.from(text)); - const hex = dump.split('\n').map(line => line.slice(6, 53).trim()).join(' ').replaceAll(' ', ''); + const hex = dump + .split('\n') + .map((line) => line.slice(6, 53).trim()) + .join(' ') + .replaceAll(' ', ''); const decoded = Buffer.from(hex, 'hex').toString(); expect(decoded).toBe(redactRconAddresses(text)); }); it('redacts complete addresses before truncating native and UDP replies', () => { - const native = agentRconStatusResult({ text: 'x'.repeat(3990) + ' 192.0.2.123:28960' }, { host: 'agent', port: 1 }); + const native = agentRconStatusResult( + { text: 'x'.repeat(3990) + ' 192.0.2.123:28960' }, + { host: 'agent', port: 1 } + ); expect(native).toMatchObject({ ok: false }); expect(native.rawReply).not.toContain('192.0.2'); expect(native.rawReply!.length).toBeLessThanOrEqual(4000); const dump = formatRawReply(Buffer.from('x'.repeat(500) + ' 192.0.2.123:28960')); - const hex = dump.split('\n').filter(line => /^[0-9a-f]{4} /.test(line)).map(line => line.slice(6, 53).trim()).join(' ').replaceAll(' ', ''); + const hex = dump + .split('\n') + .filter((line) => /^[0-9a-f]{4} {2}/.test(line)) + .map((line) => line.slice(6, 53).trim()) + .join(' ') + .replaceAll(' ', ''); expect(Buffer.from(hex, 'hex').toString()).not.toContain('192.0.2'); }); it('omits native success replies while preserving the parsed status', () => { - const result = agentRconStatusResult({ text: 'print\nmap: mp_kino\nnum score ping guid name address\n--- ----- ---- ---- ---- -------\n' }, { host: 'agent', port: 1 }); + const result = agentRconStatusResult( + { + text: 'print\nmap: mp_kino\nnum score ping guid name address\n--- ----- ---- ---- ---- -------\n' + }, + { host: 'agent', port: 1 } + ); expect(result).toMatchObject({ ok: true, status: { players: 0, map: 'mp_kino' } }); expect(result).not.toHaveProperty('rawReply'); }); @@ -169,7 +195,7 @@ describe('parsePlayerRoster', () => { score: 3040, ping: 36, guid: '2741626', - name: 'JoshWright' // ^7 stripped + name: 'JoshWright' // ^7 stripped }); }); diff --git a/tests/unit/rconError.test.ts b/tests/unit/rconError.test.ts index d5de1df..2bcd47d 100644 --- a/tests/unit/rconError.test.ts +++ b/tests/unit/rconError.test.ts @@ -17,7 +17,14 @@ describe('decodeRconError', () => { }); it('recognizes every known reason code', () => { - for (const r of ['no-config', 'no-port', 'send-error', 'timeout', 'bad-reply', 'no-players'] as const) { + for (const r of [ + 'no-config', + 'no-port', + 'send-error', + 'timeout', + 'bad-reply', + 'no-players' + ] as const) { expect(decodeRconError(`${r}:whatever`)?.reason).toBe(r); } }); @@ -33,7 +40,14 @@ describe('decodeRconError', () => { describe('rconFixHint', () => { it('returns a non-empty hint for every failure reason', () => { - for (const r of ['no-config', 'no-port', 'send-error', 'timeout', 'bad-reply', 'no-players'] as const) { + for (const r of [ + 'no-config', + 'no-port', + 'send-error', + 'timeout', + 'bad-reply', + 'no-players' + ] as const) { expect(rconFixHint(r).length).toBeGreaterThan(0); } }); diff --git a/tests/unit/rebalancing-planner.test.ts b/tests/unit/rebalancing-planner.test.ts index c62e08a..a3f98a7 100644 --- a/tests/unit/rebalancing-planner.test.ts +++ b/tests/unit/rebalancing-planner.test.ts @@ -1,88 +1,166 @@ import { describe, expect, it } from 'vitest'; import { rebalanceSettings, type RebalanceSettings } from '../../src/lib/rebalancing.js'; -import { planRebalance, rebalanceMoveBlocker, type RebalancingHost, type RebalancingInstance, type RebalancingSnapshot } from '../../src/lib/server/rebalancing/planner.js'; +import { + planRebalance, + rebalanceMoveBlocker, + type RebalancingHost, + type RebalancingInstance, + type RebalancingSnapshot +} from '../../src/lib/server/rebalancing/planner.js'; const gib = 1024 ** 3; const settings: RebalanceSettings = { hostIds: ['a', 'b'], game: 'all', maxMoves: 20 }; -const host = (id: string, cpu: number, count = 0): RebalancingHost => ({ id, name: id, cpuCount: 8, memoryTotal: 16 * gib, - used: { cpuCores: cpu, memoryBytes: gib }, running: { cpuCores: 0.5, memoryBytes: gib }, - startup: { cpuCores: 1, memoryBytes: 2 * gib }, instances: count, ports: 20, reason: null }); -const instance = (n: number, source = 'a', cpu = 1): RebalancingInstance => ({ server: `t6_zombie_town_${n}`, instanceId: `i${n}`, generation: 1, - sourceHostId: source, game: 't6', relief: { cpuCores: cpu, memoryBytes: gib }, demand: { cpuCores: cpu, memoryBytes: gib }, destinations: { a: null, b: null } }); -const snapshot = (hosts = [host('a', 6, 6), host('b', 0)], instances = Array.from({ length: 6 }, (_, n) => instance(n))): RebalancingSnapshot => ({ hosts, instances, exclusions: [], warnings: [] }); +const host = (id: string, cpu: number, count = 0): RebalancingHost => ({ + id, + name: id, + cpuCount: 8, + memoryTotal: 16 * gib, + used: { cpuCores: cpu, memoryBytes: gib }, + running: { cpuCores: 0.5, memoryBytes: gib }, + startup: { cpuCores: 1, memoryBytes: 2 * gib }, + instances: count, + ports: 20, + reason: null +}); +const instance = (n: number, source = 'a', cpu = 1): RebalancingInstance => ({ + server: `t6_zombie_town_${n}`, + instanceId: `i${n}`, + generation: 1, + sourceHostId: source, + game: 't6', + relief: { cpuCores: cpu, memoryBytes: gib }, + demand: { cpuCores: cpu, memoryBytes: gib }, + destinations: { a: null, b: null } +}); +const snapshot = ( + hosts = [host('a', 6, 6), host('b', 0)], + instances = Array.from({ length: 6 }, (_, n) => instance(n)) +): RebalancingSnapshot => ({ hosts, instances, exclusions: [], warnings: [] }); describe('rebalance planning', () => { it('validates bounded hosts and moves without silently choosing all hosts', () => { expect(rebalanceSettings({ ...settings, hostIds: ['b', 'a'] }).hostIds).toEqual(['a', 'b']); - for (const input of [{}, { ...settings, hostIds: ['a'] }, { ...settings, hostIds: ['a', 'a'] }, { ...settings, game: 't4' }, - { ...settings, maxMoves: 0 }, { ...settings, maxMoves: 21 }, { ...settings, maxMoves: 2.5 }]) expect(() => rebalanceSettings(input)).toThrow(); + for (const input of [ + {}, + { ...settings, hostIds: ['a'] }, + { ...settings, hostIds: ['a', 'a'] }, + { ...settings, game: 't4' }, + { ...settings, maxMoves: 0 }, + { ...settings, maxMoves: 21 }, + { ...settings, maxMoves: 2.5 } + ]) + expect(() => rebalanceSettings(input)).toThrow(); }); it('spreads pressure with the fewest useful moves and never mutates live measurements', () => { - const input = snapshot(), saved = structuredClone(input); + const input = snapshot(), + saved = structuredClone(input); const result = planRebalance(input, settings); - expect(result.moves.map(move => [move.server, move.sourceHostId, move.targetHostId])).toEqual([0, 1, 2].map(n => [`t6_zombie_town_${n}`, 'a', 'b'])); - expect(result.hosts.map(item => item.after.instances)).toEqual([3, 3]); - expect(result.hosts.map(item => item.after.cpuPercent)).toEqual([37.5, 37.5]); - expect(new Set(result.moves.map(move => move.instanceId)).size).toBe(result.moves.length); + expect( + result.moves.map((move) => [move.server, move.sourceHostId, move.targetHostId]) + ).toEqual([0, 1, 2].map((n) => [`t6_zombie_town_${n}`, 'a', 'b'])); + expect(result.hosts.map((item) => item.after.instances)).toEqual([3, 3]); + expect(result.hosts.map((item) => item.after.cpuPercent)).toEqual([37.5, 37.5]); + expect(new Set(result.moves.map((move) => move.instanceId)).size).toBe(result.moves.length); expect(input).toEqual(saved); }); it('uses host size and both resource dimensions rather than equal counts', () => { - const small = host('a', 3, 3); small.cpuCount = 4; - const large = host('b', 4, 4); large.cpuCount = 16; - expect(planRebalance(snapshot([small, large], [instance(1)]), settings).moves).toHaveLength(1); - small.used = { cpuCores: 1, memoryBytes: 12 * gib }; large.used = { cpuCores: 1, memoryBytes: 2 * gib }; - const memory = instance(1); memory.relief.memoryBytes = 2 * gib; memory.demand.memoryBytes = 2 * gib; - expect(planRebalance(snapshot([small, large], [memory]), settings).moves[0].improvement).toBeCloseTo(12.5); + const small = host('a', 3, 3); + small.cpuCount = 4; + const large = host('b', 4, 4); + large.cpuCount = 16; + expect(planRebalance(snapshot([small, large], [instance(1)]), settings).moves).toHaveLength( + 1 + ); + small.used = { cpuCores: 1, memoryBytes: 12 * gib }; + large.used = { cpuCores: 1, memoryBytes: 2 * gib }; + const memory = instance(1); + memory.relief.memoryBytes = 2 * gib; + memory.demand.memoryBytes = 2 * gib; + expect( + planRebalance(snapshot([small, large], [memory]), settings).moves[0].improvement + ).toBeCloseTo(12.5); }); it('respects startup fit, port capacity, compatibility, host eligibility, and reservations', () => { for (const mutate of [ - (target: RebalancingHost) => { target.ports = 0; }, - (target: RebalancingHost) => { target.reason = 'Host disabled.'; }, - (target: RebalancingHost) => { target.startup.cpuCores = 8; }, - (target: RebalancingHost) => { target.startup.memoryBytes = 16 * gib; }, - (target: RebalancingHost) => { target.used.cpuCores = 7; } + (target: RebalancingHost) => { + target.ports = 0; + }, + (target: RebalancingHost) => { + target.reason = 'Host disabled.'; + }, + (target: RebalancingHost) => { + target.startup.cpuCores = 8; + }, + (target: RebalancingHost) => { + target.startup.memoryBytes = 16 * gib; + }, + (target: RebalancingHost) => { + target.used.cpuCores = 7; + } ]) { - const input = snapshot(); mutate(input.hosts[1]); + const input = snapshot(); + mutate(input.hosts[1]); expect(planRebalance(input, settings).moves).toEqual([]); } - const input = snapshot(); input.instances.forEach(item => { item.destinations.b = 'Missing the deployed runtime.'; }); + const input = snapshot(); + input.instances.forEach((item) => { + item.destinations.b = 'Missing the deployed runtime.'; + }); expect(planRebalance(input, settings).moves).toEqual([]); - expect(planRebalance(input, settings).exclusions[0].reason).toContain('Missing the deployed runtime'); - input.instances.forEach(item => { delete item.destinations.b; }); + expect(planRebalance(input, settings).exclusions[0].reason).toContain( + 'Missing the deployed runtime' + ); + input.instances.forEach((item) => { + delete item.destinations.b; + }); expect(planRebalance(input, settings).moves).toEqual([]); }); it('requires five percentage points of improvement, preserves limits, and does not move twice', () => { const input = snapshot([host('a', 4), host('b', 2)], [instance(1, 'a', 0.3)]); expect(planRebalance(input, settings).moves).toEqual([]); - input.instances[0].relief.cpuCores = 0.4; input.instances[0].demand.cpuCores = 0.4; + input.instances[0].relief.cpuCores = 0.4; + input.instances[0].demand.cpuCores = 0.4; expect(planRebalance(input, settings).moves[0].improvement).toBeCloseTo(5); expect(planRebalance(snapshot(), { ...settings, maxMoves: 1 }).moves).toHaveLength(1); expect(planRebalance(snapshot(), { ...settings, game: 't5' }).moves).toHaveLength(0); - const onePort = snapshot(); onePort.hosts[1].ports = 1; + const onePort = snapshot(); + onePort.hosts[1].ports = 1; expect(planRebalance(onePort, settings).moves).toHaveLength(1); }); it('prefers the largest improvement deterministically and revalidates the approved pair', () => { - const input = snapshot([host('a', 6, 6), host('b', 0)], [instance(3), instance(2, 'a', 2), instance(1)]); + const input = snapshot( + [host('a', 6, 6), host('b', 0)], + [instance(3), instance(2, 'a', 2), instance(1)] + ); const move = planRebalance(input, { ...settings, maxMoves: 1 }).moves[0]; expect(move.server).toBe('t6_zombie_town_2'); expect(rebalanceMoveBlocker(input, move)).toBeNull(); - expect(rebalanceMoveBlocker(input, { ...move, generation: 2 })).toContain('generation has changed'); + expect(rebalanceMoveBlocker(input, { ...move, generation: 2 })).toContain( + 'generation has changed' + ); input.hosts[1].used.cpuCores = 6; expect(rebalanceMoveBlocker(input, move)).toBeTruthy(); - input.instances = []; input.exclusions.push({ server: move.server, reason: 'Players are present.' }); + input.instances = []; + input.exclusions.push({ server: move.server, reason: 'Players are present.' }); expect(rebalanceMoveBlocker(input, move)).toBe('Players are present.'); }); it('only credits source ports actually released inside the configured range and bind', () => { - const a = host('a', 6.8, 1), b = host('b', 5.2, 1), c = host('c', 0); - a.ports = 0; b.ports = 0; c.ports = 1; - const first = instance(1, 'a', 4), second = instance(2, 'b', 1); - first.destinations.c = null; second.destinations.c = null; + const a = host('a', 6.8, 1), + b = host('b', 5.2, 1), + c = host('c', 0); + a.ports = 0; + b.ports = 0; + c.ports = 1; + const first = instance(1, 'a', 4), + second = instance(2, 'b', 1); + first.destinations.c = null; + second.destinations.c = null; const input = snapshot([a, b, c], [first, second]); const selection = { ...settings, hostIds: ['a', 'b', 'c'] }; expect(planRebalance(input, selection).moves).toHaveLength(1); diff --git a/tests/unit/rebalancing-runs.test.ts b/tests/unit/rebalancing-runs.test.ts index 287e464..9bc2bcc 100644 --- a/tests/unit/rebalancing-runs.test.ts +++ b/tests/unit/rebalancing-runs.test.ts @@ -4,12 +4,23 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; import type { RebalanceMove } from '../../src/lib/rebalancing.js'; -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ + kick: vi.fn(async () => {}), + kickProvisioning: vi.fn(async () => {}) +})); +vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ + checkRelease: vi.fn(async () => {}), + freshRelease: () => 5354, + probeVersion: vi.fn(async () => 5354) +})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })), - rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) + rconStatusResult: vi.fn(async () => ({ + ok: true, + endpoint: { host: 'agent', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + })), + rconGetPlayerCount: vi.fn(async () => 0), + rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) })); let fixture: Fixture, sourceId: string, targetId: string, libraryId: string; @@ -22,62 +33,140 @@ const settings = () => ({ hostIds: [sourceId, targetId], game: 'all' as const, m async function heartbeat(hostId: string, cpuUsagePercent: number) { const { db } = await import('../../src/lib/server/db/index.js'); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); - const instances = db().prepare('SELECT id,generation FROM native_instances WHERE host_id=? AND retired_at IS NULL') + const instances = db() + .prepare( + 'SELECT id,generation FROM native_instances WHERE host_id=? AND retired_at IS NULL' + ) .all(hostId) as { id: string; generation: number }[]; - heartbeat(hostId, { protocolVersion: 1, cpuCount: 16, cpuUsagePercent, memoryTotal: 32 * gib, - memoryAvailable: 32 * gib - (2 + instances.length * 2) * gib, dockerRunning: true, - directoriesWritable: true, storageAvailable: 100 * gib, runtimeImages: [image], - libraries: [{ id: libraryId, state: 'ready' }], instanceResources: instances.map(instance => ({ - instanceId: instance.id, generation: instance.generation, uptimeSeconds: 3600, cpuCores: 3, memoryBytes: 2 * gib })) }); + heartbeat(hostId, { + protocolVersion: 1, + cpuCount: 16, + cpuUsagePercent, + memoryTotal: 32 * gib, + memoryAvailable: 32 * gib - (2 + instances.length * 2) * gib, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * gib, + runtimeImages: [image], + libraries: [{ id: libraryId, state: 'ready' }], + instanceResources: instances.map((instance) => ({ + instanceId: instance.id, + generation: instance.generation, + uptimeSeconds: 3600, + cpuCores: 3, + memoryBytes: 2 * gib + })) + }); } beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-10-08T16:00:00.000Z')); - fixture = await newFixture([]); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'preserved-password'); + vi.resetModules(); + vi.clearAllMocks(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-08T16:00:00.000Z')); + fixture = await newFixture([]); + vi.stubEnv('PLUTAINER_IMAGE', image); + vi.stubEnv('RCON_PASSWORD', 'preserved-password'); const { db } = await import('../../src/lib/server/db/index.js'); - const at = new Date().toISOString(), launched = new Date(Date.now() - 600_000).toISOString(); - for (const [index, name] of names.entries()) db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug, + const at = new Date().toISOString(), + launched = new Date(Date.now() - 600_000).toISOString(); + for (const [index, name] of names.entries()) + db() + .prepare( + `INSERT INTO servers(name,game,map,game_mode,map_slug, config_file,game_id,image,game_port,container_state,container_checked_at,last_players,last_map,last_seen_at,last_checked_at,last_roster) - VALUES(?,'t5','zombie_theater','zombie','theater','dedicated_sp.cfg','t5sp',?,?,'running',?,0,'zombie_theater',?,?,'[]')`) - .run(name, image, 28960 + index, at, at, at); - const prepared = await prepareNativeHost(names); sourceId = prepared.host.id; libraryId = prepared.versions.get('t5')!; + VALUES(?,'t5','zombie_theater','zombie','theater','dedicated_sp.cfg','t5sp',?,?,'running',?,0,'zombie_theater',?,?,'[]')` + ) + .run(name, image, 28960 + index, at, at, at); + const prepared = await prepareNativeHost(names); + sourceId = prepared.host.id; + libraryId = prepared.versions.get('t5')!; const hosts = await import('../../src/lib/server/hosts/index.js'); targetId = hosts.createHost({ name: 'Empty destination', portMin: 29000, portMax: 29020 }).id; - hosts.enrollHost(hosts.createEnrollment(targetId).token, { protocolVersion: 1, cpuCount: 16, cpuUsagePercent: 6.25, - memoryTotal: 32 * gib, memoryAvailable: 30 * gib, dockerRunning: true, directoriesWritable: true, - storageAvailable: 100 * gib, runtimeImages: [image], libraries: [{ id: libraryId, state: 'ready' }] }); + hosts.enrollHost(hosts.createEnrollment(targetId).token, { + protocolVersion: 1, + cpuCount: 16, + cpuUsagePercent: 6.25, + memoryTotal: 32 * gib, + memoryAvailable: 30 * gib, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100 * gib, + runtimeImages: [image], + libraries: [{ id: libraryId, state: 'ready' }] + }); const libraries = await import('../../src/lib/server/game-files/index.js'); - libraries.updateHostLibraryCaches(targetId, [{ libraryVersionId: libraryId, status: 'ready', - manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]); + libraries.updateHostLibraryCaches(targetId, [ + { + libraryVersionId: libraryId, + status: 'ready', + manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 + } + ]); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { captureRecipe, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { captureRecipe, recipeStorage } = + await import('../../src/lib/server/fleet/launch-recipes.js'); const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); for (const name of names) { const instance = nativeInstance(name)!; reserveServerKey(name, 't5', `saved-key-${name}`); - for (const protocol of ['udp', 'tcp'] as const) hosts.reserveHostPort(sourceId, instance.id, { port: instance.port, bindAddress: '0.0.0.0', protocol }); + for (const protocol of ['udp', 'tcp'] as const) + hosts.reserveHostPort(sourceId, instance.id, { + port: instance.port, + bindAddress: '0.0.0.0', + protocol + }); const id = beginDeployment(name, 'create', 'fixture'); - const operation = hosts.queueHostOperation(sourceId, 'instance.create', { name, rconPassword: 'preserved-password' }, - { instanceId: instance.id, generation: 1, idempotencyKey: randomUUID() }); - hosts.reportHostOperation(sourceId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } }); - db().prepare("UPDATE server_deployments SET state='done',submitted_at=?,finished_at=? WHERE id=?").run(launched, launched, id); - db().prepare('UPDATE native_provisioning SET operation_id=?,verified_at=? WHERE deployment_id=?').run(operation.id, launched, id); + const operation = hosts.queueHostOperation( + sourceId, + 'instance.create', + { name, rconPassword: 'preserved-password' }, + { instanceId: instance.id, generation: 1, idempotencyKey: randomUUID() } + ); + hosts.reportHostOperation(sourceId, operation.id, { + state: 'completed', + result: { state: 'running', healthy: true } + }); + db() + .prepare( + "UPDATE server_deployments SET state='done',submitted_at=?,finished_at=? WHERE id=?" + ) + .run(launched, launched, id); + db() + .prepare( + 'UPDATE native_provisioning SET operation_id=?,verified_at=? WHERE deployment_id=?' + ) + .run(operation.id, launched, id); const recipe = captureRecipe('t5', 'theater', 'preserved-password'); - const stored = recipeStorage({ ...recipe, image, native: { hostId: sourceId, libraryId, image } }); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary); - for (const file of recipe.files) recordScript(name, file.path, file.versionId, file.sha256, file.size, 'native-create'); + const stored = recipeStorage({ + ...recipe, + image, + native: { hostId: sourceId, libraryId, image } + }); + db() + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)' + ) + .run(id, stored.recipe, stored.recipeSummary); + for (const file of recipe.files) + recordScript(name, file.path, file.versionId, file.sha256, file.size, 'native-create'); } - await heartbeat(sourceId, 75); await heartbeat(targetId, 6.25); + await heartbeat(sourceId, 75); + await heartbeat(targetId, 6.25); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { if (kind !== 'instance.inspect') throw new Error(`Unexpected host request ${kind}`); return { state: 'running', healthy: true, exists: true } as never; }); }); -afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { + vi.useRealTimers(); + vi.restoreAllMocks(); + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); async function start() { const service = await import('../../src/lib/server/rebalancing/index.js'); @@ -87,7 +176,7 @@ async function start() { } async function currentMove(runId: string): Promise { const { getRebalanceRun } = await import('../../src/lib/server/rebalancing/index.js'); - return getRebalanceRun(runId)!.moves.find(move => move.state === 'moving')!; + return getRebalanceRun(runId)!.moves.find((move) => move.state === 'moving')!; } async function provision() { const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); @@ -99,19 +188,31 @@ async function finishMove(move: RebalanceMove) { const { db } = await import('../../src/lib/server/db/index.js'); const removal = hostMigration(move.operationId!)!.remove_operation_id; expect(removal).toBeTruthy(); - clockStep(); hosts.reportHostOperation(sourceId, removal!, { state: 'completed' }); - await provision(); await provision(); - const created = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(move.operationId) as { operation_id: string | null }; + clockStep(); + hosts.reportHostOperation(sourceId, removal!, { state: 'completed' }); + await provision(); + await provision(); + const created = db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(move.operationId) as { operation_id: string | null }; expect(created.operation_id).toBeTruthy(); - clockStep(); hosts.reportHostOperation(targetId, created.operation_id!, { state: 'completed', result: { state: 'running', healthy: true } }); - clockStep(); await provision(); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId)).toEqual({ state: 'done' }); + clockStep(); + hosts.reportHostOperation(targetId, created.operation_id!, { + state: 'completed', + result: { state: 'running', healthy: true } + }); + clockStep(); + await provision(); + expect( + db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId) + ).toEqual({ state: 'done' }); } describe('manual rebalance runs', () => { it('previews without reserving capacity and rejects stale plans', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { previewRebalance, startRebalance } = await import('../../src/lib/server/rebalancing/index.js'); + const { previewRebalance, startRebalance } = + await import('../../src/lib/server/rebalancing/index.js'); const ports = db().prepare('SELECT * FROM ec_host_ports').all(); const commands = db().prepare('SELECT id FROM ec_host_operations').all(); const preview = previewRebalance(settings()); @@ -119,7 +220,9 @@ describe('manual rebalance runs', () => { expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual(ports); expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual(commands); expect(db().prepare('SELECT * FROM rebalancing_runs').all()).toEqual([]); - db().prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(preview.moves[0].server); + db() + .prepare("UPDATE servers SET desired_state='stopped' WHERE name=?") + .run(preview.moves[0].server); expect(() => startRebalance(preview.id)).toThrow('fleet changed'); }); @@ -133,8 +236,8 @@ describe('manual rebalance runs', () => { expect(await service.runRebalancing()).toBe(true); expect(await service.runRebalancing()).toBe(false); const moves = service.getRebalanceRun(run.id)!.moves; - expect(moves.filter(move => move.operationId)).toHaveLength(1); - expect(moves.filter(move => move.state === 'planned').length).toBeGreaterThan(0); + expect(moves.filter((move) => move.operationId)).toHaveLength(1); + expect(moves.filter((move) => move.state === 'planned').length).toBeGreaterThan(0); }); it('rejects expired previews but returns an already-started run on a late retry', async () => { @@ -154,37 +257,55 @@ describe('manual rebalance runs', () => { const move = await currentMove(run.id); const stopped = service.stopRebalance(run.id); expect(stopped.state).toBe('stopped'); - expect(stopped.moves.every(move => move.state === 'skipped')).toBe(true); + expect(stopped.moves.every((move) => move.state === 'skipped')).toBe(true); await provision(); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(nativeInstance(move.server)?.host_id).toBe(sourceId); expect(getServerKey(move.server)).toBe(`saved-key-${move.server}`); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]); + expect( + db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() + ).toEqual([]); const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(sourceId, { mode: 'draining' }); - const { reconcileHostDrains } = await import('../../src/lib/server/provisioning/host-drain.js'); + const { reconcileHostDrains } = + await import('../../src/lib/server/provisioning/host-drain.js'); reconcileHostDrains(); - expect(db().prepare("SELECT id FROM server_deployments WHERE server=? AND triggered_by='host-drain'").get(move.server)).toBeTruthy(); + expect( + db() + .prepare( + "SELECT id FROM server_deployments WHERE server=? AND triggered_by='host-drain'" + ) + .get(move.server) + ).toBeTruthy(); }); it('finishes a committed move after Stop while withdrawing the remaining moves', async () => { const run = await start(); const service = await import('../../src/lib/server/rebalancing/index.js'); - await service.runRebalancing(); await provision(); + await service.runRebalancing(); + await provision(); const move = await currentMove(run.id); expect(service.stopRebalance(run.id).state).toBe('stopping'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(move.operationId!)).toBe(false); - await finishMove(move); await service.runRebalancing(); + await finishMove(move); + await service.runRebalancing(); const finished = service.getRebalanceRun(run.id)!; expect(finished.state).toBe('stopped'); - expect(finished.moves.filter(move => move.state === 'done')).toHaveLength(1); - expect(finished.moves.filter(move => move.state === 'skipped')).toHaveLength(run.moves.length - 1); + expect(finished.moves.filter((move) => move.state === 'done')).toHaveLength(1); + expect(finished.moves.filter((move) => move.state === 'skipped')).toHaveLength( + run.moves.length - 1 + ); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const { getHost } = await import('../../src/lib/server/hosts/index.js'); - expect(nativeInstance(move.server)).toMatchObject({ id: move.instanceId, generation: 2, host_id: targetId }); + expect(nativeInstance(move.server)).toMatchObject({ + id: move.instanceId, + generation: 2, + host_id: targetId + }); expect(getHost(sourceId)?.mode).toBe('active'); }); @@ -193,27 +314,36 @@ describe('manual rebalance runs', () => { const service = await import('../../src/lib/server/rebalancing/index.js'); await service.runRebalancing(); const move = await currentMove(run.id); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + const { cancelProvisioningJob } = + await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(move.operationId!)).toBe(true); expect(service.getRebalanceRun(run.id)?.state).toBe('stopping'); await service.runRebalancing(); expect(service.getRebalanceRun(run.id)?.state).toBe('stopped'); - expect(service.getRebalanceRun(run.id)?.moves.every(move => move.state === 'skipped')).toBe(true); + expect( + service.getRebalanceRun(run.id)?.moves.every((move) => move.state === 'skipped') + ).toBe(true); }); it('waits for both hosts to report fresh measurements before admitting the next move', async () => { const run = await start(); const service = await import('../../src/lib/server/rebalancing/index.js'); - await service.runRebalancing(); await provision(); + await service.runRebalancing(); + await provision(); await finishMove(await currentMove(run.id)); expect(await service.runRebalancing()).toBe(false); expect(service.getRebalanceRun(run.id)?.message).toContain('fresh source and destination'); - expect(service.getRebalanceRun(run.id)?.moves.filter(move => move.operationId)).toHaveLength(1); - clockStep(); await heartbeat(sourceId, 56.25); + expect( + service.getRebalanceRun(run.id)?.moves.filter((move) => move.operationId) + ).toHaveLength(1); + clockStep(); + await heartbeat(sourceId, 56.25); expect(await service.runRebalancing()).toBe(false); await heartbeat(targetId, 25); expect(await service.runRebalancing()).toBe(true); - expect(service.getRebalanceRun(run.id)?.moves.filter(move => move.operationId)).toHaveLength(2); + expect( + service.getRebalanceRun(run.id)?.moves.filter((move) => move.operationId) + ).toHaveLength(2); }); it('skips an uncommitted move that becomes occupied and continues other approved work', async () => { @@ -226,8 +356,8 @@ describe('manual rebalance runs', () => { await service.runRebalancing(); const fresh = service.getRebalanceRun(run.id)!; expect(fresh.state).toBe('running'); - expect(fresh.moves.find(item => item.id === move.id)?.state).toBe('skipped'); - expect(fresh.moves.filter(item => item.state === 'moving')).toHaveLength(1); + expect(fresh.moves.find((item) => item.id === move.id)?.state).toBe('skipped'); + expect(fresh.moves.filter((item) => item.state === 'moving')).toHaveLength(1); }); it('halts remaining moves after a provisioning failure', async () => { @@ -236,12 +366,21 @@ describe('manual rebalance runs', () => { await service.runRebalancing(); const move = await currentMove(run.id); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE server_deployments SET state='failed',error='Recovery needed' WHERE id=?").run(move.operationId); + db() + .prepare( + "UPDATE server_deployments SET state='failed',error='Recovery needed' WHERE id=?" + ) + .run(move.operationId); expect(await service.runRebalancing()).toBe(false); const failed = service.getRebalanceRun(run.id)!; expect(failed.state).toBe('failed'); - expect(failed.moves.find(item => item.id === move.id)).toMatchObject({ state: 'failed', message: 'Recovery needed' }); - expect(failed.moves.filter(item => item.state === 'skipped')).toHaveLength(run.moves.length - 1); + expect(failed.moves.find((item) => item.id === move.id)).toMatchObject({ + state: 'failed', + message: 'Recovery needed' + }); + expect(failed.moves.filter((item) => item.state === 'skipped')).toHaveLength( + run.moves.length - 1 + ); }); it('keeps the source running until a global startup slot is available', async () => { @@ -263,37 +402,66 @@ describe('manual rebalance runs', () => { it('retains its global startup reservation between source removal and destination creation', async () => { const run = await start(); const service = await import('../../src/lib/server/rebalancing/index.js'); - await service.runRebalancing(); await provision(); + await service.runRebalancing(); + await provision(); const move = await currentMove(run.id); const { hostMigration } = await import('../../src/lib/server/provisioning/migrations.js'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); - reportHostOperation(sourceId, hostMigration(move.operationId!)!.remove_operation_id!, { state: 'completed' }); + reportHostOperation(sourceId, hostMigration(move.operationId!)!.remove_operation_id!, { + state: 'completed' + }); await provision(); const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId)).toEqual({ state: 'waiting' }); + expect( + db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId) + ).toEqual({ state: 'waiting' }); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); - const extra = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', - hostId: targetId, libraryId, image, plutoKey: 'another-key' }); + const extra = queueNativeCreation({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + hostId: targetId, + libraryId, + image, + plutoKey: 'another-key' + }); expect(extra.ok).toBe(true); if (!extra.ok) throw new Error(extra.message); await provision(); - expect(db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(extra.deploymentId)).toBeUndefined(); - expect(db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(move.operationId)) - .toEqual({ operation_id: expect.any(String) }); + expect( + db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(extra.deploymentId) + ).toBeUndefined(); + expect( + db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(move.operationId) + ).toEqual({ operation_id: expect.any(String) }); }); - it.each([{ state: 'running', healthy: false }, { state: 'stopped', healthy: false }])('holds removal when fresh inspection returns $state with healthy=$healthy', async health => { - const run = await start(); - const service = await import('../../src/lib/server/rebalancing/index.js'); - await service.runRebalancing(); - const move = await currentMove(run.id); - const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); - vi.mocked(nativeRequest).mockResolvedValueOnce(health); - await provision(); - const { hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js'); - expect(hostMigration(move.operationId!)?.remove_operation_id).toBeNull(); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT stage FROM server_deployments WHERE id=?').get(move.operationId)) - .toEqual({ stage: expect.stringContaining('no longer healthy') }); - }); + it.each([ + { state: 'running', healthy: false }, + { state: 'stopped', healthy: false } + ])( + 'holds removal when fresh inspection returns $state with healthy=$healthy', + async (health) => { + const run = await start(); + const service = await import('../../src/lib/server/rebalancing/index.js'); + await service.runRebalancing(); + const move = await currentMove(run.id); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeRequest).mockResolvedValueOnce(health); + await provision(); + const { hostMigration } = + await import('../../src/lib/server/provisioning/host-drain.js'); + expect(hostMigration(move.operationId!)?.remove_operation_id).toBeNull(); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare('SELECT stage FROM server_deployments WHERE id=?') + .get(move.operationId) + ).toEqual({ stage: expect.stringContaining('no longer healthy') }); + } + ); }); diff --git a/tests/unit/rebalancing-snapshot.test.ts b/tests/unit/rebalancing-snapshot.test.ts index 12bf3ae..c3e1cf7 100644 --- a/tests/unit/rebalancing-snapshot.test.ts +++ b/tests/unit/rebalancing-snapshot.test.ts @@ -5,40 +5,118 @@ import type { HostInventory } from '../../src/lib/hosts.js'; import type { RebalanceSettings } from '../../src/lib/rebalancing.js'; // Artifact preservation has its own migration tests; these exercise live admission evidence. -vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({ captureMoveRecipe: vi.fn(() => ({})) })); -const server = 't6_zombie_town_1', gib = 1024 ** 3; +vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({ + captureMoveRecipe: vi.fn(() => ({})) +})); +const server = 't6_zombie_town_1', + gib = 1024 ** 3; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.useRealTimers(); + vi.restoreAllMocks(); + await fixture.cleanup(); +}); async function setup() { - const source = await prepareNativeHost([server]), target = await prepareNativeHost(); + const source = await prepareNativeHost([server]), + target = await prepareNativeHost(); const { db } = await import('../../src/lib/server/db/index.js'); const { ensureRebalancingSchema } = await import('../../src/lib/server/rebalancing/schema.js'); ensureRebalancingSchema(db()); - const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base); + const base = Date.now(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(base); const at = new Date().toISOString(); - const native = db().prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?').get(server) as { id: string; generation: number; library_id: string }; - const original = db().prepare('SELECT inventory FROM ec_hosts WHERE id=?').get(source.host.id) as { inventory: string }; + const native = db() + .prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?') + .get(server) as { id: string; generation: number; library_id: string }; + const original = db() + .prepare('SELECT inventory FROM ec_hosts WHERE id=?') + .get(source.host.id) as { inventory: string }; const inventory = JSON.parse(original.inventory) as HostInventory; - const resources = { instanceId: native.id, generation: native.generation, cpuCores: 2, memoryBytes: 2 * gib, uptimeSeconds: 600 }; - db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 75, - memoryTotal: 16 * gib, memoryAvailable: 8 * gib, instanceResources: [resources] }), at, source.host.id); - db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 0, - memoryTotal: 16 * gib, memoryAvailable: 15 * gib, instanceResources: [] }), at, target.host.id); + const resources = { + instanceId: native.id, + generation: native.generation, + cpuCores: 2, + memoryBytes: 2 * gib, + uptimeSeconds: 600 + }; + db() + .prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?') + .run( + JSON.stringify({ + ...inventory, + cpuCount: 8, + cpuUsagePercent: 75, + memoryTotal: 16 * gib, + memoryAvailable: 8 * gib, + instanceResources: [resources] + }), + at, + source.host.id + ); + db() + .prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?') + .run( + JSON.stringify({ + ...inventory, + cpuCount: 8, + cpuUsagePercent: 0, + memoryTotal: 16 * gib, + memoryAvailable: 15 * gib, + instanceResources: [] + }), + at, + target.host.id + ); // Target has a verified copy of the source's deployed library version. - const cache = db().prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?').get(native.library_id) as { manifest_sha256: string }; + const cache = db() + .prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?') + .get(native.library_id) as { manifest_sha256: string }; const libraries = await import('../../src/lib/server/game-files/index.js'); - libraries.updateHostLibraryCaches(target.host.id, [{ libraryVersionId: native.library_id, status: 'ready', manifestSha256: cache.manifest_sha256 }]); - db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players) - VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(server, at, at, at); - db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')").run(); + libraries.updateHostLibraryCaches(target.host.id, [ + { + libraryVersionId: native.library_id, + status: 'ready', + manifestSha256: cache.manifest_sha256 + } + ]); + db() + .prepare( + `INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players) + VALUES(?,'t6','zm_transit','town','running',?,?,?,0)` + ) + .run(server, at, at, at); + db() + .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')") + .run(); db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key')").run(server); - const settings: RebalanceSettings = { hostIds: [source.host.id, target.host.id], game: 'all', maxMoves: 10 }; + const settings: RebalanceSettings = { + hostIds: [source.host.id, target.host.id], + game: 'all', + maxMoves: 10 + }; const api = await import('../../src/lib/server/rebalancing/snapshot.js'); const planner = await import('../../src/lib/server/rebalancing/planner.js'); const read = (except = '') => api.rebalanceSnapshot(settings, except); - return { db, source, target, native, inventory, resources, settings, read, ...api, ...planner, base, at }; + return { + db, + source, + target, + native, + inventory, + resources, + settings, + read, + ...api, + ...planner, + base, + at + }; } it('selects healthy empty native instances on active hosts without creating host work', async () => { @@ -48,106 +126,208 @@ it('selects healthy empty native instances on active hosts without creating host expect(snapshot.instances).toHaveLength(1); const plan = s.planRebalance(snapshot, s.settings); expect(plan.moves).toHaveLength(1); - expect(plan.moves[0]).toMatchObject({ sourceHostId: s.source.host.id, targetHostId: s.target.host.id }); + expect(plan.moves[0]).toMatchObject({ + sourceHostId: s.source.host.id, + targetHostId: s.target.host.id + }); expect(s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get()).toEqual(before); - expect(s.rebalanceHosts().every(host => host.eligible)).toBe(true); - expect(s.read().hosts.find(host => host.id === s.source.host.id)?.used.cpuCores).toBe(6); + expect(s.rebalanceHosts().every((host) => host.eligible)).toBe(true); + expect(s.read().hosts.find((host) => host.id === s.source.host.id)?.used.cpuCores).toBe(6); }); -it.each(['occupied', 'unknown', 'stopped', 'maintenance', 'recovery', 'lifecycle', 'generation', 'stale-host', 'future-host', 'startup', 'artifacts'] as const) - ('excludes %s instances with an explanation', async condition => { - const s = await setup(); - if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run(); - if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run(); - if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run(); - if (condition === 'maintenance') s.db().prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)').run(server, s.at); - if (condition === 'recovery') s.db().prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, s.at); - if (condition === 'lifecycle') s.db().prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) - VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`).run(s.source.host.id, s.native.id, s.at, s.at); - if (condition === 'generation') s.db().prepare('UPDATE native_instances SET generation=2').run(); - if (condition === 'stale-host' || condition === 'future-host') s.db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') - .run(new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(), s.source.host.id); - if (condition === 'startup') s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?").run(s.source.host.id); - if (condition === 'artifacts') vi.mocked((await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe).mockImplementationOnce(() => { throw new Error('Deployed files are unavailable.'); }); - const snapshot = s.read(); - expect(snapshot.instances).toEqual([]); - expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) }); - }); +it.each([ + 'occupied', + 'unknown', + 'stopped', + 'maintenance', + 'recovery', + 'lifecycle', + 'generation', + 'stale-host', + 'future-host', + 'startup', + 'artifacts' +] as const)('excludes %s instances with an explanation', async (condition) => { + const s = await setup(); + if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run(); + if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run(); + if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run(); + if (condition === 'maintenance') + s.db() + .prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)') + .run(server, s.at); + if (condition === 'recovery') + s.db() + .prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)') + .run(server, s.at); + if (condition === 'lifecycle') + s.db() + .prepare( + `INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) + VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)` + ) + .run(s.source.host.id, s.native.id, s.at, s.at); + if (condition === 'generation') + s.db().prepare('UPDATE native_instances SET generation=2').run(); + if (condition === 'stale-host' || condition === 'future-host') + s.db() + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run( + new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(), + s.source.host.id + ); + if (condition === 'startup') + s.db() + .prepare( + "UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?" + ) + .run(s.source.host.id); + if (condition === 'artifacts') + vi.mocked( + (await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe + ).mockImplementationOnce(() => { + throw new Error('Deployed files are unavailable.'); + }); + const snapshot = s.read(); + expect(snapshot.instances).toEqual([]); + expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) }); +}); it('honors a current positive RCON veto even when telemetry reports empty', async () => { - const s = await setup(), status = await import('../../src/lib/server/fleet/status.js'); + const s = await setup(), + status = await import('../../src/lib/server/fleet/status.js'); const observe = status.observationReader(); - vi.spyOn(status, 'observationReader').mockReturnValue(row => ({ ...observe(row), last_players: 0, status_source: 'telemetry' })); + vi.spyOn(status, 'observationReader').mockReturnValue((row) => ({ + ...observe(row), + last_players: 0, + status_source: 'telemetry' + })); s.db().prepare('UPDATE servers SET last_players=1').run(); expect(s.read().exclusions[0].reason).toContain('RCON'); }); it('uses conservative recent host pressure and credits only matching observed generation relief', async () => { const s = await setup(); - s.db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) - VALUES(?,?,8,90,?,?)`).run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib); - s.db().prepare(`INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) - VALUES(?,?,1,'running',?,1,?)`).run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib); + s.db() + .prepare( + `INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) + VALUES(?,?,8,90,?,?)` + ) + .run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib); + s.db() + .prepare( + `INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) + VALUES(?,?,1,'running',?,1,?)` + ) + .run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib); const snapshot = s.read(); - expect(snapshot.hosts.find(host => host.id === s.source.host.id)?.used).toEqual({ cpuCores: 7.2, memoryBytes: 10 * gib }); + expect(snapshot.hosts.find((host) => host.id === s.source.host.id)?.used).toEqual({ + cpuCores: 7.2, + memoryBytes: 10 * gib + }); expect(snapshot.instances[0].relief).toEqual({ cpuCores: 1, memoryBytes: gib }); expect(snapshot.instances[0].demand).toEqual({ cpuCores: 2, memoryBytes: 2 * gib }); }); it('counts a pending migration once before and after ownership changes and never assumes unplaced pinning', async () => { const s = await setup(); - s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) - VALUES('moving',?,'rebuild','waiting','Move',?,'test')`).run(server, s.at); - s.db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run('moving', s.native.id); - s.db().prepare(`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id) - VALUES('moving',?,?,?)`).run(s.source.host.id, s.native.id, s.target.host.id); - const target = () => s.read().hosts.find(host => host.id === s.target.host.id)!; + s.db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('moving',?,'rebuild','waiting','Move',?,'test')` + ) + .run(server, s.at); + s.db() + .prepare( + 'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)' + ) + .run('moving', s.native.id); + s.db() + .prepare( + `INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id) + VALUES('moving',?,?,?)` + ) + .run(s.source.host.id, s.native.id, s.target.host.id); + const target = () => s.read().hosts.find((host) => host.id === s.target.host.id)!; expect(target().used.cpuCores).toBe(0.5); expect(target().used.memoryBytes).toBe(2 * gib); - expect(s.read('moving').hosts.find(host => host.id === s.target.host.id)?.used.cpuCores).toBe(0); + expect(s.read('moving').hosts.find((host) => host.id === s.target.host.id)?.used.cpuCores).toBe( + 0 + ); s.db().prepare('UPDATE native_instances SET host_id=?,generation=2').run(s.target.host.id); s.db().prepare('UPDATE native_provisioning SET generation=2').run(); s.db().prepare('UPDATE ec_host_migrations SET removed_at=?').run(s.at); expect(target().used.cpuCores).toBe(0.5); expect(target().used.memoryBytes).toBe(2 * gib); - s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?") + s.db() + .prepare( + "UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?" + ) .run(JSON.stringify([{ ...s.resources, generation: 2 }]), 13 * gib, s.target.host.id); expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib }); - s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) - VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`).run(s.at); - s.db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run('unplaced', JSON.stringify({ hostId: s.target.host.id })); + s.db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')` + ) + .run(s.at); + s.db() + .prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)') + .run('unplaced', JSON.stringify({ hostId: s.target.host.id })); expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib }); expect(target().instances).toBe(1); }); it('allows admission to ignore its own unsubmitted rebuild while retaining the source empty evidence', async () => { const s = await setup(); - s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) - VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`).run(server, s.at); + s.db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')` + ) + .run(server, s.at); expect(s.read().instances).toHaveLength(0); expect(s.read('own').instances).toHaveLength(1); }); it('enforces the 30 minute cooldown across generations after a successful move', async () => { const s = await setup(); - s.db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`).run(s.at, s.at); - s.db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at) - VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`).run(server, s.native.id, s.target.host.id, s.source.host.id, s.at); + s.db() + .prepare( + `INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')` + ) + .run(s.at, s.at); + s.db() + .prepare( + `INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at) + VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)` + ) + .run(server, s.native.id, s.target.host.id, s.source.host.id, s.at); expect(s.read().exclusions[0].reason).toContain('30 minutes'); - s.db().prepare('UPDATE rebalancing_moves SET updated_at=?').run(new Date(s.base - 30 * 60_000 - 1).toISOString()); + s.db() + .prepare('UPDATE rebalancing_moves SET updated_at=?') + .run(new Date(s.base - 30 * 60_000 - 1).toISOString()); expect(s.read().instances).toHaveLength(1); }); it('credits a freed game port only when the current range and bind will actually gain a port', async () => { const s = await setup(); expect(s.read().instances[0].releasesPort).toBe(false); - s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) - VALUES(?,?,'0.0.0.0','udp',28960,?)`).run(s.source.host.id, s.native.id, s.at); + s.db() + .prepare( + `INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) + VALUES(?,?,'0.0.0.0','udp',28960,?)` + ) + .run(s.source.host.id, s.native.id, s.at); expect(s.read().instances[0].releasesPort).toBe(true); s.db().prepare('UPDATE ec_hosts SET port_min=28970 WHERE id=?').run(s.source.host.id); expect(s.read().instances[0].releasesPort).toBe(false); s.db().prepare('UPDATE ec_hosts SET port_min=28960 WHERE id=?').run(s.source.host.id); - s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) - VALUES(?,'other','127.0.0.2','tcp',28960,?)`).run(s.source.host.id, s.at); + s.db() + .prepare( + `INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) + VALUES(?,'other','127.0.0.2','tcp',28960,?)` + ) + .run(s.source.host.id, s.at); expect(s.read().instances[0].releasesPort).toBe(false); }); diff --git a/tests/unit/report-workers.test.ts b/tests/unit/report-workers.test.ts index 9436d46..d24527e 100644 --- a/tests/unit/report-workers.test.ts +++ b/tests/unit/report-workers.test.ts @@ -3,8 +3,14 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { parseMetricsQuery } from '../../src/lib/metrics.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); it('runs reports off the request thread and invalidates cached history after writes', async () => { const { db } = await import('../../src/lib/server/db/index.js'); @@ -16,7 +22,10 @@ it('runs reports off the request thread and invalidates cached history after wri recordMetrics([sample], now - 30_000); const query = parseMetricsQuery(new URLSearchParams('days=1')); const first = metricsReport(query); - const winner = await Promise.race([first.then(() => 'report'), new Promise(resolve => setTimeout(() => resolve('timer'), 0))]); + const winner = await Promise.race([ + first.then(() => 'report'), + new Promise((resolve) => setTimeout(() => resolve('timer'), 0)) + ]); expect(winner).toBe('timer'); expect(await first).toEqual(readMetrics(query, now)); expect(await metricsReport(query)).toEqual(await first); diff --git a/tests/unit/retention-storage.test.ts b/tests/unit/retention-storage.test.ts index c9df51a..936ebaf 100644 --- a/tests/unit/retention-storage.test.ts +++ b/tests/unit/retention-storage.test.ts @@ -1,11 +1,19 @@ import Database from 'better-sqlite3'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { enableIncrementalVacuum, retentionClocks } from '../../src/lib/server/db/retention-clocks.js'; +import { + enableIncrementalVacuum, + retentionClocks +} from '../../src/lib/server/db/retention-clocks.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); const old = new Date(Date.now() - 90 * 86400000).toISOString(); it('expires completed file payloads while keeping history, replay identity, holds and active work', async () => { @@ -14,9 +22,16 @@ it('expires completed file payloads while keeping history, replay identity, hold const { pruneHistory, setRetentionControl } = await import('../../src/lib/server/retention.js'); const { putFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); const host = hosts.createHost({ name: 'Files' }); - const payload = { files: [{ path: 'plugin.dll', contentBase64: Buffer.from('file').toString('base64') }] }; - const completed = hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'finished' }); - hosts.reportHostOperation(host.id, completed.id, { state: 'completed', result: { before: [{ path: 'plugin.dll', contentBase64: 'YmVmb3Jl' }] } }); + const payload = { + files: [{ path: 'plugin.dll', contentBase64: Buffer.from('file').toString('base64') }] + }; + const completed = hosts.queueHostOperation(host.id, 'files.stage', payload, { + idempotencyKey: 'finished' + }); + hosts.reportHostOperation(host.id, completed.id, { + state: 'completed', + result: { before: [{ path: 'plugin.dll', contentBase64: 'YmVmb3Jl' }] } + }); const active = hosts.queueHostOperation(host.id, 'files.stage', payload); const held = hosts.queueHostOperation(host.id, 'files.stage', payload); hosts.reportHostOperation(host.id, held.id, { state: 'completed', result: {} }); @@ -26,22 +41,49 @@ it('expires completed file payloads while keeping history, replay identity, hold db().prepare('UPDATE ec_host_operations SET updated_at=?').run(old); db().exec("INSERT INTO servers(name) VALUES('s')"); const hash = putFileBlob(Buffer.from('recipe-only')); - db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) - VALUES('launch','s','create','done','done',?,?,'test')`).run(old, old); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)') - .run('launch', JSON.stringify({ files: [{ path: 'launch.dll', sha256: hash, contentHash: hash }] })); + db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) + VALUES('launch','s','create','done','done',?,?,'test')` + ) + .run(old, old); + db() + .prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)') + .run( + 'launch', + JSON.stringify({ files: [{ path: 'launch.dll', sha256: hash, contentHash: hash }] }) + ); expect(pruneHistory('workflow-payload')).toBe(1); - expect(db().prepare("SELECT state FROM server_deployments WHERE id='launch'").get()).toEqual({ state: 'done' }); - expect(JSON.parse((db().prepare('SELECT recipe FROM server_launch_recipes').get() as { recipe: string }).recipe)) - .toEqual({ files: [{ path: 'launch.dll', sha256: hash }] }); + expect(db().prepare("SELECT state FROM server_deployments WHERE id='launch'").get()).toEqual({ + state: 'done' + }); + expect( + JSON.parse( + (db().prepare('SELECT recipe FROM server_launch_recipes').get() as { recipe: string }) + .recipe + ) + ).toEqual({ files: [{ path: 'launch.dll', sha256: hash }] }); expect(db().prepare('SELECT 1 FROM file_blobs WHERE sha256=?').get(hash)).toBeUndefined(); expect(pruneHistory('operation-payload')).toBe(1); - expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', payload: { files: [{ path: 'plugin.dll' }] }, result: { before: [{ path: 'plugin.dll' }] } }); + expect(hosts.getHostOperation(completed.id)).toMatchObject({ + state: 'completed', + payload: { files: [{ path: 'plugin.dll' }] }, + result: { before: [{ path: 'plugin.dll' }] } + }); expect(hosts.getHostOperation(active.id)?.payload).toEqual(payload); expect(hosts.getHostOperation(held.id)?.payload).toEqual(payload); expect(hosts.getHostOperation(failed.id)?.payload).toEqual(payload); - expect(hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'finished' }).id).toBe(completed.id); - expect(() => hosts.queueHostOperation(host.id, 'files.stage', { files: [] }, { idempotencyKey: 'finished' })).toThrow('different request'); + expect( + hosts.queueHostOperation(host.id, 'files.stage', payload, { idempotencyKey: 'finished' }).id + ).toBe(completed.id); + expect(() => + hosts.queueHostOperation( + host.id, + 'files.stage', + { files: [] }, + { idempotencyKey: 'finished' } + ) + ).toThrow('different request'); expect(pruneHistory('operation-payload')).toBe(0); }); @@ -52,16 +94,37 @@ it('retains rebalance recovery files until the move is reconciled, then lets his const { putFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); ensureHostSchema(db()); - saveFleetSettings({ ...fleetSettings(), historyKeepCount: 0, provisioningKeepDays: 1, payloadKeepDays: 1 }); - db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) - VALUES('move','s','rebuild','done','healthy',?,?,'rebalance:run')`).run(old, old); + saveFleetSettings({ + ...fleetSettings(), + historyKeepCount: 0, + provisioningKeepDays: 1, + payloadKeepDays: 1 + }); + db() + .prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) + VALUES('move','s','rebuild','done','healthy',?,?,'rebalance:run')` + ) + .run(old, old); const hash = putFileBlob(Buffer.from('frozen deployed file')); - db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)') - .run('move', JSON.stringify({ files: [{ path: 'script.gsc', sha256: hash, contentHash: hash }] })); - db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) - VALUES('run','{}','running',?,?,'waiting')`).run(old, old); - db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,operation_id,message,updated_at) - VALUES('m','run',0,'s','i',1,'a','b','{}','moving','move','waiting',?)`).run(old); + db() + .prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)') + .run( + 'move', + JSON.stringify({ files: [{ path: 'script.gsc', sha256: hash, contentHash: hash }] }) + ); + db() + .prepare( + `INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) + VALUES('run','{}','running',?,?,'waiting')` + ) + .run(old, old); + db() + .prepare( + `INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,operation_id,message,updated_at) + VALUES('m','run',0,'s','i',1,'a','b','{}','moving','move','waiting',?)` + ) + .run(old); expect(pruneHistory('workflow-payload')).toBe(0); expect(pruneHistory('provisioning')).toBe(0); db().prepare("UPDATE rebalancing_moves SET state='failed'").run(); @@ -70,7 +133,10 @@ it('retains rebalance recovery files until the move is reconciled, then lets his db().prepare("UPDATE rebalancing_moves SET state='done'").run(); expect(pruneHistory('workflow-payload')).toBe(1); expect(pruneHistory('provisioning')).toBe(1); - expect(db().prepare('SELECT state,operation_id FROM rebalancing_moves').get()).toEqual({ state: 'done', operation_id: null }); + expect(db().prepare('SELECT state,operation_id FROM rebalancing_moves').get()).toEqual({ + state: 'done', + operation_id: null + }); }); it('normalizes legacy retention clocks once and reuses clock indexes', () => { @@ -80,13 +146,17 @@ it('normalizes legacy retention clocks once and reuses clock indexes', () => { INSERT INTO history(at) VALUES('2026-01-01 12:00:00'),('2026-01-01T08:00:00-04:00'),(NULL)`); retentionClocks(handle, { history: ['at'] }); expect(handle.prepare('SELECT at FROM history ORDER BY id').all()).toEqual([ - { at: '2026-01-01T12:00:00.000Z' }, { at: '2026-01-01T12:00:00.000Z' }, { at: null } + { at: '2026-01-01T12:00:00.000Z' }, + { at: '2026-01-01T12:00:00.000Z' }, + { at: null } ]); expect(handle.pragma('index_list(history)')).toHaveLength(1); const prepare = vi.spyOn(handle, 'prepare'); retentionClocks(handle, { history: ['at'] }); expect(prepare.mock.calls.some(([sql]) => sql.startsWith('UPDATE'))).toBe(false); - } finally { handle.close(); } + } finally { + handle.close(); + } }); it('enables incremental vacuum once and preserves host enable command boundaries', () => { @@ -98,12 +168,19 @@ it('enables incremental vacuum once and preserves host enable command boundaries INSERT INTO ec_host_enable_requests VALUES('h',20)`); enableIncrementalVacuum(handle); expect(handle.pragma('auto_vacuum', { simple: true })).toBe(2); - const eligible = () => handle.prepare(`SELECT id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id - WHERE o.rowid>e.operation_floor`).all(); + const eligible = () => + handle + .prepare( + `SELECT id FROM ec_host_operations o JOIN ec_host_enable_requests e ON e.host_id=o.host_id + WHERE o.rowid>e.operation_floor` + ) + .all(); expect(eligible()).toEqual([{ id: 'after' }]); const exec = vi.spyOn(handle, 'exec'); enableIncrementalVacuum(handle); expect(exec.mock.calls).toHaveLength(0); expect(eligible()).toEqual([{ id: 'after' }]); - } finally { handle.close(); } + } finally { + handle.close(); + } }); diff --git a/tests/unit/retention.test.ts b/tests/unit/retention.test.ts index f8d97d2..2733cb2 100644 --- a/tests/unit/retention.test.ts +++ b/tests/unit/retention.test.ts @@ -2,45 +2,74 @@ import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -const old = new Date(Date.now()-90*86400000).toISOString(); -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +const old = new Date(Date.now() - 90 * 86400000).toISOString(); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); async function api() { const { db } = await import('../../src/lib/server/db/index.js'); const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js'); ensureHostSchema(db()); - return { db: db(), ...await import('../../src/lib/server/retention.js'), ...await import('../../src/lib/server/settings.js') }; + return { + db: db(), + ...(await import('../../src/lib/server/retention.js')), + ...(await import('../../src/lib/server/settings.js')) + }; } it('previews eligible campaigns, respects latest-N and explicit holds, and treats zero as indefinite', async () => { const a = await api(); - for (const id of ['a','b','c']) a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old); - a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 1,historyKeepCount: 1 }); - a.setRetentionControl({ category: 'campaign',id: 'a',protected: true }); - expect(a.retentionPreview().find(p=>p.category==='campaign')?.eligible).toBe(1); + for (const id of ['a', 'b', 'c']) + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)" + ) + .run(id, old, old); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, historyKeepCount: 1 }); + a.setRetentionControl({ category: 'campaign', id: 'a', protected: true }); + expect(a.retentionPreview().find((p) => p.category === 'campaign')?.eligible).toBe(1); expect(a.pruneHistory('campaign')).toBe(1); - expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([{ id:'a' },{ id:'c' }]); - a.setRetentionControl({ category: 'campaign',id: 'a',protected: false }); - a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,historyKeepCount: 0 }); + expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([ + { id: 'a' }, + { id: 'c' } + ]); + a.setRetentionControl({ category: 'campaign', id: 'a', protected: false }); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, historyKeepCount: 0 }); expect(a.pruneHistory('campaign')).toBe(0); }); it('seeks expired campaigns and uses indexed dependency checks even with failure retention enabled', async () => { const a = await api(); const now = new Date().toISOString(); - const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)"); + const insert = a.db.prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)" + ); a.db.transaction(() => { - for (let n=0;n<8000;n++) insert.run(`recent-${n}`,now,now); + for (let n = 0; n < 8000; n++) insert.run(`recent-${n}`, now, now); })(); - a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7,failureKeepDays:7}); - const prepare = vi.spyOn(a.db,'prepare'); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 7, failureKeepDays: 7 }); + const prepare = vi.spyOn(a.db, 'prepare'); let sql: string; try { expect(a.pruneHistory('campaign')).toBe(0); sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT o.rowid AS rid'))![0]; - } finally { prepare.mockRestore(); } - const plan = a.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all({category:'campaign',cutoff:old,failureCutoff:old}) as {detail:string}[]; - const details = plan.map(row=>row.detail).join('\n'); - expect(details).toMatch(/SEARCH o USING (?:COVERING )?INDEX idx_deployments_started \(started_at<\?\)/); - for (const index of ['idx_deployments_rolled_back_by','idx_watchdog_actions_campaign','idx_provisioning_sources_source']) { + } finally { + prepare.mockRestore(); + } + const plan = a.db + .prepare(`EXPLAIN QUERY PLAN ${sql}`) + .all({ category: 'campaign', cutoff: old, failureCutoff: old }) as { detail: string }[]; + const details = plan.map((row) => row.detail).join('\n'); + expect(details).toMatch( + /SEARCH o USING (?:COVERING )?INDEX idx_deployments_started \(started_at<\?\)/ + ); + for (const index of [ + 'idx_deployments_rolled_back_by', + 'idx_watchdog_actions_campaign', + 'idx_provisioning_sources_source' + ]) { expect(details).toContain(index); } expect(details).not.toMatch(/SCAN [odas]\b/); @@ -48,36 +77,76 @@ it('seeks expired campaigns and uses indexed dependency checks even with failure it('selects resolved failures separately while preserving age, protection and batch limits', async () => { const a = await api(); const now = new Date().toISOString(); - for (const id of ['normal','unresolved','resolved','held','new-resolution','changed']) { - a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)").run(id,old,old); - if (id !== 'normal') a.db.prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,'s',1,'failed')").run(id); + for (const id of ['normal', 'unresolved', 'resolved', 'held', 'new-resolution', 'changed']) { + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES(?,'drift',?,?)" + ) + .run(id, old, old); + if (id !== 'normal') + a.db + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES(?,'s',1,'failed')" + ) + .run(id); } - for (const id of ['resolved','held','new-resolution','changed']) { - a.setRetentionControl({category:'campaign',id,resolve:true,protected:id==='held'}); - if (id !== 'new-resolution') a.db.prepare("UPDATE retention_controls SET resolved_at=? WHERE record_id=?").run(old,id); + for (const id of ['resolved', 'held', 'new-resolution', 'changed']) { + a.setRetentionControl({ + category: 'campaign', + id, + resolve: true, + protected: id === 'held' + }); + if (id !== 'new-resolution') + a.db + .prepare('UPDATE retention_controls SET resolved_at=? WHERE record_id=?') + .run(old, id); } a.db.prepare("UPDATE deployments SET started_at=? WHERE id='changed'").run(now); // Failure retention works independently of normal campaign retention. - a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0,failureKeepDays:7}); - expect(a.retentionPreview().find(p=>p.category==='campaign')?.records).toEqual([{id:'resolved'}]); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, failureKeepDays: 7 }); + expect(a.retentionPreview().find((p) => p.category === 'campaign')?.records).toEqual([ + { id: 'resolved' } + ]); expect(a.pruneHistory('campaign')).toBe(1); - a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:7}); - const insert = a.db.prepare("INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)"); - a.db.transaction(() => { for (let n=0;n<600;n++) insert.run(`old-${n}`,old); })(); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 7 }); + const insert = a.db.prepare( + "INSERT INTO deployments(id,action,started_at) VALUES(?,'drift',?)" + ); + a.db.transaction(() => { + for (let n = 0; n < 600; n++) insert.run(`old-${n}`, old); + })(); expect(a.pruneHistory('campaign')).toBe(500); expect(a.pruneHistory('campaign')).toBe(101); expect(a.db.prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([ - {id:'changed'},{id:'held'},{id:'new-resolution'},{id:'unresolved'} + { id: 'changed' }, + { id: 'held' }, + { id: 'new-resolution' }, + { id: 'unresolved' } ]); }); it('expires file bodies without silently turning rollback into file deletion', async () => { const a = await api(); - a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)").run(old,old); - a.db.prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash,content_after_hash) VALUES('p','s','updated','test.cfg',?,?)").run(putFileBlob(Buffer.from('before'), a.db),putFileBlob(Buffer.from('after'), a.db)); - a.saveFleetSettings({ ...a.fleetSettings(),deployKeepDays: 0,fileContentKeepDays: 1 }); - expect(a.retentionPreview().find(p=>p.category==='file-content')?.payloadBytes).toBe(11); + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES('p','promote',?,?)" + ) + .run(old, old); + a.db + .prepare( + "INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_before_hash,content_after_hash) VALUES('p','s','updated','test.cfg',?,?)" + ) + .run(putFileBlob(Buffer.from('before'), a.db), putFileBlob(Buffer.from('after'), a.db)); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0, fileContentKeepDays: 1 }); + expect(a.retentionPreview().find((p) => p.category === 'file-content')?.payloadBytes).toBe(11); expect(a.pruneHistory('file-content')).toBe(1); - expect(a.db.prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,(SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files").get()).toEqual({ content_before:null,content_after:null }); + expect( + a.db + .prepare( + 'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,(SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files' + ) + .get() + ).toEqual({ content_before: null, content_after: null }); const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); await expect(rollbackPromoteDeployment('p')).rejects.toThrow('expired'); expect(a.pruneHistory('file-content')).toBe(0); @@ -85,147 +154,304 @@ it('expires file bodies without silently turning rollback into file deletion', a it('requires explicit failure resolution and preserves recovery slots even after resolution', async () => { const a = await api(); a.db.prepare("INSERT INTO servers(name) VALUES('s')").run(); - a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('failure','s','restart','failed','failed',?,?,'test')").run(old,old); - a.db.prepare("INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('s','failure',?)").run(old); - a.saveFleetSettings({ ...a.fleetSettings(),provisioningKeepDays: 1,failureKeepDays: 1 }); + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('failure','s','restart','failed','failed',?,?,'test')" + ) + .run(old, old); + a.db + .prepare( + "INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('s','failure',?)" + ) + .run(old); + a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, failureKeepDays: 1 }); expect(a.pruneHistory('provisioning')).toBe(0); - a.setRetentionControl({ category:'provisioning',id:'failure',resolve:true }); - a.db.prepare("UPDATE retention_controls SET resolved_at=?").run(old); + a.setRetentionControl({ category: 'provisioning', id: 'failure', resolve: true }); + a.db.prepare('UPDATE retention_controls SET resolved_at=?').run(old); expect(a.pruneHistory('provisioning')).toBe(0); - a.db.prepare("UPDATE restart_rollout SET healthy_at=?").run(new Date().toISOString()); + a.db.prepare('UPDATE restart_rollout SET healthy_at=?').run(new Date().toISOString()); expect(a.pruneHistory('provisioning')).toBe(1); }); it('compacts action history but preserves idempotency and terminal receipts', async () => { const a = await api(); const hosts = await import('../../src/lib/server/hosts/index.js'); - const host = hosts.createHost({ name:'Host' }); - const op = hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' }); - hosts.reportHostOperation(host.id,op.id,{ state:'completed',result:{ done:true } }); - a.db.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(old,op.id); - a.saveFleetSettings({ ...a.fleetSettings(),actionKeepDays:1 }); + const host = hosts.createHost({ name: 'Host' }); + const op = hosts.queueHostOperation( + host.id, + 'files.stage', + { file: 'large' }, + { idempotencyKey: 'original' } + ); + hosts.reportHostOperation(host.id, op.id, { state: 'completed', result: { done: true } }); + a.db.prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(old, op.id); + a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1 }); expect(a.pruneHistory('operation')).toBe(1); expect(hosts.listHostOperations(host.id)).toHaveLength(0); - expect(hosts.queueHostOperation(host.id,'files.stage',{ file:'large' },{ idempotencyKey:'original' }).id).toBe(op.id); - expect(()=>hosts.queueHostOperation(host.id,'files.stage',{ file:'different' },{ idempotencyKey:'original' })).toThrow('different request'); + expect( + hosts.queueHostOperation( + host.id, + 'files.stage', + { file: 'large' }, + { idempotencyKey: 'original' } + ).id + ).toBe(op.id); + expect(() => + hosts.queueHostOperation( + host.id, + 'files.stage', + { file: 'different' }, + { idempotencyKey: 'original' } + ) + ).toThrow('different request'); expect(hosts.pendingHostOperations(host.id)).toHaveLength(0); - expect(hosts.reportHostOperation(host.id,op.id,{state:'running'}).state).toBe('completed'); + expect(hosts.reportHostOperation(host.id, op.id, { state: 'running' }).state).toBe('completed'); }); it('deletes old maintenance jobs while preserving update suppression evidence and protected jobs', async () => { const a = await api(); - for (const kind of ['legacy','schedule','update']) a.db.prepare("INSERT INTO maintenance_jobs(server,state,kind,requested_at,finished_at) VALUES(?,'done',?,?,?)").run(kind,kind,old,old); - a.setRetentionControl({category:'maintenance',id:'1',protected:true}); + for (const kind of ['legacy', 'schedule', 'update']) + a.db + .prepare( + "INSERT INTO maintenance_jobs(server,state,kind,requested_at,finished_at) VALUES(?,'done',?,?,?)" + ) + .run(kind, kind, old, old); + a.setRetentionControl({ category: 'maintenance', id: '1', protected: true }); expect(a.pruneHistory('maintenance')).toBe(1); - expect(a.db.prepare('SELECT kind FROM maintenance_jobs ORDER BY id').all()).toEqual([{kind:'legacy'},{kind:'update'}]); + expect(a.db.prepare('SELECT kind FROM maintenance_jobs ORDER BY id').all()).toEqual([ + { kind: 'legacy' }, + { kind: 'update' } + ]); }); it('keeps current and referenced artifact versions and honors the minimum version count', async () => { const a = await api(); - a.db.prepare("INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)").run(); - for(let n=1;n<=4;n++) a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash,uploaded_at) VALUES(?,1,?,'sha',1,?,?)").run(n,n,putFileBlob(Buffer.from([1]), a.db),old); - a.saveFleetSettings({...a.fleetSettings(),versionKeepCount:2}); + a.db + .prepare( + "INSERT INTO scripts(id,name,target_rel_path,current_version_id) VALUES(1,'test','test.gsc',1)" + ) + .run(); + for (let n = 1; n <= 4; n++) + a.db + .prepare( + "INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash,uploaded_at) VALUES(?,1,?,'sha',1,?,?)" + ) + .run(n, n, putFileBlob(Buffer.from([1]), a.db), old); + a.saveFleetSettings({ ...a.fleetSettings(), versionKeepCount: 2 }); expect(a.pruneHistory('version')).toBe(1); - expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([{id:1},{id:3},{id:4}]); + expect(a.db.prepare('SELECT id FROM script_versions ORDER BY id').all()).toEqual([ + { id: 1 }, + { id: 3 }, + { id: 4 } + ]); }); it('prunes metrics and diagnostics while leaving unlimited histories intact', async () => { const a = await api(); - a.db.prepare("INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,'s','t5','kino',0,4)").run(Date.parse(old)); - a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:0}); + a.db + .prepare( + "INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,'s','t5','kino',0,4)" + ) + .run(Date.parse(old)); + a.saveFleetSettings({ ...a.fleetSettings(), metricsKeepDays: 0 }); a.pruneSampleHistory(); - expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:1}); - a.saveFleetSettings({...a.fleetSettings(),metricsKeepDays:1}); + expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({ n: 1 }); + a.saveFleetSettings({ ...a.fleetSettings(), metricsKeepDays: 1 }); a.pruneSampleHistory(); - expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({n:0}); + expect(a.db.prepare('SELECT count(*) AS n FROM metric_samples').get()).toEqual({ n: 0 }); }); it('prunes closed matches and audit events without deleting active matches or protected audit records', async () => { const a = await api(); - for (const status of ['active','completed','interrupted']) a.db.prepare(`INSERT INTO leaderboard_matches(server,game,map,map_key,map_name,started_at,last_received_at,status,last_event,last_fingerprint) - VALUES('s','t5','kino','kino','Kino',?,?,?,'game_end','test')`).run(old,old,status); + for (const status of ['active', 'completed', 'interrupted']) + a.db + .prepare( + `INSERT INTO leaderboard_matches(server,game,map,map_key,map_name,started_at,last_received_at,status,last_event,last_fingerprint) + VALUES('s','t5','kino','kino','Kino',?,?,?,'game_end','test')` + ) + .run(old, old, status); // Preview initializes the optional authentication schema. a.retentionPreview(); - a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old)); - a.db.prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)").run(Date.parse(old)); - a.setRetentionControl({category:'audit',id:'1',protected:true}); - a.saveFleetSettings({...a.fleetSettings(),matchKeepDays:1,auditKeepDays:1}); + a.db + .prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)") + .run(Date.parse(old)); + a.db + .prepare("INSERT INTO auth_audit(action,details,created_at) VALUES('login','{}',?)") + .run(Date.parse(old)); + a.setRetentionControl({ category: 'audit', id: '1', protected: true }); + a.saveFleetSettings({ ...a.fleetSettings(), matchKeepDays: 1, auditKeepDays: 1 }); expect(a.pruneHistory('match')).toBe(2); expect(a.pruneHistory('audit')).toBe(1); - expect(a.db.prepare('SELECT status FROM leaderboard_matches').all()).toEqual([{status:'active'}]); + expect(a.db.prepare('SELECT status FROM leaderboard_matches').all()).toEqual([ + { status: 'active' } + ]); expect(a.db.pragma('foreign_key_check')).toEqual([]); }); it('keeps active beta evidence while expiring closed snapshots and then closed history', async () => { const a = await api(); a.db.prepare("INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'s','s.gsc')").run(); - a.db.prepare("INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash) VALUES(1,1,1,'sha',3,?)").run(putFileBlob(Buffer.from([1,2,3]), a.db)); - for (const [id,state] of [[1,'running'],[2,'rolled-back']] as const) { - a.db.prepare("INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)").run(id,state,old,old); - a.db.prepare("INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)").run(id,old); - a.db.prepare("INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content_hash) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',?)").run(id,putFileBlob(Buffer.from([1,2,3]), a.db)); + a.db + .prepare( + "INSERT INTO script_versions(id,script_id,version,sha256,size,content_hash) VALUES(1,1,1,'sha',3,?)" + ) + .run(putFileBlob(Buffer.from([1, 2, 3]), a.db)); + for (const [id, state] of [ + [1, 'running'], + [2, 'rolled-back'] + ] as const) { + a.db + .prepare( + "INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,opened_at,closed_at) VALUES(?,1,1,'test',?,?,?)" + ) + .run(id, state, old, old); + a.db + .prepare('INSERT INTO beta_revisions(beta_id,revision,started_at) VALUES(?,1,?)') + .run(id, old); + a.db + .prepare( + "INSERT INTO beta_revision_files(beta_id,revision,script_id,name,kind,path,version_id,version,sha256,content_hash) VALUES(?,1,1,'s','script','s.gsc',1,1,'sha',?)" + ) + .run(id, putFileBlob(Buffer.from([1, 2, 3]), a.db)); } - a.saveFleetSettings({...a.fleetSettings(),fileContentKeepDays:1}); + a.saveFleetSettings({ ...a.fleetSettings(), fileContentKeepDays: 1 }); expect(a.pruneHistory('beta-file-content')).toBe(1); - expect(a.db.prepare('SELECT beta_id,COALESCE((SELECT size FROM file_blobs WHERE sha256=content_hash),0) AS bytes FROM beta_revision_files ORDER BY beta_id').all()).toEqual([{beta_id:1,bytes:3},{beta_id:2,bytes:0}]); - a.saveFleetSettings({...a.fleetSettings(),betaKeepDays:1}); + expect( + a.db + .prepare( + 'SELECT beta_id,COALESCE((SELECT size FROM file_blobs WHERE sha256=content_hash),0) AS bytes FROM beta_revision_files ORDER BY beta_id' + ) + .all() + ).toEqual([ + { beta_id: 1, bytes: 3 }, + { beta_id: 2, bytes: 0 } + ]); + a.saveFleetSettings({ ...a.fleetSettings(), betaKeepDays: 1 }); expect(a.pruneHistory('beta')).toBe(1); - expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{id:1}]); + expect(a.db.prepare('SELECT id FROM betas').all()).toEqual([{ id: 1 }]); expect(a.db.pragma('foreign_key_check')).toEqual([]); }); it('applies minimum count and holds to routine cleanup and its preview consistently', async () => { const a = await api(); const hosts = await import('../../src/lib/server/hosts/index.js'); - const host = hosts.createHost({name:'Host'}); - for (let n=1;n<=3;n++) a.db.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,observation_success,routine) VALUES(?,?,'files.read','{}','completed',?,?,1,1)").run(String(n),host.id,old,old); - a.setRetentionControl({category:'observation',id:'1',protected:true}); - a.saveFleetSettings({...a.fleetSettings(),historyKeepCount:1}); - expect(a.retentionPreview().find(p=>p.category==='observation')?.eligible).toBe(1); + const host = hosts.createHost({ name: 'Host' }); + for (let n = 1; n <= 3; n++) + a.db + .prepare( + "INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,observation_success,routine) VALUES(?,?,'files.read','{}','completed',?,?,1,1)" + ) + .run(String(n), host.id, old, old); + a.setRetentionControl({ category: 'observation', id: '1', protected: true }); + a.saveFleetSettings({ ...a.fleetSettings(), historyKeepCount: 1 }); + expect(a.retentionPreview().find((p) => p.category === 'observation')?.eligible).toBe(1); expect(a.pruneHistory('observation')).toBe(1); - expect(a.db.prepare('SELECT id FROM ec_host_operations ORDER BY id').all()).toEqual([{id:'1'},{id:'3'}]); + expect(a.db.prepare('SELECT id FROM ec_host_operations ORDER BY id').all()).toEqual([ + { id: '1' }, + { id: '3' } + ]); }); it('never removes the calibration window when choosing a shorter resource retention policy', async () => { const a = await api(); const hosts = await import('../../src/lib/server/hosts/index.js'); - const host = hosts.createHost({name:'Host'}); - a.db.prepare('INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,?)').run(host.id,old); - a.db.prepare("INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,'i',1,'startup',?,1,1024)").run(host.id,old); - a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:0,instanceResourceKeepDays:0}); + const host = hosts.createHost({ name: 'Host' }); + a.db + .prepare('INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,?)') + .run(host.id, old); + a.db + .prepare( + "INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,'i',1,'startup',?,1,1024)" + ) + .run(host.id, old); + a.saveFleetSettings({ + ...a.fleetSettings(), + hostResourceKeepDays: 0, + instanceResourceKeepDays: 0 + }); a.pruneSampleHistory(); - expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:1}); - a.saveFleetSettings({...a.fleetSettings(),hostResourceKeepDays:1,instanceResourceKeepDays:1}); + expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({ + n: 1 + }); + a.saveFleetSettings({ + ...a.fleetSettings(), + hostResourceKeepDays: 1, + instanceResourceKeepDays: 1 + }); a.pruneSampleHistory(); - expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({n:0}); - expect(a.db.prepare('SELECT count(*) AS n FROM ec_host_resource_samples').get()).toEqual({n:0}); + expect(a.db.prepare('SELECT count(*) AS n FROM ec_instance_resource_samples').get()).toEqual({ + n: 0 + }); + expect(a.db.prepare('SELECT count(*) AS n FROM ec_host_resource_samples').get()).toEqual({ + n: 0 + }); }); it('previews draft retention without saving it and restricts retention tools to administrators', async () => { const a = await api(); - a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('preview','drift',?,?)").run(old,old); - a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:0}); - const {POST} = await import('../../src/routes/api/settings/retention/preview/+server.js'); - const response = await POST({request:new Request('http://localhost/api/settings/retention/preview',{method:'POST',body:JSON.stringify({...a.fleetSettings(),deployKeepDays:1})})}); + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES('preview','drift',?,?)" + ) + .run(old, old); + a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 0 }); + const { POST } = await import('../../src/routes/api/settings/retention/preview/+server.js'); + const response = await POST({ + request: new Request('http://localhost/api/settings/retention/preview', { + method: 'POST', + body: JSON.stringify({ ...a.fleetSettings(), deployKeepDays: 1 }) + }) + }); expect(response.status).toBe(200); - expect((await response.json()).policies.find((p:{category:string})=>p.category==='campaign').eligible).toBe(1); + expect( + (await response.json()).policies.find( + (p: { category: string }) => p.category === 'campaign' + ).eligible + ).toBe(1); expect(a.fleetSettings().deployKeepDays).toBe(0); - expect(a.db.prepare('SELECT id FROM deployments').get()).toEqual({id:'preview'}); - const {routePermission} = await import('../../src/lib/server/auth/access.js'); - expect(routePermission('/api/settings/retention','GET')).toBe('admin'); - expect(routePermission('/api/settings/retention/preview','POST')).toBe('admin'); + expect(a.db.prepare('SELECT id FROM deployments').get()).toEqual({ id: 'preview' }); + const { routePermission } = await import('../../src/lib/server/auth/access.js'); + expect(routePermission('/api/settings/retention', 'GET')).toBe('admin'); + expect(routePermission('/api/settings/retention/preview', 'POST')).toBe('admin'); }); it('protects cancelled workflows and linked file history through ambiguous and unreconciled receipts', async () => { const a = await api(); const hosts = await import('../../src/lib/server/hosts/index.js'); - const host=hosts.createHost({name:'Host'}); - a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id,old); - a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','restart','cancelled','cancelled',?,?,'test')").run(old,old); - const command=hosts.queueHostOperation(host.id,'instance.restart',{stageId:'campaign'}); + const host = hosts.createHost({ name: 'Host' }); + a.db + .prepare( + "INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)" + ) + .run(host.id, old); + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','restart','cancelled','cancelled',?,?,'test')" + ) + .run(old, old); + const command = hosts.queueHostOperation(host.id, 'instance.restart', { stageId: 'campaign' }); a.db.prepare("UPDATE ec_host_operations SET state='unknown' WHERE id=?").run(command.id); - a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,files_json) VALUES('cancelled','instance',1,?,'[{\"id\":1}]')").run(command.id); - a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old,old); - a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run(); - a.saveFleetSettings({...a.fleetSettings(),deployKeepDays:1,provisioningKeepDays:1,fileContentKeepDays:1}); + a.db + .prepare( + "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,files_json) VALUES('cancelled','instance',1,?,'[{\"id\":1}]')" + ) + .run(command.id); + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)" + ) + .run(old, old); + a.db + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')" + ) + .run(); + a.saveFleetSettings({ + ...a.fleetSettings(), + deployKeepDays: 1, + provisioningKeepDays: 1, + fileContentKeepDays: 1 + }); expect(a.pruneHistory('campaign')).toBe(0); expect(a.pruneHistory('file-content')).toBe(0); expect(a.pruneHistory('provisioning')).toBe(0); a.db.prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(command.id); expect(a.pruneHistory('campaign')).toBe(0); expect(a.pruneHistory('provisioning')).toBe(0); - a.db.prepare("UPDATE native_provisioning SET applied_at=? WHERE deployment_id='cancelled'").run(new Date().toISOString()); + a.db + .prepare("UPDATE native_provisioning SET applied_at=? WHERE deployment_id='cancelled'") + .run(new Date().toISOString()); expect(a.pruneHistory('campaign')).toBe(1); expect(a.pruneHistory('provisioning')).toBe(1); expect(a.db.pragma('foreign_key_check')).toEqual([]); @@ -236,25 +462,77 @@ it('retains a cancelled rebuild and its source until the partial-failure receipt const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Host' }); a.db.prepare("INSERT INTO servers(name,container_state) VALUES('s','running')").run(); - a.db.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)").run(host.id, old); - a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')").run(old, old); - const command = hosts.queueHostOperation(host.id, 'instance.rebuild', { nextGeneration: 2 }, { instanceId: 'instance', generation: 1 }); - a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)").run(command.id); - a.db.prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)").run(old, old); - a.db.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')").run(); - hosts.reportHostOperation(host.id, command.id, { state: 'failed', error: 'Replacement creation failed', result: { - reconciled: true, rebuildFailure: { phase: 'disrupting', generation: 1, nextGeneration: 2, - oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true } - } }); - a.saveFleetSettings({ ...a.fleetSettings(), deployKeepDays: 1, provisioningKeepDays: 1, historyKeepCount: 0 }); + a.db + .prepare( + "INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','s',?,'lib','image',1,?)" + ) + .run(host.id, old); + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')" + ) + .run(old, old); + const command = hosts.queueHostOperation( + host.id, + 'instance.rebuild', + { nextGeneration: 2 }, + { instanceId: 'instance', generation: 1 } + ); + a.db + .prepare( + "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES('cancelled','instance',1,?)" + ) + .run(command.id); + a.db + .prepare( + "INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','promote',?,?)" + ) + .run(old, old); + a.db + .prepare( + "INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('cancelled','campaign','campaign','s')" + ) + .run(); + hosts.reportHostOperation(host.id, command.id, { + state: 'failed', + error: 'Replacement creation failed', + result: { + reconciled: true, + rebuildFailure: { + phase: 'disrupting', + generation: 1, + nextGeneration: 2, + oldContainerMissing: true, + replacementContainerMissing: true, + replacementFilesRemoved: true + } + } + }); + a.saveFleetSettings({ + ...a.fleetSettings(), + deployKeepDays: 1, + provisioningKeepDays: 1, + historyKeepCount: 0 + }); expect(a.pruneHistory('campaign')).toBe(0); expect(a.pruneHistory('provisioning')).toBe(0); - expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: null }); + expect( + a.db + .prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'") + .get() + ).toEqual({ applied_at: null }); - const { reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js'); + const { reconcileNativeProvisioningReceipt } = + await import('../../src/lib/server/provisioning/native.js'); reconcileNativeProvisioningReceipt('cancelled'); - expect(a.db.prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'").get()).toEqual({ applied_at: expect.any(String) }); - expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({ container_state: 'missing' }); + expect( + a.db + .prepare("SELECT applied_at FROM native_provisioning WHERE deployment_id='cancelled'") + .get() + ).toEqual({ applied_at: expect.any(String) }); + expect(a.db.prepare("SELECT container_state FROM servers WHERE name='s'").get()).toEqual({ + container_state: 'missing' + }); expect(a.pruneHistory('campaign')).toBe(1); expect(a.pruneHistory('provisioning')).toBe(1); expect(a.db.pragma('foreign_key_check')).toEqual([]); @@ -265,77 +543,164 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Recovery host' }); const { config } = await import('../../src/lib/server/config.js'); - hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, dockerRunning: true, - directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [config.defaultRuntimeImage], - libraries: [{ id: 'lib', state: 'ready' }] }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, { + protocolVersion: 1, + dockerRunning: true, + directoriesWritable: true, + storageAvailable: 100_000_000_000, + runtimeImages: [config.defaultRuntimeImage], + libraries: [{ id: 'lib', state: 'ready' }] + }); const generation = action === 'create' ? 1 : 2; - a.db.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)") + a.db + .prepare( + "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,?,'lib','image',1,?)" + ) .run(host.id, generation, old); - a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')") + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('recovering','s',?,'awaiting-health','Recovery',?,'test')" + ) .run(action === 'create' ? 'create' : 'rebuild', old); const sha256 = putFileBlob(Buffer.from('original launch files'), a.db); - const launch = hosts.queueHostOperation(host.id, action === 'rebuild' ? 'instance.rebuild' : 'instance.create', { - name: 's', files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }] - }, { instanceId: 'instance', generation: action === 'rebuild' ? 1 : generation, - idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering' }); + const launch = hosts.queueHostOperation( + host.id, + action === 'rebuild' ? 'instance.rebuild' : 'instance.create', + { + name: 's', + files: [{ path: 'configs/server.cfg', sha256, contentSha256: sha256 }] + }, + { + instanceId: 'instance', + generation: action === 'rebuild' ? 1 : generation, + idempotencyKey: action === 'migration' ? 'recovering:migration-create:0' : 'recovering' + } + ); hosts.reportHostOperation(host.id, launch.id, { state: 'completed' }); - const restart = hosts.queueHostOperation(host.id, 'instance.restart', { unhealthyRecovery: true }, { instanceId: 'instance', generation }); - a.db.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')") + const restart = hosts.queueHostOperation( + host.id, + 'instance.restart', + { unhealthyRecovery: true }, + { instanceId: 'instance', generation } + ); + a.db + .prepare( + "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,recovery_step) VALUES('recovering','instance',?,?,'restart')" + ) .run(generation, restart.id); a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old); - a.saveFleetSettings({ ...a.fleetSettings(), actionKeepDays: 1, payloadKeepDays: 1, historyKeepCount: 0 }); + a.saveFleetSettings({ + ...a.fleetSettings(), + actionKeepDays: 1, + payloadKeepDays: 1, + historyKeepCount: 0 + }); return { ...a, hosts, host, generation, launch, restart, sha256 }; } -it.each(['create', 'rebuild', 'migration'] as const)('keeps original %s launch files and ownership through recovery and releases them after success', async action => { - const a = await recoveryLaunch(action); - expect(a.pruneHistory('operation-payload')).toBe(0); - expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([ - { path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') } - ]); - a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' }); - const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: a.generation + 1 }, - { instanceId: 'instance', generation: a.generation }); - a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' }); - a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'") - .run(rebuild.id, old); - a.db.prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'").run(); - a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old); - a.pruneHistory('operation-payload'); - a.pruneHistory('operation'); - expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id)) - .toEqual({ history_expired_at: null }); - expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([ - { path: 'configs/server.cfg', sha256: a.sha256, contentBase64: Buffer.from('original launch files').toString('base64') } - ]); - a.db.prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'").run(old); - a.pruneHistory('operation-payload'); - expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([{ path: 'configs/server.cfg', sha256: a.sha256 }]); - a.pruneHistory('operation'); - expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id)) - .toEqual({ history_expired_at: expect.any(String) }); -}); +it.each(['create', 'rebuild', 'migration'] as const)( + 'keeps original %s launch files and ownership through recovery and releases them after success', + async (action) => { + const a = await recoveryLaunch(action); + expect(a.pruneHistory('operation-payload')).toBe(0); + expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([ + { + path: 'configs/server.cfg', + sha256: a.sha256, + contentBase64: Buffer.from('original launch files').toString('base64') + } + ]); + a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' }); + const rebuild = a.hosts.queueHostOperation( + a.host.id, + 'instance.rebuild', + { unhealthyRecovery: true, nextGeneration: a.generation + 1 }, + { instanceId: 'instance', generation: a.generation } + ); + a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' }); + a.db + .prepare( + "UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'" + ) + .run(rebuild.id, old); + a.db + .prepare("UPDATE native_instances SET generation=generation+1 WHERE id='instance'") + .run(); + a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old); + a.pruneHistory('operation-payload'); + a.pruneHistory('operation'); + expect( + a.db + .prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?') + .get(a.launch.id) + ).toEqual({ history_expired_at: null }); + expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([ + { + path: 'configs/server.cfg', + sha256: a.sha256, + contentBase64: Buffer.from('original launch files').toString('base64') + } + ]); + a.db + .prepare( + "UPDATE server_deployments SET state='done',finished_at=? WHERE id='recovering'" + ) + .run(old); + a.pruneHistory('operation-payload'); + expect(a.hosts.getHostOperation(a.launch.id)?.payload.files).toEqual([ + { path: 'configs/server.cfg', sha256: a.sha256 } + ]); + a.pruneHistory('operation'); + expect( + a.db + .prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?') + .get(a.launch.id) + ).toEqual({ history_expired_at: expect.any(String) }); + } +); -it.each(['cancelled', 'failed'])('keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup', async state => { - const a = await recoveryLaunch('create'); - a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' }); - const rebuild = a.hosts.queueHostOperation(a.host.id, 'instance.rebuild', { unhealthyRecovery: true, nextGeneration: 2 }, - { instanceId: 'instance', generation: 1 }); - a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' }); - a.db.prepare("UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'") - .run(rebuild.id, old); - a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run(); - a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old); - a.db.prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'").run(state, old); - const { canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); - expect(canCancelCreation('recovering')).toBe(true); - a.pruneHistory('operation'); - expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id)) - .toEqual({ history_expired_at: null }); - a.db.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')") - .run(old, old); - expect(canCancelCreation('recovering')).toBe(false); - a.pruneHistory('operation'); - expect(a.db.prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?').get(a.launch.id)) - .toEqual({ history_expired_at: expect.any(String) }); -}); +it.each(['cancelled', 'failed'])( + 'keeps original ownership for a %s recovered creation until a later lifecycle operation supersedes cleanup', + async (state) => { + const a = await recoveryLaunch('create'); + a.hosts.reportHostOperation(a.host.id, a.restart.id, { state: 'completed' }); + const rebuild = a.hosts.queueHostOperation( + a.host.id, + 'instance.rebuild', + { unhealthyRecovery: true, nextGeneration: 2 }, + { instanceId: 'instance', generation: 1 } + ); + a.hosts.reportHostOperation(a.host.id, rebuild.id, { state: 'completed' }); + a.db + .prepare( + "UPDATE native_provisioning SET operation_id=?,recovery_step='rebuild',applied_at=? WHERE deployment_id='recovering'" + ) + .run(rebuild.id, old); + a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run(); + a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(old); + a.db + .prepare("UPDATE server_deployments SET state=?,finished_at=? WHERE id='recovering'") + .run(state, old); + const { canCancelCreation } = + await import('../../src/lib/server/provisioning/cancel-creation.js'); + expect(canCancelCreation('recovering')).toBe(true); + a.pruneHistory('operation'); + expect( + a.db + .prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?') + .get(a.launch.id) + ).toEqual({ history_expired_at: null }); + a.db + .prepare( + "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('replacement','s','restart','done','Done',?,?,'test')" + ) + .run(old, old); + expect(canCancelCreation('recovering')).toBe(false); + a.pruneHistory('operation'); + expect( + a.db + .prepare('SELECT history_expired_at FROM ec_host_operations WHERE id=?') + .get(a.launch.id) + ).toEqual({ history_expired_at: expect.any(String) }); + } +); diff --git a/tests/unit/rollback-staging.test.ts b/tests/unit/rollback-staging.test.ts index 1176fc1..65c15bd 100644 --- a/tests/unit/rollback-staging.test.ts +++ b/tests/unit/rollback-staging.test.ts @@ -14,37 +14,87 @@ beforeEach(async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); async function promotion(kind: 'script' | 'plugin' | 'baseline', staged: boolean) { const { db } = await import('../../src/lib/server/db/index.js'); - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, uploadNewVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const before = Buffer.from(kind === 'baseline' ? 'sv_voice 0\n' : 'original'); const after = Buffer.from(kind === 'baseline' ? 'sv_voice 1\n' : 'promoted'); - const relPath = kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc'; - const { script, version: prior } = createScript({ name: 'managed', kind, targetRelPath: relPath, scopeType: 'game', scopeValue: 't5', content: before }); + const relPath = + kind === 'baseline' + ? 'dedicated_sp.cfg' + : kind === 'plugin' + ? 'managed.dll' + : 'managed.gsc'; + const { script, version: prior } = createScript({ + name: 'managed', + kind, + targetRelPath: relPath, + scopeType: 'game', + scopeValue: 't5', + content: before + }); const version = uploadNewVersion(script.id, after); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('promote', 'promote', ?, 'ui')").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('promote', ?, 1, 'pending')").run(server); - db().prepare("INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES ('promote', ?, ?, ?, ?)") + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(version.id, script.id); + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('promote', 'promote', ?, 'ui')" + ) + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('promote', ?, 1, 'pending')" + ) + .run(server); + db() + .prepare( + "INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES ('promote', ?, ?, ?, ?)" + ) .run(script.id, version.id, prior.id, kind); - const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) - VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)`).run(server, relPath, sha256Buffer(before), sha256Buffer(after), putFileBlob(before, db()), putFileBlob(after, db())); + const file = db() + .prepare( + `INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before_hash, content_after_hash) + VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)` + ) + .run( + server, + relPath, + sha256Buffer(before), + sha256Buffer(after), + putFileBlob(before, db()), + putFileBlob(after, db()) + ); const path = join(destOfKind(server, kind), relPath); await mkdir(dirname(path), { recursive: true }); await writeFile(path, staged ? before : after); - if (staged) queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind, relPath, content: after, - scriptVersionId: version.id, inventorySource: 'promote:promote', deploymentFileId: Number(file.lastInsertRowid) }); + if (staged) + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'promote', + server, + kind, + relPath, + content: after, + scriptVersionId: version.id, + inventorySource: 'promote:promote', + deploymentFileId: Number(file.lastInsertRowid) + }); return { path, before, after, script, prior }; } async function applyCampaign(id: string) { const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); - const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js'); + const { applyStagedFilesForOperation } = + await import('../../src/lib/server/fleet/staged-files.js'); const operation = requestRestart(server, { kind: 'campaign', id }); applyStagedFilesForOperation(operation.id, server); return operation; @@ -55,45 +105,86 @@ describe('deferred promotion rollback', () => { const { path, before } = await promotion('script', true); const { db } = await import('../../src/lib/server/db/index.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); - const { applyStagedFilesForOperation, isFileApplied } = await import('../../src/lib/server/fleet/staged-files.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const { applyStagedFilesForOperation, isFileApplied } = + await import('../../src/lib/server/fleet/staged-files.js'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); const operation = requestRestart(server, { kind: 'campaign', id: 'promote' }); const rollback = await rollbackPromoteDeployment('promote'); expect(rollback.affectedServers).toEqual([]); expect(await readFile(path)).toEqual(before); expect(isFileApplied('campaign', 'promote', server, 'script', 'managed.gsc')).toBe(false); - expect(db().prepare("SELECT restart_state FROM deployment_servers WHERE deployment_id = 'promote'").get()).toEqual({ restart_state: 'skipped' }); - expect(db().prepare('SELECT * FROM deployment_servers WHERE deployment_id = ?').all(rollback.deploymentId)).toEqual([]); + expect( + db() + .prepare( + "SELECT restart_state FROM deployment_servers WHERE deployment_id = 'promote'" + ) + .get() + ).toEqual({ restart_state: 'skipped' }); + expect( + db() + .prepare('SELECT * FROM deployment_servers WHERE deployment_id = ?') + .all(rollback.deploymentId) + ).toEqual([]); applyStagedFilesForOperation(operation.id, server); expect(await readFile(path)).toEqual(before); }); - it.each(['script', 'plugin', 'baseline'] as const)('queues restoration of applied %s bytes until rollback activation', async (kind) => { - const { path, before, after, prior } = await promotion(kind, true); - await applyCampaign('promote'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const result = await rollbackPromoteDeployment('promote'); - expect(result.affectedServers).toEqual([server]); - expect(await readFile(path)).toEqual(after); - await applyCampaign(result.deploymentId); - expect(await readFile(path)).toEqual(before); - expect(getServerScript(server, kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc')?.script_version_id).toBe(prior.id); - }); + it.each(['script', 'plugin', 'baseline'] as const)( + 'queues restoration of applied %s bytes until rollback activation', + async (kind) => { + const { path, before, after, prior } = await promotion(kind, true); + await applyCampaign('promote'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); + const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); + const result = await rollbackPromoteDeployment('promote'); + expect(result.affectedServers).toEqual([server]); + expect(await readFile(path)).toEqual(after); + await applyCampaign(result.deploymentId); + expect(await readFile(path)).toEqual(before); + expect( + getServerScript( + server, + kind === 'baseline' + ? 'dedicated_sp.cfg' + : kind === 'plugin' + ? 'managed.dll' + : 'managed.gsc' + )?.script_version_id + ).toBe(prior.id); + } + ); it('restores only applied files when a promotion was partly installed', async () => { const { path, before } = await promotion('script', true); await applyCampaign('promote'); const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); - const pending = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after_hash) - VALUES ('promote', ?, 'added', 'never.gsc', ?)`).run(server, putFileBlob(Buffer.from('queued'), db())); - queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind: 'script', relPath: 'never.gsc', - content: Buffer.from('queued'), inventorySource: 'promote:promote', deploymentFileId: Number(pending.lastInsertRowid) }); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); + const pending = db() + .prepare( + `INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after_hash) + VALUES ('promote', ?, 'added', 'never.gsc', ?)` + ) + .run(server, putFileBlob(Buffer.from('queued'), db())); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'promote', + server, + kind: 'script', + relPath: 'never.gsc', + content: Buffer.from('queued'), + inventorySource: 'promote:promote', + deploymentFileId: Number(pending.lastInsertRowid) + }); const result = await rollbackPromoteDeployment('promote'); - expect(db().prepare('SELECT rel_path FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId)) - .toEqual([{ rel_path: 'managed.gsc' }]); + expect( + db() + .prepare('SELECT rel_path FROM deployment_files WHERE deployment_id = ?') + .all(result.deploymentId) + ).toEqual([{ rel_path: 'managed.gsc' }]); await applyCampaign(result.deploymentId); expect(await readFile(path)).toEqual(before); expect(existsSync(join(dirname(path), 'never.gsc'))).toBe(false); @@ -104,11 +195,27 @@ describe('deferred promotion rollback', () => { const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('other', 'drift', ?, 'ui')").run(new Date().toISOString()); - db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('other', ?, 1, 'pending')").run(server); - queueFileChange({ sourceKind: 'campaign', sourceId: 'other', server, kind: 'script', relPath: 'other.gsc', - content: Buffer.from('other campaign'), inventorySource: 'drift:other' }); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('other', 'drift', ?, 'ui')" + ) + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('other', ?, 1, 'pending')" + ) + .run(server); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'other', + server, + kind: 'script', + relPath: 'other.gsc', + content: Buffer.from('other campaign'), + inventorySource: 'drift:other' + }); const shared = requestRestart(server, { kind: 'campaign', id: 'promote' }); expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).not.toBe(shared.id); await rollbackPromoteDeployment('promote'); @@ -119,7 +226,8 @@ describe('deferred promotion rollback', () => { it('stages rollback for legacy promotions with no deferred file records', async () => { const { path, before, after } = await promotion('plugin', false); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); const result = await rollbackPromoteDeployment('promote'); expect(result.affectedServers).toEqual([server]); expect(await readFile(path)).toEqual(after); diff --git a/tests/unit/runtime-config.test.ts b/tests/unit/runtime-config.test.ts index 2a748c1..26f6a04 100644 --- a/tests/unit/runtime-config.test.ts +++ b/tests/unit/runtime-config.test.ts @@ -3,19 +3,28 @@ import { applyRuntimePassword } from '../../src/lib/server/fleet/runtime-config. import { newFixture, type Fixture } from '../helpers/fixture.js'; describe('Plutainer password rendering', () => { - it.each(['set rcon_password', 'seta rcon_password', 'RCON_PASSWORD'])('rewrites %s while preserving comments and other settings', prefix => { - const source = `// set rcon_password "comment"\n${prefix} 'old' // keep\nset sv_voice "1"\n`; - const rendered = applyRuntimePassword(source, 'abc&/\\$xyz'); - expect(rendered).toBe(`// set rcon_password "comment"\n${prefix} "abc&/\\$xyz" // keep\nset sv_voice "1"\n`); - expect(applyRuntimePassword(rendered, 'abc&/\\$xyz')).toBe(rendered); - }); + it.each(['set rcon_password', 'seta rcon_password', 'RCON_PASSWORD'])( + 'rewrites %s while preserving comments and other settings', + (prefix) => { + const source = `// set rcon_password "comment"\n${prefix} 'old' // keep\nset sv_voice "1"\n`; + const rendered = applyRuntimePassword(source, 'abc&/\\$xyz'); + expect(rendered).toBe( + `// set rcon_password "comment"\n${prefix} "abc&/\\$xyz" // keep\nset sv_voice "1"\n` + ); + expect(applyRuntimePassword(rendered, 'abc&/\\$xyz')).toBe(rendered); + } + ); it('matches startup handling of missing, duplicate, quoted and unquoted assignments', () => { expect(applyRuntimePassword('set sv_voice "1"\n', 'secret')).toBe( - 'set sv_voice "1"\n\n// Added by Plutainer from PLUTAINER_RCON_PASSWORD.\nset rcon_password "secret"\n'); - expect(applyRuntimePassword('set rcon_password old\nrcon_password "later"', 'sec"ret')) - .toBe('set rcon_password "secret"\nrcon_password "secret"'); - expect(applyRuntimePassword('set rcon_password old\r\n', '')).toBe('set rcon_password old\r\n'); + 'set sv_voice "1"\n\n// Added by Plutainer from PLUTAINER_RCON_PASSWORD.\nset rcon_password "secret"\n' + ); + expect( + applyRuntimePassword('set rcon_password old\nrcon_password "later"', 'sec"ret') + ).toBe('set rcon_password "secret"\nrcon_password "secret"'); + expect(applyRuntimePassword('set rcon_password old\r\n', '')).toBe( + 'set rcon_password old\r\n' + ); }); it('normalizes CRLF only when a password write is needed, matching Plutainer', () => { @@ -29,104 +38,209 @@ describe('Plutainer password rendering', () => { }); let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); async function setup(game: 't5' | 't6') { const { db } = await import('../../src/lib/server/db/index.js'); const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Runtime config' }); - hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1, capabilities: ['files.manifest'] }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, { + protocolVersion: 1, + capabilities: ['files.manifest'] + }); const server = `${game}_runtime_1`; db().prepare('INSERT INTO servers(name,game) VALUES(?,?)').run(server, game); - db().prepare(`INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) - VALUES('instance',?,?,1,'library','image',28960,?)`).run(server, host.id, new Date().toISOString()); - const launch = (id: string, kind: string, generation: number, password: string, serialized = false, state = 'completed') => { - db().prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) - VALUES(?,?,'instance',?,?,?,?,?,?)`).run(id, host.id, generation, kind, - JSON.stringify(serialized ? {} : { rconPassword: password }), state, new Date().toISOString(), new Date().toISOString()); - if (serialized) db().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)') - .run(id, JSON.stringify({ rconPassword: password })); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) + VALUES('instance',?,?,1,'library','image',28960,?)` + ) + .run(server, host.id, new Date().toISOString()); + const launch = ( + id: string, + kind: string, + generation: number, + password: string, + serialized = false, + state = 'completed' + ) => { + db() + .prepare( + `INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) + VALUES(?,?,'instance',?,?,?,?,?,?)` + ) + .run( + id, + host.id, + generation, + kind, + JSON.stringify(serialized ? {} : { rconPassword: password }), + state, + new Date().toISOString(), + new Date().toISOString() + ); + if (serialized) + db() + .prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)') + .run(id, JSON.stringify({ rconPassword: password })); }; return { db: db(), host, server, launch }; } it.each([ - ['t5', 'a'.repeat(64)], ['t6', 'a'.repeat(64)], ['t5', 'shared-password'], ['t6', 'shared-password'] -] as const)('uses the assigned password for %s launches, drift and beta writes (%s)', async (game, password) => { - const a = await setup(game); - a.launch('create', 'instance.create', 1, password, true); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const baseline = createScript({ name: 'Baseline', kind: 'baseline', scopeType: 'game', scopeValue: game, - content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') }); - a.db.prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(baseline.version.id, baseline.script.id); - const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js'); - const { captureRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { configFileForGame } = await import('../../src/lib/server/config.js'); - const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); - const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const target = configFileForGame(game); - const promoted = getPromotedServerConfig(game, null)!; - const effective = getPromotedServerConfig(game, null, null, password)!; - expect(effective.length - promoted.length).toBe(password.length - 8); - const recipe = captureRecipe(game, game === 't5' ? 'theater' : 'transit', password); - const recipeConfig = recipe.files.find(file => file.path === target)!; - expect(Buffer.from(recipeConfig.content, 'base64').toString()).toContain(`set rcon_password "${password}"`); - expect(recipeConfig.sha256).toBe(sha256Buffer(Buffer.from(recipeConfig.content, 'base64'))); - recordScript(a.server, target, null, sha256Buffer(promoted), promoted.length, 'native-create'); - const backend = await import('../../src/lib/server/backends/native.js'); - const { prepareNativeLaunch } = await import('../../src/lib/server/provisioning/launch-preparation.js'); - const prepared = await prepareNativeLaunch({ game, mapSlug: recipe.mapSlug, instance: backend.nativeInstance(a.server)!, - serverKey: 'test-key', rconPassword: password, rebuild: false }); - const actualConfig = prepared.recipe.files.find(file => file.path === target)!; - expect(actualConfig.sha256).toBe(recipeConfig.sha256); - const actualBytes = prepared.blobs!.find(blob => blob.sha256 === actualConfig.sha256)!.content; - expect(Buffer.from(actualBytes)).toEqual(Buffer.from(recipeConfig.content, 'base64')); - expect(applyRuntimePassword(Buffer.from(actualBytes).toString(), password)).toBe(Buffer.from(actualBytes).toString()); - const payload = JSON.parse(Buffer.from(prepared.payloadJson).toString()); - expect(payload.rconPassword).toBe(password); - expect(payload.files).toContainEqual(expect.objectContaining({ path: `configs/${target}`, sha256: actualConfig.sha256 })); - expect(prepared.recipeSummary).not.toContain(password); - let observed = effective; - vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { - expect(kind).toBe('files.manifest'); - return { files: [{ path: `configs/${target}`, size: observed.length, sha256: sha256Buffer(observed) }] } as never; - }); - const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); - expect(await scanServerDrift(a.server)).toEqual([]); - expect(getServerScript(a.server, target)?.sha256).toBe(sha256Buffer(effective)); - observed = Buffer.from(effective.toString().replace(password, 'b'.repeat(password.length))); - expect(await scanServerDrift(a.server)).toEqual([expect.objectContaining({ relPath: target, driftKind: 'replace' })]); + ['t5', 'a'.repeat(64)], + ['t6', 'a'.repeat(64)], + ['t5', 'shared-password'], + ['t6', 'shared-password'] +] as const)( + 'uses the assigned password for %s launches, drift and beta writes (%s)', + async (game, password) => { + const a = await setup(game); + a.launch('create', 'instance.create', 1, password, true); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const baseline = createScript({ + name: 'Baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: game, + content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') + }); + a.db + .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') + .run(baseline.version.id, baseline.script.id); + const { getPromotedServerConfig } = + await import('../../src/lib/server/fleet/server-config.js'); + const { captureRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { configFileForGame } = await import('../../src/lib/server/config.js'); + const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); + const { recordScript, getServerScript } = + await import('../../src/lib/server/fleet/inventory.js'); + const target = configFileForGame(game); + const promoted = getPromotedServerConfig(game, null)!; + const effective = getPromotedServerConfig(game, null, null, password)!; + expect(effective.length - promoted.length).toBe(password.length - 8); + const recipe = captureRecipe(game, game === 't5' ? 'theater' : 'transit', password); + const recipeConfig = recipe.files.find((file) => file.path === target)!; + expect(Buffer.from(recipeConfig.content, 'base64').toString()).toContain( + `set rcon_password "${password}"` + ); + expect(recipeConfig.sha256).toBe(sha256Buffer(Buffer.from(recipeConfig.content, 'base64'))); + recordScript( + a.server, + target, + null, + sha256Buffer(promoted), + promoted.length, + 'native-create' + ); + const backend = await import('../../src/lib/server/backends/native.js'); + const { prepareNativeLaunch } = + await import('../../src/lib/server/provisioning/launch-preparation.js'); + const prepared = await prepareNativeLaunch({ + game, + mapSlug: recipe.mapSlug, + instance: backend.nativeInstance(a.server)!, + serverKey: 'test-key', + rconPassword: password, + rebuild: false + }); + const actualConfig = prepared.recipe.files.find((file) => file.path === target)!; + expect(actualConfig.sha256).toBe(recipeConfig.sha256); + const actualBytes = prepared.blobs!.find( + (blob) => blob.sha256 === actualConfig.sha256 + )!.content; + expect(Buffer.from(actualBytes)).toEqual(Buffer.from(recipeConfig.content, 'base64')); + expect(applyRuntimePassword(Buffer.from(actualBytes).toString(), password)).toBe( + Buffer.from(actualBytes).toString() + ); + const payload = JSON.parse(Buffer.from(prepared.payloadJson).toString()); + expect(payload.rconPassword).toBe(password); + expect(payload.files).toContainEqual( + expect.objectContaining({ path: `configs/${target}`, sha256: actualConfig.sha256 }) + ); + expect(prepared.recipeSummary).not.toContain(password); + let observed = effective; + vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { + expect(kind).toBe('files.manifest'); + return { + files: [ + { + path: `configs/${target}`, + size: observed.length, + sha256: sha256Buffer(observed) + } + ] + } as never; + }); + const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); + expect(await scanServerDrift(a.server)).toEqual([]); + expect(getServerScript(a.server, target)?.sha256).toBe(sha256Buffer(effective)); + observed = Buffer.from(effective.toString().replace(password, 'b'.repeat(password.length))); + expect(await scanServerDrift(a.server)).toEqual([ + expect.objectContaining({ relPath: target, driftKind: 'replace' }) + ]); - const overlay = createScript({ name: 'Overlay', kind: 'config', scopeType: 'game', scopeValue: game, - content: Buffer.from('set sv_voice "0"\nset rcon_password "overlay-password"\n') }); - const { openBeta, previewBetaPromotion } = await import('../../src/lib/server/fleet/beta.js'); - const { betaId } = await openBeta({ scripts: [{ scriptId: overlay.script.id, scriptVersionId: overlay.version.id }], - servers: [a.server], waitForHealth: false, waitForMatch: false }); - const staged = a.db.prepare(`SELECT content_hash FROM staged_file_changes WHERE source_kind='beta' AND source_id=? AND server=?`) - .get(String(betaId), a.server) as { content_hash: string }; - const { getFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); - const stagedContent = getFileBlob(staged.content_hash).toString(); - expect(stagedContent).toContain(`set rcon_password "${password}"`); - expect(stagedContent).toContain('set sv_voice "0"'); - expect(previewBetaPromotion(betaId).configWrites).toContainEqual(expect.objectContaining({ server: a.server, sha256: staged.content_hash })); -}); + const overlay = createScript({ + name: 'Overlay', + kind: 'config', + scopeType: 'game', + scopeValue: game, + content: Buffer.from('set sv_voice "0"\nset rcon_password "overlay-password"\n') + }); + const { openBeta, previewBetaPromotion } = + await import('../../src/lib/server/fleet/beta.js'); + const { betaId } = await openBeta({ + scripts: [{ scriptId: overlay.script.id, scriptVersionId: overlay.version.id }], + servers: [a.server], + waitForHealth: false, + waitForMatch: false + }); + const staged = a.db + .prepare( + `SELECT content_hash FROM staged_file_changes WHERE source_kind='beta' AND source_id=? AND server=?` + ) + .get(String(betaId), a.server) as { content_hash: string }; + const { getFileBlob } = await import('../../src/lib/server/db/file-blobs.js'); + const stagedContent = getFileBlob(staged.content_hash).toString(); + expect(stagedContent).toContain(`set rcon_password "${password}"`); + expect(stagedContent).toContain('set sv_voice "0"'); + expect(previewBetaPromotion(betaId).configWrites).toContainEqual( + expect.objectContaining({ server: a.server, sha256: staged.content_hash }) + ); + } +); it('retains live launch credentials through history cleanup, allowing file bodies and retired credentials to expire', async () => { const a = await setup('t5'); a.launch('create', 'instance.create', 1, 'original', true); a.launch('rebuild', 'instance.rebuild', 1, 'current', true); a.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run(); - a.db.prepare('UPDATE ec_host_operations SET updated_at=?').run(new Date(Date.now() - 90 * 86400000).toISOString()); + a.db + .prepare('UPDATE ec_host_operations SET updated_at=?') + .run(new Date(Date.now() - 90 * 86400000).toISOString()); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { pruneHistory } = await import('../../src/lib/server/retention.js'); const { runtimePasswordForServer } = await import('../../src/lib/server/backends/native.js'); - saveFleetSettings({ ...fleetSettings(), actionKeepDays: 1, payloadKeepDays: 1, historyKeepCount: 0 }); + saveFleetSettings({ + ...fleetSettings(), + actionKeepDays: 1, + payloadKeepDays: 1, + historyKeepCount: 0 + }); expect(pruneHistory('operation-payload')).toBe(2); expect(runtimePasswordForServer(a.server)).toBe('current'); expect(pruneHistory('operation')).toBe(1); expect(runtimePasswordForServer(a.server)).toBe('current'); - a.db.prepare("UPDATE native_instances SET retired_at=? WHERE id='instance'").run(new Date().toISOString()); + a.db + .prepare("UPDATE native_instances SET retired_at=? WHERE id='instance'") + .run(new Date().toISOString()); expect(pruneHistory('operation')).toBe(1); }); diff --git a/tests/unit/runtime-image-policy.test.ts b/tests/unit/runtime-image-policy.test.ts index 1b33daf..f9e80f9 100644 --- a/tests/unit/runtime-image-policy.test.ts +++ b/tests/unit/runtime-image-policy.test.ts @@ -7,14 +7,21 @@ vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () let fixture: Fixture; let prepared: Awaited>; const custom = 'registry.example/untrusted:latest'; -const launch = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' }; +const launch = { + game: 't5' as const, + gameMode: 'zombie' as const, + mapSlug: 'theater', + plutoKey: 'saved-key' +}; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); prepared = await prepareNativeHost(); }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); async function cache(images: string[]) { const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -23,19 +30,31 @@ async function cache(images: string[]) { } describe('runtime image launch policy', () => { - it.each([false, true])('rejects an uncached custom image at POST /api/servers (automatic: %s)', async automatic => { - const { POST } = await import('../../src/routes/api/servers/+server.js'); - const response = await POST({ request: new Request('https://controller.test/api/servers', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ ...launch, image: custom, ...(automatic ? {} : { hostId: prepared.host.id }) }) - }) }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ error: expect.stringContaining('prepared by an admin') }); - const { db } = await import('../../src/lib/server/db/index.js'); - for (const table of ['native_creation_intents', 'server_deployments', 'server_keys']) { - expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); + it.each([false, true])( + 'rejects an uncached custom image at POST /api/servers (automatic: %s)', + async (automatic) => { + const { POST } = await import('../../src/routes/api/servers/+server.js'); + const response = await POST({ + request: new Request('https://controller.test/api/servers', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + ...launch, + image: custom, + ...(automatic ? {} : { hostId: prepared.host.id }) + }) + }) + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('prepared by an admin') + }); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const table of ['native_creation_intents', 'server_deployments', 'server_keys']) { + expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); + } } - }); + ); it('allows the configured default before it is cached', async () => { await cache([]); @@ -47,45 +66,74 @@ describe('runtime image launch policy', () => { await cache([custom]); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Unprepared host' }); - const { nativeLaunchSelection, nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); - expect(nativeLaunchSelection({ hostId: prepared.host.id, image: custom }, 't5').image).toBe(custom); + const { nativeLaunchSelection, nativeSelection } = + await import('../../src/lib/server/provisioning/native.js'); + expect(nativeLaunchSelection({ hostId: prepared.host.id, image: custom }, 't5').image).toBe( + custom + ); expect(nativeSelection({ image: custom }, 't5').hostId).toBe(prepared.host.id); - expect(() => nativeLaunchSelection({ hostId: other.id, image: custom }, 't5')).toThrow(/prepared by an admin/); + expect(() => nativeLaunchSelection({ hostId: other.id, image: custom }, 't5')).toThrow( + /prepared by an admin/ + ); hosts.updateHost(prepared.host.id, { mode: 'disabled' }); - expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow( + /prepared by an admin/ + ); }); it('requires exact image references and does not treat a queued admin pull as cached', async () => { const hosts = await import('../../src/lib/server/hosts/index.js'); hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom }); - const { nativeLaunchSelection } = await import('../../src/lib/server/provisioning/native.js'); - expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + const { nativeLaunchSelection } = + await import('../../src/lib/server/provisioning/native.js'); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow( + /prepared by an admin/ + ); await cache([custom + '-other']); - expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow( + /prepared by an admin/ + ); }); it('rechecks cached images when submitting a saved single-server preview', async () => { await cache([custom]); - const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); + const { previewNativeCreation, queueNativeCreation } = + await import('../../src/lib/server/provisioning/native.js'); const preview = previewNativeCreation({ ...launch, image: custom }); await cache([]); expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({ - ok: false, message: expect.stringContaining('prepared by an admin') + ok: false, + message: expect.stringContaining('prepared by an admin') }); }); it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const blueprint = saveBlueprint({ name: 'Image policy', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] }); - expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(/prepared by an admin/); + const { previewCollection, queueCollection } = + await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ + name: 'Image policy', + entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] + }); + expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow( + /prepared by an admin/ + ); await cache([custom]); const preview = previewCollection(blueprint.id, 'fill', { image: custom }); - const keys = Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])); + const keys = Object.fromEntries( + preview.servers.map((server) => [server.name!, 'key-' + server.name]) + ); await cache([]); - expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/prepared by an admin/); + expect(() => queueCollection({ previewId: preview.id, keys })).toThrow( + /prepared by an admin/ + ); const { db } = await import('../../src/lib/server/db/index.js'); - for (const table of ['server_launch_collections', 'native_creation_intents', 'server_deployments', 'server_keys']) { + for (const table of [ + 'server_launch_collections', + 'native_creation_intents', + 'server_deployments', + 'server_keys' + ]) { expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); } await cache([custom]); diff --git a/tests/unit/script-flow.test.ts b/tests/unit/script-flow.test.ts index 043d390..b4efa5c 100644 --- a/tests/unit/script-flow.test.ts +++ b/tests/unit/script-flow.test.ts @@ -25,7 +25,8 @@ async function seedFleet(): Promise { describe('scope options', () => { it('lists distinct games and maps by game from the fleet', async () => { await seedFleet(); - const { listDistinctGames, listMapsByGame } = await import('../../src/lib/server/fleet/scripts.js'); + const { listDistinctGames, listMapsByGame } = + await import('../../src/lib/server/fleet/scripts.js'); // Both fixture servers are t5, so games is a single-entry list. expect(listDistinctGames()).toEqual(['t5']); @@ -36,7 +37,11 @@ describe('scope options', () => { }); describe('editing promoted script scopes', () => { - async function saveScope(scriptId: number, scopeType: 'game' | 'map' | null, scopeValue: string | null) { + async function saveScope( + scriptId: number, + scopeType: 'game' | 'map' | null, + scopeValue: string | null + ) { const { POST } = await import('../../src/routes/api/scripts/[id]/scope/+server.js'); return POST({ params: { id: String(scriptId) }, @@ -48,85 +53,127 @@ describe('editing promoted script scopes', () => { }); } - it.each(['game', 'map'] as const)('restores a cleared %s scope and can promote again', async (scopeType) => { - await seedFleet(); - const { createScript, getScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const scopeValue = scopeType === 'game' ? 't5' : 'zombie_theater'; - const { script, version } = createScript({ - name: 'restore scope', targetRelPath: 'restore.gsc', - scopeType, scopeValue, content: Buffer.from('main(){}\n') - }); - async function promote() { + it.each(['game', 'map'] as const)( + 'restores a cleared %s scope and can promote again', + async (scopeType) => { + await seedFleet(); + const { createScript, getScript } = + await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const scopeValue = scopeType === 'game' ? 't5' : 'zombie_theater'; + const { script, version } = createScript({ + name: 'restore scope', + targetRelPath: 'restore.gsc', + scopeType, + scopeValue, + content: Buffer.from('main(){}\n') + }); + async function promote() { + const { betaId } = await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: ['t5_zombie_theater_1'], + waitForHealth: false, + waitForMatch: false + }); + await activateFiles('beta', betaId); + db() + .prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?") + .run(betaId); + db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId); + const result = await promoteBeta(betaId); + await activateFiles('campaign', result.deploymentId); + return result; + } + await promote(); + + const cleared = await saveScope(script.id, null, null); + expect(cleared.status).toBe(200); + expect((await cleared.json()).detached).toContain('t5_zombie_theater_1'); + expect(getScript(script.id)).toMatchObject({ + scope_type: null, + current_version_id: version.id + }); + + const restored = await saveScope(script.id, scopeType, scopeValue); + expect(restored.status).toBe(200); + expect((await restored.json()).script).toMatchObject({ + scope_type: scopeType, + scope_value: scopeValue, + current_version_id: version.id + }); + // Saving scope must not deploy or reattach the detached inventory. + expect( + db() + .prepare('SELECT * FROM server_scripts WHERE script_version_id = ?') + .all(version.id) + ).toHaveLength(0); + const result = await promote(); + expect(result.landed.sort()).toEqual( + scopeType === 'game' + ? ['t5_zombie_cosmodrome_1', 't5_zombie_theater_1'] + : ['t5_zombie_theater_1'] + ); + + // A promoted script can also change scope without first clearing it. + const changed = await saveScope(script.id, 'map', 'zombie_cosmodrome'); + expect(changed.status).toBe(200); + expect(getScript(script.id)).toMatchObject({ + scope_type: 'map', + scope_value: 'zombie_cosmodrome', + current_version_id: version.id + }); + } + ); + + it.each(['running', 'ready-promote'])( + 'rejects scope edits while a beta is %s without detaching inventory', + async (state) => { + await seedFleet(); + const { createScript, getScript } = + await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { script, version } = createScript({ + name: 'active scope', + targetRelPath: 'active.gsc', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('main(){}\n') + }); const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false + servers: ['t5_zombie_theater_1'], + waitForHealth: false, + waitForMatch: false }); await activateFiles('beta', betaId); - db().prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(betaId); - db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId); - const result = await promoteBeta(betaId); - await activateFiles('campaign', result.deploymentId); - return result; + db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId); + const inventory = db().prepare('SELECT * FROM server_scripts').all(); + expect(inventory.length).toBeGreaterThan(0); + for (const scopeType of [null, 'map'] as const) { + const response = await saveScope( + script.id, + scopeType, + scopeType ? 'zombie_cosmodrome' : null + ); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ ok: false, kind: 'has-active-beta' }); + expect(getScript(script.id)).toMatchObject({ + scope_type: 'game', + scope_value: 't5' + }); + expect(db().prepare('SELECT * FROM server_scripts').all()).toEqual(inventory); + } } - await promote(); - - const cleared = await saveScope(script.id, null, null); - expect(cleared.status).toBe(200); - expect((await cleared.json()).detached).toContain('t5_zombie_theater_1'); - expect(getScript(script.id)).toMatchObject({ scope_type: null, current_version_id: version.id }); - - const restored = await saveScope(script.id, scopeType, scopeValue); - expect(restored.status).toBe(200); - expect((await restored.json()).script).toMatchObject({ - scope_type: scopeType, scope_value: scopeValue, current_version_id: version.id - }); - // Saving scope must not deploy or reattach the detached inventory. - expect(db().prepare('SELECT * FROM server_scripts WHERE script_version_id = ?').all(version.id)).toHaveLength(0); - const result = await promote(); - expect(result.landed.sort()).toEqual(scopeType === 'game' - ? ['t5_zombie_cosmodrome_1', 't5_zombie_theater_1'] : ['t5_zombie_theater_1']); - - // A promoted script can also change scope without first clearing it. - const changed = await saveScope(script.id, 'map', 'zombie_cosmodrome'); - expect(changed.status).toBe(200); - expect(getScript(script.id)).toMatchObject({ - scope_type: 'map', scope_value: 'zombie_cosmodrome', current_version_id: version.id - }); - }); - - it.each(['running', 'ready-promote'])('rejects scope edits while a beta is %s without detaching inventory', async (state) => { - await seedFleet(); - const { createScript, getScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { script, version } = createScript({ - name: 'active scope', targetRelPath: 'active.gsc', - scopeType: 'game', scopeValue: 't5', content: Buffer.from('main(){}\n') - }); - const { betaId } = await openBeta({ - scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false - }); - await activateFiles('beta', betaId); - db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId); - const inventory = db().prepare('SELECT * FROM server_scripts').all(); - expect(inventory.length).toBeGreaterThan(0); - for (const scopeType of [null, 'map'] as const) { - const response = await saveScope(script.id, scopeType, scopeType ? 'zombie_cosmodrome' : null); - expect(response.status).toBe(409); - expect(await response.json()).toMatchObject({ ok: false, kind: 'has-active-beta' }); - expect(getScript(script.id)).toMatchObject({ scope_type: 'game', scope_value: 't5' }); - expect(db().prepare('SELECT * FROM server_scripts').all()).toEqual(inventory); - } - }); + ); }); describe('promote creates a deployment', () => { it('records deployment_files for every landed server and marks beta servers done, others pending', async () => { await seedFleet(); - const { createScript, getScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, getScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); @@ -151,31 +198,40 @@ describe('promote creates a deployment', () => { await activateFiles('beta', betaId); // Force all beta_servers to 'ready' and beta to 'ready-promote'. - db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) + db() + .prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) .run(new Date().toISOString(), betaId); db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId); const result = await promoteBeta(betaId); expect(result.deploymentId).toMatch(/^dep-.*-promote-\d+$/); - expect(new Set(result.landed)).toEqual(new Set(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'])); + expect(new Set(result.landed)).toEqual( + new Set(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']) + ); expect(result.scriptIds).toEqual([script.id]); // Deployment row is action='promote'. The multi-script schema stores // scripts in deployment_scripts, not the legacy singular columns. - const dep = db().prepare(`SELECT * FROM deployments WHERE id = ?`).get(result.deploymentId) as any; + const dep = db() + .prepare(`SELECT * FROM deployments WHERE id = ?`) + .get(result.deploymentId) as Record; expect(dep.action).toBe('promote'); expect(dep.beta_id).toBe(betaId); expect(dep.script_id).toBeNull(); // new promotes populate deployment_scripts - const depScripts = db().prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ?`).all(result.deploymentId) as any[]; + const depScripts = db() + .prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ?`) + .all(result.deploymentId) as Record[]; expect(depScripts.length).toBe(1); expect(depScripts[0].script_id).toBe(script.id); expect(depScripts[0].script_version_id).toBe(version.id); expect(depScripts[0].prior_version_id).toBeNull(); // never promoted before // deployment_files has one row per landed server. - const files = db().prepare(`SELECT * FROM deployment_files WHERE deployment_id = ?`).all(result.deploymentId) as any[]; + const files = db() + .prepare(`SELECT * FROM deployment_files WHERE deployment_id = ?`) + .all(result.deploymentId) as Record[]; expect(files.length).toBe(2); for (const f of files) { expect(f.verb).toBe('added'); @@ -187,7 +243,7 @@ describe('promote creates a deployment', () => { // server row is 'pending' (sweep will drain-and-restart). const servers = db() .prepare(`SELECT * FROM deployment_servers WHERE deployment_id = ? ORDER BY server`) - .all(result.deploymentId) as any[]; + .all(result.deploymentId) as { server: string; restart_state: string }[]; const byName = Object.fromEntries(servers.map((s) => [s.server, s.restart_state])); expect(byName['t5_zombie_theater_1']).toBe('done'); expect(byName['t5_zombie_cosmodrome_1']).toBe('pending'); @@ -195,7 +251,10 @@ describe('promote creates a deployment', () => { // Script's current_version_id and beta state land. const after = getScript(script.id)!; expect(after.current_version_id).toBe(version.id); - const beta = db().prepare(`SELECT state FROM betas WHERE id = ?`).get(betaId) as any; + const beta = db().prepare(`SELECT state FROM betas WHERE id = ?`).get(betaId) as Record< + string, + unknown + >; expect(beta.state).toBe('promoted'); }); }); @@ -203,7 +262,8 @@ describe('promote creates a deployment', () => { describe('rollback beta on ready-promote marks version test-failed', () => { it('flips script_versions.state to test-failed when the version has no other promoting beta', async () => { await seedFleet(); - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); @@ -232,7 +292,9 @@ describe('rollback beta on ready-promote marks version test-failed', () => { await rollbackBeta(betaId); - const after = db().prepare(`SELECT state FROM script_versions WHERE id = ?`).get(v.id) as any; + const after = db() + .prepare(`SELECT state FROM script_versions WHERE id = ?`) + .get(v.id) as Record; expect(after.state).toBe('test-failed'); }); }); @@ -240,9 +302,11 @@ describe('rollback beta on ready-promote marks version test-failed', () => { describe('rollback promote deployment', () => { it('restores current_version_id + files and creates an action=rollback deployment with pending servers', async () => { await seedFleet(); - const { createScript, getScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { createScript, getScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script } = createScript({ @@ -261,7 +325,8 @@ describe('rollback promote deployment', () => { waitForMatch: false }); await activateFiles('beta', betaId); - db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) + db() + .prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) .run(new Date().toISOString(), betaId); db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId); const promote = await promoteBeta(betaId); @@ -272,42 +337,43 @@ describe('rollback promote deployment', () => { const result = await rollbackPromoteDeployment(promote.deploymentId); expect(result.deploymentId).toMatch(/^dep-.*-rollback-/); - expect(new Set(result.affectedServers)).toEqual(new Set(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'])); + expect(new Set(result.affectedServers)).toEqual( + new Set(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']) + ); expect(result.scriptIds).toEqual([script.id]); // Script's current_version_id reverted (was null before the promote). expect(getScript(script.id)!.current_version_id).toBeNull(); // Original promote is now marked rolled_back_by. - const promoteRow = db().prepare(`SELECT rolled_back_by FROM deployments WHERE id = ?`).get(promote.deploymentId) as any; + const promoteRow = db() + .prepare(`SELECT rolled_back_by FROM deployments WHERE id = ?`) + .get(promote.deploymentId) as Record; expect(promoteRow.rolled_back_by).toBe(result.deploymentId); // Rollback deployment_servers rows are pending. const rbServers = db() .prepare(`SELECT restart_state FROM deployment_servers WHERE deployment_id = ?`) - .all(result.deploymentId) as any[]; + .all(result.deploymentId) as Record[]; expect(rbServers.length).toBe(2); for (const s of rbServers) expect(s.restart_state).toBe('pending'); // Second rollback attempt is rejected. - await expect(rollbackPromoteDeployment(promote.deploymentId)).rejects.toMatchObject({ kind: 'already-rolled-back' }); + await expect(rollbackPromoteDeployment(promote.deploymentId)).rejects.toMatchObject({ + kind: 'already-rolled-back' + }); }); }); describe('multi-script beta', () => { it('open + promote + rollback bundle two scripts as one atomic unit', async () => { await seedFleet(); - const { - createScript, - getScript, - getLatestVersion - } = await import('../../src/lib/server/fleet/scripts.js'); - const { - openBeta, - promoteBeta, - listBetaScripts - } = await import('../../src/lib/server/fleet/beta.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const { createScript, getScript, getLatestVersion } = + await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta, promoteBeta, listBetaScripts } = + await import('../../src/lib/server/fleet/beta.js'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); const { heldByBeta } = await import('../../src/lib/server/fleet/servers.js'); const { db } = await import('../../src/lib/server/db/index.js'); @@ -343,7 +409,9 @@ describe('multi-script beta', () => { // Both scripts are joined to the beta. const bundle = listBetaScripts(betaId); expect(bundle.length).toBe(2); - expect(new Set(bundle.map((r) => r.script_id))).toEqual(new Set([a.script.id, b.script.id])); + expect(new Set(bundle.map((r) => r.script_id))).toEqual( + new Set([a.script.id, b.script.id]) + ); // Both target rel paths are held on both servers. const held1 = heldByBeta('t5_zombie_theater_1'); @@ -352,25 +420,32 @@ describe('multi-script beta', () => { // Force ready-promote and simulate tested transition (the beta-runner // would do this for real). - db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) + db() + .prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`) .run(new Date().toISOString(), betaId); db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId); - db().prepare(`UPDATE script_versions SET state = 'tested' WHERE id IN (?, ?)`).run(va.id, vb.id); + db() + .prepare(`UPDATE script_versions SET state = 'tested' WHERE id IN (?, ?)`) + .run(va.id, vb.id); const promote = await promoteBeta(betaId); expect(new Set(promote.scriptIds)).toEqual(new Set([a.script.id, b.script.id])); // ONE deployment, TWO deployment_scripts rows. - const dep = db().prepare(`SELECT * FROM deployments WHERE id = ?`).get(promote.deploymentId) as any; + const dep = db() + .prepare(`SELECT * FROM deployments WHERE id = ?`) + .get(promote.deploymentId) as Record; expect(dep.action).toBe('promote'); expect(dep.beta_id).toBe(betaId); const depScripts = db() .prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ? ORDER BY script_id`) - .all(promote.deploymentId) as any[]; + .all(promote.deploymentId) as Record[]; expect(depScripts.length).toBe(2); // Two files per server (hud + flag) → 4 deployment_files rows. - const files = db().prepare(`SELECT * FROM deployment_files WHERE deployment_id = ?`).all(promote.deploymentId) as any[]; + const files = db() + .prepare(`SELECT * FROM deployment_files WHERE deployment_id = ?`) + .all(promote.deploymentId) as Record[]; expect(files.length).toBe(4); // Both scripts' current_version_id updated. @@ -386,7 +461,7 @@ describe('multi-script beta', () => { // The rollback deployment mirrors both scripts. const rbScripts = db() .prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ?`) - .all(rb.deploymentId) as any[]; + .all(rb.deploymentId) as Record[]; expect(rbScripts.length).toBe(2); }); }); diff --git a/tests/unit/script-library.test.ts b/tests/unit/script-library.test.ts index e51bee7..0ac69a6 100644 --- a/tests/unit/script-library.test.ts +++ b/tests/unit/script-library.test.ts @@ -1,17 +1,44 @@ import { describe, expect, it } from 'vitest'; -import { betaSelectionHref, canTest, mapLabel, parseBetaSelection, scriptStatus, type LibraryScript } from '../../src/lib/script-library.js'; +import { + betaSelectionHref, + canTest, + mapLabel, + parseBetaSelection, + scriptStatus, + type LibraryScript +} from '../../src/lib/script-library.js'; const script: LibraryScript = { - id: 1, name: 'HUD', target_rel_path: 'hud.gsc', scope_type: null, scope_value: null, - current_version_id: null, currentVersion: null, versionCount: 1, servers: [], runningBetaId: null, - latestVersion: { id: 10, version: 1, sha256: 'abc', size: 12, state: 'untested', uploaded_at: '2026-09-18' } + id: 1, + name: 'HUD', + target_rel_path: 'hud.gsc', + scope_type: null, + scope_value: null, + current_version_id: null, + currentVersion: null, + versionCount: 1, + servers: [], + runningBetaId: null, + latestVersion: { + id: 10, + version: 1, + sha256: 'abc', + size: 12, + state: 'untested', + uploaded_at: '2026-09-18' + } }; describe('script library state', () => { - it.each([['untested', 'untested'], ['tested', 'tested'], ['test-failed', 'failed']] as const)( - 'preserves %s for a never-promoted version', (state, expected) => { - expect(scriptStatus({ ...script, latestVersion: { ...script.latestVersion!, state } })).toBe(expected); - }); + it.each([ + ['untested', 'untested'], + ['tested', 'tested'], + ['test-failed', 'failed'] + ] as const)('preserves %s for a never-promoted version', (state, expected) => { + expect( + scriptStatus({ ...script, latestVersion: { ...script.latestVersion!, state } }) + ).toBe(expected); + }); it('distinguishes an active beta, a promoted release, and a newer upload', () => { expect(scriptStatus({ ...script, runningBetaId: 5 })).toBe('testing'); expect(canTest({ ...script, runningBetaId: 5 })).toBe(false); @@ -30,14 +57,27 @@ describe('script library state', () => { describe('beta selection links', () => { it('round-trips exact versions in a multi-script selection', () => { - const picks = [{ scriptId: 4, versionId: 20 }, { scriptId: 7, versionId: 25 }]; - expect(parseBetaSelection(new URL(betaSelectionHref(picks), 'http://localhost').searchParams)).toEqual(picks); + const picks = [ + { scriptId: 4, versionId: 20 }, + { scriptId: 7, versionId: 25 } + ]; + expect( + parseBetaSelection(new URL(betaSelectionHref(picks), 'http://localhost').searchParams) + ).toEqual(picks); }); it('preserves existing plugin import links', () => { - expect(parseBetaSelection(new URLSearchParams('pluginId=8&versionId=32'))).toEqual([{ scriptId: 8, versionId: 32 }]); + expect(parseBetaSelection(new URLSearchParams('pluginId=8&versionId=32'))).toEqual([ + { scriptId: 8, versionId: 32 } + ]); + }); + it.each([ + 'artifact=0:1', + 'artifact=1:NaN', + 'artifact=1:2junk', + 'artifact=1:2&artifact=1:3', + 'artifact=9007199254740992:1', + 'pluginId=8' + ])('rejects malformed or ambiguous selections: %s', (query) => { + expect(() => parseBetaSelection(new URLSearchParams(query))).toThrow(); }); - it.each(['artifact=0:1', 'artifact=1:NaN', 'artifact=1:2junk', 'artifact=1:2&artifact=1:3', 'artifact=9007199254740992:1', 'pluginId=8'])( - 'rejects malformed or ambiguous selections: %s', query => { - expect(() => parseBetaSelection(new URLSearchParams(query))).toThrow(); - }); }); diff --git a/tests/unit/script-upload.test.ts b/tests/unit/script-upload.test.ts index a4cb035..c295a4f 100644 --- a/tests/unit/script-upload.test.ts +++ b/tests/unit/script-upload.test.ts @@ -3,28 +3,49 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; beforeEach(async () => { - vi.resetModules(); fixture = await newFixture(); + vi.resetModules(); + fixture = await newFixture(); delete process.env.RCON_PASSWORD; }); -afterEach(async () => { await fixture.cleanup(); }); +afterEach(async () => { + await fixture.cleanup(); +}); function request(content: string, reuse = false) { - const form = new FormData(); form.append('content', new File([content], 'hud.gsc')); + const form = new FormData(); + form.append('content', new File([content], 'hud.gsc')); if (reuse) form.append('reuseExisting', 'true'); return new Request('http://localhost/api/scripts/inspect', { method: 'POST', body: form }); } async function seed() { const mod = await import('../../src/lib/server/fleet/scripts.js'); - return { ...mod, ...mod.createScript({ name: 'HUD', targetRelPath: 'hud.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('main(){}') }) }; + return { + ...mod, + ...mod.createScript({ + name: 'HUD', + targetRelPath: 'hud.gsc', + scopeType: 'game', + scopeValue: 't5', + content: Buffer.from('main(){}') + }) + }; } describe('script upload review', () => { it('inspects content against all versions without creating records or including plugins', async () => { const { script, version, uploadNewVersion, createScript, listVersions } = await seed(); uploadNewVersion(script.id, Buffer.from('changed(){}')); - createScript({ name: 'Plugin', targetRelPath: 'plugin.dll', kind: 'plugin', content: Buffer.from('main(){}') }); + createScript({ + name: 'Plugin', + targetRelPath: 'plugin.dll', + kind: 'plugin', + content: Buffer.from('main(){}') + }); const { POST } = await import('../../src/routes/api/scripts/inspect/+server.js'); const res = await POST({ request: request('main(){}') }); - expect(await res.json()).toEqual({ sha256: version.sha256, duplicates: [{ scriptId: script.id, versionId: version.id, version: 1 }] }); + expect(await res.json()).toEqual({ + sha256: version.sha256, + duplicates: [{ scriptId: script.id, versionId: version.id, version: 1 }] + }); expect(listVersions(script.id)).toHaveLength(2); expect((await POST({ request: request('') })).status).toBe(400); }); @@ -32,9 +53,14 @@ describe('script upload review', () => { const { script, version, uploadNewVersion, listVersions, getScript } = await seed(); uploadNewVersion(script.id, Buffer.from('changed(){}')); const { POST } = await import('../../src/routes/api/scripts/[id]/versions/+server.js'); - const invoke = (content: string) => POST({ params: { id: String(script.id) }, request: request(content, true) }); + const invoke = (content: string) => + POST({ params: { id: String(script.id) }, request: request(content, true) }); const res = await invoke('main(){}'); - expect(await res.json()).toMatchObject({ ok: true, reused: true, version: { id: version.id, version: 1 } }); + expect(await res.json()).toMatchObject({ + ok: true, + reused: true, + version: { id: version.id, version: 1 } + }); expect(listVersions(script.id)).toHaveLength(2); const added = await invoke('third(){}'); expect(await added.json()).toMatchObject({ ok: true, version: { version: 3 } }); @@ -48,9 +74,17 @@ describe('script upload review', () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false }); + await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: ['t5_zombie_theater_1'], + waitForHealth: false, + waitForMatch: false + }); const { POST } = await import('../../src/routes/api/scripts/[id]/versions/+server.js'); - const res = await POST({ params: { id: String(script.id) }, request: request('changed(){}', true) }); + const res = await POST({ + params: { id: String(script.id) }, + request: request('changed(){}', true) + }); expect(res.status).toBe(409); expect(listVersions(script.id)).toHaveLength(1); }); @@ -58,10 +92,17 @@ describe('script upload review', () => { const { script, version, uploadNewVersion } = await seed(); const latest = uploadNewVersion(script.id, Buffer.from('changed(){}')); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(version.id, script.id); const { GET } = await import('../../src/routes/api/scripts/+server.js'); - const body = await (await GET({ url: new URL('http://localhost/api/scripts?kind=script') })).json(); - expect(body.scripts[0]).toMatchObject({ currentVersion: { id: version.id, version: 1 }, latestVersion: { id: latest.id, version: 2 } }); + const body = await ( + await GET({ url: new URL('http://localhost/api/scripts?kind=script') }) + ).json(); + expect(body.scripts[0]).toMatchObject({ + currentVersion: { id: version.id, version: 1 }, + latestVersion: { id: latest.id, version: 2 } + }); expect(body.scripts[0].currentVersion).not.toHaveProperty('content'); }); }); diff --git a/tests/unit/seed-fixture.test.ts b/tests/unit/seed-fixture.test.ts index 7b176b7..e146d8d 100644 --- a/tests/unit/seed-fixture.test.ts +++ b/tests/unit/seed-fixture.test.ts @@ -1,12 +1,26 @@ import Database from 'better-sqlite3'; -import { access, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { + access, + mkdtemp, + readFile, + realpath, + rm, + stat, + symlink, + writeFile +} from 'node:fs/promises'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { parseFixtureOptions, reserveFixture } from '../../scripts/fixture-options.js'; -import { FIXTURE_KEY, FIXTURE_VERSION, refreshFixtureData, seedFixtureData } from '../../scripts/fixture-data.js'; +import { + FIXTURE_KEY, + FIXTURE_VERSION, + refreshFixtureData, + seedFixtureData +} from '../../scripts/fixture-data.js'; import { newFixture, type Fixture } from '../helpers/fixture.js'; const NOW = Date.parse('2026-10-07T12:00:00Z'); @@ -14,24 +28,42 @@ const run = promisify(execFile); describe('fixture destination isolation', () => { let root: string; - beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'cherry-fixture-safety-')); }); - afterEach(async () => { vi.unstubAllEnvs(); await rm(root, { recursive: true, force: true }); }); + beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'cherry-fixture-safety-')); + }); + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); + }); it('requires an explicit absolute destination even with a production database in the environment', () => { vi.stubEnv('DB_PATH', join(root, 'production.db')); expect(() => parseFixtureOptions([], NOW)).toThrow(/--db/); expect(() => parseFixtureOptions(['--db', './data/cherry.db'], NOW)).toThrow(/absolute/); - expect(parseFixtureOptions(['--db', join(root, 'preview', 'fixture.db')], NOW)) - .toEqual({ dbPath: join(root, 'preview', 'fixture.db'), now: NOW, watch: false }); - expect(() => parseFixtureOptions(['--db', join(root, 'preview', 'fixture.db'), '--force'], NOW)).toThrow(/Unknown option/); + expect(parseFixtureOptions(['--db', join(root, 'preview', 'fixture.db')], NOW)).toEqual({ + dbPath: join(root, 'preview', 'fixture.db'), + now: NOW, + watch: false + }); + expect(() => + parseFixtureOptions(['--db', join(root, 'preview', 'fixture.db'), '--force'], NOW) + ).toThrow(/Unknown option/); }); it('keeps deterministic snapshot time separate from a live watch', () => { const target = ['--db', join(root, 'preview', 'fixture.db')]; - expect(parseFixtureOptions([...target, '--now', '2026-10-07T08:00:00-04:00'], 0).now).toBe(NOW); - expect(() => parseFixtureOptions([...target, '--now', '2026-10-07T12:00:00'], NOW)).toThrow(/timezone/); - expect(() => parseFixtureOptions([...target, '--now', '2026-10-07T12:00:00Z', '--watch'], NOW)).toThrow(/separately/); - expect(() => parseFixtureOptions([...target, '--db', join(root, 'second', 'fixture.db')], NOW)).toThrow(/Duplicate/); + expect(parseFixtureOptions([...target, '--now', '2026-10-07T08:00:00-04:00'], 0).now).toBe( + NOW + ); + expect(() => parseFixtureOptions([...target, '--now', '2026-10-07T12:00:00'], NOW)).toThrow( + /timezone/ + ); + expect(() => + parseFixtureOptions([...target, '--now', '2026-10-07T12:00:00Z', '--watch'], NOW) + ).toThrow(/separately/); + expect(() => + parseFixtureOptions([...target, '--db', join(root, 'second', 'fixture.db')], NOW) + ).toThrow(/Duplicate/); }); it('reserves a private new directory and refuses reuse without touching the database or sidecars', async () => { @@ -49,7 +81,9 @@ describe('fixture destination isolation', () => { const target = await reserveFixture(join(root, 'production', 'database.db')); await writeFile(target.dbPath, 'do not replace'); await symlink(target.root, join(root, 'preview'), 'dir'); - await expect(reserveFixture(join(root, 'preview', 'database.db'))).rejects.toThrow(/already exists/); + await expect(reserveFixture(join(root, 'preview', 'database.db'))).rejects.toThrow( + /already exists/ + ); expect(await readFile(target.dbPath, 'utf8')).toBe('do not replace'); }); @@ -57,12 +91,32 @@ describe('fixture destination isolation', () => { const ambient = join(root, 'production.db'); await writeFile(ambient, 'original production sentinel'); const destination = join(root, 'preview', 'fixture.db'); - const { stdout } = await run(process.execPath, ['--import', 'tsx', 'scripts/seed-fixture.ts', '--db', destination, '--now', '2026-10-07T12:00:00Z'], { - cwd: process.cwd(), timeout: 30_000, maxBuffer: 200_000, - env: { ...process.env, DB_PATH: ambient, SERVERS_ROOT: join(root, 'production-t5'), - T6_SERVERS_ROOT: join(root, 'production-t6'), BETA_ARCHIVE_DIR: join(root, 'production-archive'), - GAME_FILES_ROOT: join(root, 'production-files'), RCON_PASSWORD: 'ambient-secret-must-not-escape' } - }); + const { stdout } = await run( + process.execPath, + [ + '--import', + 'tsx', + 'scripts/seed-fixture.ts', + '--db', + destination, + '--now', + '2026-10-07T12:00:00Z' + ], + { + cwd: process.cwd(), + timeout: 30_000, + maxBuffer: 200_000, + env: { + ...process.env, + DB_PATH: ambient, + SERVERS_ROOT: join(root, 'production-t5'), + T6_SERVERS_ROOT: join(root, 'production-t6'), + BETA_ARCHIVE_DIR: join(root, 'production-archive'), + GAME_FILES_ROOT: join(root, 'production-files'), + RCON_PASSWORD: 'ambient-secret-must-not-escape' + } + } + ); expect(await readFile(ambient, 'utf8')).toBe('original production sentinel'); await expect(access(join(root, 'production-t5'))).rejects.toThrow(); await expect(access(join(root, 'production-files'))).rejects.toThrow(); @@ -78,19 +132,50 @@ describe('fixture destination isolation', () => { try { expect(seeded.pragma('integrity_check', { simple: true })).toBe('ok'); expect(seeded.pragma('foreign_key_check')).toEqual([]); - expect(seeded.prepare('SELECT COUNT(*) n FROM leaderboard_matches').get()).toEqual({ n: 500 }); - expect(seeded.prepare("SELECT COUNT(*) n FROM auth_users WHERE username='demo' AND role='admin'").get()).toEqual({ n: 1 }); - expect(seeded.prepare("SELECT COUNT(*) n FROM ec_host_operations WHERE state IN ('pending','running','accepted')").get()).toEqual({ n: 0 }); - } finally { seeded.close(); } + expect(seeded.prepare('SELECT COUNT(*) n FROM leaderboard_matches').get()).toEqual({ + n: 500 + }); + expect( + seeded + .prepare( + "SELECT COUNT(*) n FROM auth_users WHERE username='demo' AND role='admin'" + ) + .get() + ).toEqual({ n: 1 }); + expect( + seeded + .prepare( + "SELECT COUNT(*) n FROM ec_host_operations WHERE state IN ('pending','running','accepted')" + ) + .get() + ).toEqual({ n: 0 }); + } finally { + seeded.close(); + } }, 35_000); it('cleans up only its newly owned directory when fixture creation cannot complete', async () => { - const retained = join(root, 'keep.txt'); await writeFile(retained, 'keep'); + const retained = join(root, 'keep.txt'); + await writeFile(retained, 'keep'); // This filename collides with the generated environment file. Whether // rejected up front or at write time, no partial preview should remain. - await expect(run(process.execPath, ['--import', 'tsx', 'scripts/seed-fixture.ts', '--db', join(root, 'failed-preview', 'preview.env')], { - cwd: process.cwd(), timeout: 30_000, maxBuffer: 200_000 - })).rejects.toThrow(); + await expect( + run( + process.execPath, + [ + '--import', + 'tsx', + 'scripts/seed-fixture.ts', + '--db', + join(root, 'failed-preview', 'preview.env') + ], + { + cwd: process.cwd(), + timeout: 30_000, + maxBuffer: 200_000 + } + ) + ).rejects.toThrow(); await expect(access(join(root, 'failed-preview'))).rejects.toThrow(); expect(await readFile(retained, 'utf8')).toBe('keep'); }, 35_000); @@ -100,17 +185,23 @@ describe('useful synthetic fixture data', () => { let fixture: Fixture; let handle: Database.Database; beforeAll(async () => { - vi.resetModules(); fixture = await newFixture([]); + vi.resetModules(); + fixture = await newFixture([]); const { db } = await import('../../src/lib/server/db/index.js'); const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js'); const { ensureAuthSchema } = await import('../../src/lib/server/auth/schema.js'); const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js'); - handle = db(); ensureHostSchema(handle); ensureAuthSchema(handle); ensureGameFileSchema(handle); + handle = db(); + ensureHostSchema(handle); + ensureAuthSchema(handle); + ensureGameFileSchema(handle); seedFixtureData(handle, NOW); }, 20_000); beforeEach(() => handle.exec('SAVEPOINT fixture_test')); afterEach(() => handle.exec('ROLLBACK TO fixture_test; RELEASE fixture_test')); - afterAll(async () => { await fixture?.cleanup(); }); + afterAll(async () => { + await fixture?.cleanup(); + }); const count = (sql: string) => (handle.prepare(sql).get() as { n: number }).n; it('provides complete paginated match, fleet and metrics evidence with valid relationships', () => { @@ -119,78 +210,187 @@ describe('useful synthetic fixture data', () => { expect(count('SELECT COUNT(*) n FROM ec_hosts')).toBe(3); expect(count('SELECT COUNT(*) n FROM leaderboard_matches')).toBe(500); expect(count("SELECT COUNT(*) n FROM leaderboard_matches WHERE status='active'")).toBe(6); - expect(count("SELECT COUNT(*) n FROM leaderboard_matches WHERE status='completed'")).toBeGreaterThan(400); - expect(handle.prepare('SELECT DISTINCT start_players FROM leaderboard_matches WHERE start_players IS NOT NULL ORDER BY start_players').all()) - .toEqual([1, 2, 3, 4].map(start_players => ({ start_players }))); - expect(count('SELECT COUNT(*) n FROM leaderboard_matches WHERE start_players IS NULL')).toBeGreaterThan(0); - expect(count('SELECT COUNT(DISTINCT guid) n FROM leaderboard_match_players')).toBeGreaterThan(50); - const metrics = handle.prepare('SELECT COUNT(DISTINCT at) samples,MIN(at) oldest,MAX(at) newest FROM metric_samples').get() as { samples: number; oldest: number; newest: number }; + expect( + count("SELECT COUNT(*) n FROM leaderboard_matches WHERE status='completed'") + ).toBeGreaterThan(400); + expect( + handle + .prepare( + 'SELECT DISTINCT start_players FROM leaderboard_matches WHERE start_players IS NOT NULL ORDER BY start_players' + ) + .all() + ).toEqual([1, 2, 3, 4].map((start_players) => ({ start_players }))); + expect( + count('SELECT COUNT(*) n FROM leaderboard_matches WHERE start_players IS NULL') + ).toBeGreaterThan(0); + expect( + count('SELECT COUNT(DISTINCT guid) n FROM leaderboard_match_players') + ).toBeGreaterThan(50); + const metrics = handle + .prepare( + 'SELECT COUNT(DISTINCT at) samples,MIN(at) oldest,MAX(at) newest FROM metric_samples' + ) + .get() as { samples: number; oldest: number; newest: number }; expect(metrics.samples).toBe(7 * 24 * 60); expect(metrics.newest).toBe(NOW); expect(metrics.newest - metrics.oldest).toBe((metrics.samples - 1) * 60_000); - expect(count('SELECT COUNT(*) n FROM metric_samples WHERE players<0 OR players>capacity')).toBe(0); + expect( + count('SELECT COUNT(*) n FROM metric_samples WHERE players<0 OR players>capacity') + ).toBe(0); }); it('supports actual leaderboard pagination and evidence for every exact squad size', async () => { const { getLeaderboard } = await import('../../src/lib/server/fleet/leaderboard.js'); const first = getLeaderboard({ game: 't5', map: 'theater', players: 2, limit: 25 }); - const second = getLeaderboard({ game: 't5', map: 'theater', players: 2, limit: 25, offset: 25 }); + const second = getLeaderboard({ + game: 't5', + map: 'theater', + players: 2, + limit: 25, + offset: 25 + }); expect(first.total).toBeGreaterThan(50); expect(first.entries).toHaveLength(25); expect(second.entries).toHaveLength(25); - expect(new Set([...first.entries, ...second.entries].map(entry => entry.guid)).size).toBe(50); + expect(new Set([...first.entries, ...second.entries].map((entry) => entry.guid)).size).toBe( + 50 + ); for (const players of [1, 2, 3, 4] as const) { const board = getLeaderboard({ players }); expect(board.total).toBeGreaterThan(0); const evidence = board.entries[0].records.highestRound; - expect(handle.prepare('SELECT start_players FROM leaderboard_matches WHERE id=?').get(evidence.matchId)).toEqual({ start_players: players }); + expect( + handle + .prepare('SELECT start_players FROM leaderboard_matches WHERE id=?') + .get(evidence.matchId) + ).toEqual({ start_players: players }); } }); it('never seeds executable work or usable host credentials and pauses automated lifecycle policies', async () => { - expect(count("SELECT COUNT(*) n FROM server_deployments WHERE state IN ('queued','waiting','running','awaiting-health')")).toBe(0); - expect(count("SELECT COUNT(*) n FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')")).toBe(0); - expect(count("SELECT COUNT(*) n FROM maintenance_jobs WHERE state IN ('queued','running','waiting')")).toBe(0); - expect(count("SELECT COUNT(*) n FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE b.state IN ('running','ready-promote')")).toBe(0); - expect(count("SELECT COUNT(*) n FROM deployment_servers WHERE restart_state IN ('pending','failed','held')")).toBe(0); - const { maintenancePolicies } = await import('../../src/lib/server/maintenance/policies.js'); - expect(maintenancePolicies()).toEqual({ paused: true, health: false, drift: false, updates: false }); - for (const row of handle.prepare('SELECT token_hash FROM ec_host_credentials').all() as { token_hash: string }[]) + expect( + count( + "SELECT COUNT(*) n FROM server_deployments WHERE state IN ('queued','waiting','running','awaiting-health')" + ) + ).toBe(0); + expect( + count( + "SELECT COUNT(*) n FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')" + ) + ).toBe(0); + expect( + count( + "SELECT COUNT(*) n FROM maintenance_jobs WHERE state IN ('queued','running','waiting')" + ) + ).toBe(0); + expect( + count( + "SELECT COUNT(*) n FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE b.state IN ('running','ready-promote')" + ) + ).toBe(0); + expect( + count( + "SELECT COUNT(*) n FROM deployment_servers WHERE restart_state IN ('pending','failed','held')" + ) + ).toBe(0); + const { maintenancePolicies } = + await import('../../src/lib/server/maintenance/policies.js'); + expect(maintenancePolicies()).toEqual({ + paused: true, + health: false, + drift: false, + updates: false + }); + for (const row of handle.prepare('SELECT token_hash FROM ec_host_credentials').all() as { + token_hash: string; + }[]) expect(row.token_hash).not.toMatch(/^[0-9a-f]{64}$/); expect(() => seedFixtureData(handle, NOW)).toThrow(/new database/); expect(count('SELECT COUNT(*) n FROM leaderboard_matches')).toBe(500); }); it('refuses watch updates without the exact ownership marker', () => { - const before = handle.prepare('SELECT name,last_checked_at FROM servers ORDER BY name').all(); + const before = handle + .prepare('SELECT name,last_checked_at FROM servers ORDER BY name') + .all(); handle.prepare('DELETE FROM settings WHERE key=?').run(FIXTURE_KEY); expect(() => refreshFixtureData(handle, NOW + 60_000)).toThrow(/unmarked/); - handle.prepare('INSERT INTO settings(key,value) VALUES(?,?)').run(FIXTURE_KEY, 'some-other-fixture'); + handle + .prepare('INSERT INTO settings(key,value) VALUES(?,?)') + .run(FIXTURE_KEY, 'some-other-fixture'); expect(() => refreshFixtureData(handle, NOW + 60_000)).toThrow(/unmarked/); - expect(handle.prepare('SELECT name,last_checked_at FROM servers ORDER BY name').all()).toEqual(before); + expect( + handle.prepare('SELECT name,last_checked_at FROM servers ORDER BY name').all() + ).toEqual(before); }); it('refreshes only owned observations while preserving unrelated resources and retained history', () => { - handle.prepare('INSERT INTO servers(name,last_checked_at) VALUES(?,?)').run('unowned-server', '2020-01-01'); - handle.prepare('INSERT INTO ec_hosts(id,name,created_at,last_heartbeat) VALUES(?,?,?,?)').run('unowned-host', 'Keep this host', '2020-01-01', '2020-01-01'); - const history = handle.prepare("SELECT id,last_received_at FROM leaderboard_matches WHERE status!='active' ORDER BY id").all(); - const operations = handle.prepare('SELECT id,state,updated_at FROM ec_host_operations ORDER BY id').all(); + handle + .prepare('INSERT INTO servers(name,last_checked_at) VALUES(?,?)') + .run('unowned-server', '2020-01-01'); + handle + .prepare('INSERT INTO ec_hosts(id,name,created_at,last_heartbeat) VALUES(?,?,?,?)') + .run('unowned-host', 'Keep this host', '2020-01-01', '2020-01-01'); + const history = handle + .prepare( + "SELECT id,last_received_at FROM leaderboard_matches WHERE status!='active' ORDER BY id" + ) + .all(); + const operations = handle + .prepare('SELECT id,state,updated_at FROM ec_host_operations ORDER BY id') + .all(); refreshFixtureData(handle, NOW + 60_000); - expect(handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY)).toEqual({ value: FIXTURE_VERSION }); - expect(handle.prepare('SELECT last_checked_at FROM servers WHERE name=?').get('unowned-server')).toEqual({ last_checked_at: '2020-01-01' }); - expect(handle.prepare('SELECT last_heartbeat FROM ec_hosts WHERE id=?').get('unowned-host')).toEqual({ last_heartbeat: '2020-01-01' }); - expect(handle.prepare("SELECT id,last_received_at FROM leaderboard_matches WHERE status!='active' ORDER BY id").all()).toEqual(history); - expect(handle.prepare('SELECT id,state,updated_at FROM ec_host_operations ORDER BY id').all()).toEqual(operations); - expect(count(`SELECT COUNT(*) n FROM servers WHERE last_checked_at='${new Date(NOW + 60_000).toISOString()}'`)).toBe(12); + expect(handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY)).toEqual({ + value: FIXTURE_VERSION + }); + expect( + handle.prepare('SELECT last_checked_at FROM servers WHERE name=?').get('unowned-server') + ).toEqual({ last_checked_at: '2020-01-01' }); + expect( + handle.prepare('SELECT last_heartbeat FROM ec_hosts WHERE id=?').get('unowned-host') + ).toEqual({ last_heartbeat: '2020-01-01' }); + expect( + handle + .prepare( + "SELECT id,last_received_at FROM leaderboard_matches WHERE status!='active' ORDER BY id" + ) + .all() + ).toEqual(history); + expect( + handle.prepare('SELECT id,state,updated_at FROM ec_host_operations ORDER BY id').all() + ).toEqual(operations); + expect( + count( + `SELECT COUNT(*) n FROM servers WHERE last_checked_at='${new Date(NOW + 60_000).toISOString()}'` + ) + ).toBe(12); }); it('restores synthetic observations after monitoring without making the offline host live', () => { - const original = handle.prepare('SELECT name,container_state,last_error,rcon_failures,last_players,last_roster FROM servers ORDER BY name').all(); - handle.prepare("UPDATE servers SET container_state='unknown',last_error='monitor timeout',rcon_failures=2,last_players=NULL,last_roster=NULL").run(); + const original = handle + .prepare( + 'SELECT name,container_state,last_error,rcon_failures,last_players,last_roster FROM servers ORDER BY name' + ) + .all(); + handle + .prepare( + "UPDATE servers SET container_state='unknown',last_error='monitor timeout',rcon_failures=2,last_players=NULL,last_roster=NULL" + ) + .run(); refreshFixtureData(handle, NOW + 60_000); - expect(handle.prepare('SELECT name,container_state,last_error,rcon_failures,last_players,last_roster FROM servers ORDER BY name').all()).toEqual(original); - const hosts = handle.prepare('SELECT last_heartbeat FROM ec_hosts').all() as { last_heartbeat: string }[]; - expect(hosts.filter(host => NOW + 60_000 - Date.parse(host.last_heartbeat) > 90_000)).toHaveLength(1); + expect( + handle + .prepare( + 'SELECT name,container_state,last_error,rcon_failures,last_players,last_roster FROM servers ORDER BY name' + ) + .all() + ).toEqual(original); + const hosts = handle.prepare('SELECT last_heartbeat FROM ec_hosts').all() as { + last_heartbeat: string; + }[]; + expect( + hosts.filter((host) => NOW + 60_000 - Date.parse(host.last_heartbeat) > 90_000) + ).toHaveLength(1); expect(handle.pragma('foreign_key_check')).toEqual([]); }); }); diff --git a/tests/unit/server-capacity.test.ts b/tests/unit/server-capacity.test.ts index b81eb06..37ee2e5 100644 --- a/tests/unit/server-capacity.test.ts +++ b/tests/unit/server-capacity.test.ts @@ -9,26 +9,61 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); - vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(now); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.restoreAllMocks(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); async function setup(count = 1) { const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); const names = Array.from({ length: count }, (_, i) => `server-${i}`); - for (const name of names) database.prepare(`INSERT INTO servers(name,created_at,game_port,container_state, - container_checked_at,last_checked_at,last_seen_at,last_players) VALUES(?, ?, 28960, 'running', ?, ?, ?, 3)`) - .run(name, iso(-60_000), iso(), iso(), iso()); - const { monitoringStatusWriter } = await import('../../src/lib/server/fleet/health-observations.js'); - function publish(name: string, capacity: number | null | undefined = 4, observedAt = iso(), ok = true) { - monitoringStatusWriter(name)({ observedAt: new Date().toISOString(), result: { ok: true, state: 'running' } }, - { observedAt, result: ok ? { ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 3, capacity, map: 'zombie_theater', gametype: 'zom', roster: [] } } - : { ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' } }); + for (const name of names) + database + .prepare( + `INSERT INTO servers(name,created_at,game_port,container_state, + container_checked_at,last_checked_at,last_seen_at,last_players) VALUES(?, ?, 28960, 'running', ?, ?, ?, 3)` + ) + .run(name, iso(-60_000), iso(), iso(), iso()); + const { monitoringStatusWriter } = + await import('../../src/lib/server/fleet/health-observations.js'); + function publish( + name: string, + capacity: number | null | undefined = 4, + observedAt = iso(), + ok = true + ) { + monitoringStatusWriter(name)( + { observedAt: new Date().toISOString(), result: { ok: true, state: 'running' } }, + { + observedAt, + result: ok + ? { + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { + players: 3, + capacity, + map: 'zombie_theater', + gametype: 'zom', + roster: [] + } + } + : { ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' } + } + ); } - return { database, names, publish, ...await import('../../src/lib/server/fleet/player-capacity.js') }; + return { + database, + names, + publish, + ...(await import('../../src/lib/server/fleet/player-capacity.js')) + }; } describe('current player capacity', () => { @@ -39,17 +74,26 @@ describe('current player capacity', () => { expect(currentCapacity(known, now + 90_001)).toBeNull(); expect(currentCapacity(known, now - 2000)).toBeNull(); expect(currentCapacity({ ...known, telemetry_after: iso(-1) }, now)).toBe(4); - for (const telemetry_after of [iso(), iso(1), 'invalid']) expect(currentCapacity({ ...known, telemetry_after }, now)).toBeNull(); - for (const capacity of [null, undefined, 0, -1, 4.5, NaN, Infinity]) expect(currentCapacity({ ...known, capacity }, now)).toBeNull(); - for (const capacity_observed_at of [null, undefined, 'invalid']) expect(currentCapacity({ ...known, capacity_observed_at }, now)).toBeNull(); + for (const telemetry_after of [iso(), iso(1), 'invalid']) + expect(currentCapacity({ ...known, telemetry_after }, now)).toBeNull(); + for (const capacity of [null, undefined, 0, -1, 4.5, NaN, Infinity]) + expect(currentCapacity({ ...known, capacity }, now)).toBeNull(); + for (const capacity_observed_at of [null, undefined, 'invalid']) + expect(currentCapacity({ ...known, capacity_observed_at }, now)).toBeNull(); }); it('projects the accepted structured sample into REST and initial SSE without probing', async () => { const api = await setup(); api.publish('server-0'); - const rcon = vi.spyOn(await import('../../src/lib/server/integrations/rcon.js'), 'rconStatusResult'); + const rcon = vi.spyOn( + await import('../../src/lib/server/integrations/rcon.js'), + 'rconStatusResult' + ); const { fleetSnapshot } = await import('../../src/lib/server/fleet/snapshot.js'); - expect(fleetSnapshot().servers[0]).toMatchObject({ capacity: 4, capacity_observed_at: iso() }); + expect(fleetSnapshot().servers[0]).toMatchObject({ + capacity: 4, + capacity_observed_at: iso() + }); const { GET } = await import('../../src/routes/api/events/servers/+server.js'); const reader = (await GET()).body!.getReader(); try { @@ -57,7 +101,9 @@ describe('current player capacity', () => { const frame = new TextDecoder().decode((await reader.read()).value); const body = JSON.parse(frame.split('data: ')[1]); expect(body.entries[0]).toMatchObject({ capacity: 4, capacityObservedAt: iso() }); - } finally { await reader.cancel(); } + } finally { + await reader.cancel(); + } expect(rcon).not.toHaveBeenCalled(); }); @@ -74,7 +120,9 @@ describe('current player capacity', () => { api.publish('server-0', 4, iso(), false); expect(read()).toEqual(unavailable); api.publish('server-0'); - api.database.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:newer failure'").run(iso(1)); + api.database + .prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:newer failure'") + .run(iso(1)); expect(read()).toEqual(unavailable); }); @@ -86,30 +134,45 @@ describe('current player capacity', () => { expect(api.playerCapacityReader()('server-0')).toEqual(unavailable); }); - it.each(['generation', 'lifecycle', 'registration', 'offline'] as const)('rejects capacity after %s changes', async change => { - const api = await setup(); - const hosts = await import('../../src/lib/server/hosts/index.js'); - const host = hosts.createHost({ name: 'Capacity host' }); - hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); - api.database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('instance','server-0',?,'library','image',28960,?)`).run(host.id, iso()); - api.publish('server-0'); - expect(api.playerCapacityReader()('server-0').capacity).toBe(4); - if (change === 'generation') api.database.prepare('UPDATE native_instances SET generation=2').run(); - if (change === 'lifecycle') api.database.prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('server-0',?)").run(iso(1)); - if (change === 'registration') api.database.prepare('UPDATE servers SET created_at=?').run(iso(1)); - if (change === 'offline') api.database.prepare('UPDATE ec_hosts SET last_heartbeat=?').run(iso(-91_000)); - expect(api.playerCapacityReader()('server-0')).toEqual(unavailable); - }); + it.each(['generation', 'lifecycle', 'registration', 'offline'] as const)( + 'rejects capacity after %s changes', + async (change) => { + const api = await setup(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Capacity host' }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); + api.database + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('instance','server-0',?,'library','image',28960,?)` + ) + .run(host.id, iso()); + api.publish('server-0'); + expect(api.playerCapacityReader()('server-0').capacity).toBe(4); + if (change === 'generation') + api.database.prepare('UPDATE native_instances SET generation=2').run(); + if (change === 'lifecycle') + api.database + .prepare( + "INSERT INTO restart_rollout(server,requested_at) VALUES('server-0',?)" + ) + .run(iso(1)); + if (change === 'registration') + api.database.prepare('UPDATE servers SET created_at=?').run(iso(1)); + if (change === 'offline') + api.database.prepare('UPDATE ec_hosts SET last_heartbeat=?').run(iso(-91_000)); + expect(api.playerCapacityReader()('server-0')).toEqual(unavailable); + } + ); it('reads a fleet in one joined query and keeps server-detail reads scoped', async () => { const api = await setup(20); - api.names.forEach(name => api.publish(name)); + api.names.forEach((name) => api.publish(name)); // Initialize schema before measuring the snapshot's actual reads. api.playerCapacityReader(); const prepare = vi.spyOn(api.database, 'prepare'); const fleet = api.playerCapacityReader(); - expect(api.names.map(name => fleet(name).capacity)).toEqual(Array(20).fill(4)); + expect(api.names.map((name) => fleet(name).capacity)).toEqual(Array(20).fill(4)); expect(prepare).toHaveBeenCalledTimes(1); expect(prepare.mock.calls[0][0]).toContain('WHERE s.destroyed_at IS NULL ORDER BY s.name'); prepare.mockClear(); diff --git a/tests/unit/server-chat.test.ts b/tests/unit/server-chat.test.ts index 12af024..f5975dd 100644 --- a/tests/unit/server-chat.test.ts +++ b/tests/unit/server-chat.test.ts @@ -4,6 +4,9 @@ import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent } from '../helpers/telemetry.js'; import type { GameEvent } from '../../src/lib/telemetry.js'; +import type { ContextualChatEntry } from '../../src/lib/game-chat.js'; + +type ChatMessageEntry = Extract; const SERVER = 't5_zombie_theater_1'; const T6_SERVER = 't6_zombie_town_1'; @@ -29,21 +32,36 @@ afterEach(async () => { async function log(lines: string[], lastWrite = 60, server = SERVER, filename?: string) { const folder = join(server.startsWith('t6_') ? fixture.t6 : fixture.t5, server, 'logs'); await mkdir(folder, { recursive: true }); - const path = join(folder, filename ?? (server.startsWith('t6_') ? 'games_zm.log' : 'games_sp.log')); + const path = join( + folder, + filename ?? (server.startsWith('t6_') ? 'games_zm.log' : 'games_sp.log') + ); await writeFile(path, lines.join('\n') + '\n'); await utimes(path, new Date(at(lastWrite)), new Date(at(lastWrite))); } -async function report(event: GameEvent['event'], seconds: number, round = 1, status = 200, diagnostic = true) { +async function report( + event: GameEvent['event'], + seconds: number, + round = 1, + status = 200, + diagnostic = true +) { const value = gameEvent(event, round, 1, SERVER); if (status === 200) { const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); saveTelemetry({ ...value, receivedAt: at(seconds) }); } if (diagnostic) { - const { recordTelemetryRequest } = await import('../../src/lib/server/fleet/telemetry-requests.js'); - recordTelemetryRequest({ value, receivedAt: at(seconds), status, - error: status === 200 ? null : 'Rejected request', contentType: 'application/json' }); + const { recordTelemetryRequest } = + await import('../../src/lib/server/fleet/telemetry-requests.js'); + recordTelemetryRequest({ + value, + receivedAt: at(seconds), + status, + error: status === 200 ? null : 'Rejected request', + contentType: 'application/json' + }); } } @@ -61,8 +79,12 @@ describe('server chat API', () => { const { db } = await import('../../src/lib/server/db/index.js'); const prepare = vi.spyOn(db(), 'prepare'); await get(); - expect(prepare.mock.calls.some(([sql]) => sql.includes('FROM telemetry_requests'))).toBe(false); - expect(prepare.mock.calls.some(([sql]) => sql.includes('last_received_at AS lastReceivedAt'))).toBe(false); + expect(prepare.mock.calls.some(([sql]) => sql.includes('FROM telemetry_requests'))).toBe( + false + ); + expect( + prepare.mock.calls.some(([sql]) => sql.includes('last_received_at AS lastReceivedAt')) + ).toBe(false); prepare.mockRestore(); const path = join(fixture.t5, SERVER, 'logs/games_sp.log'); const line = Buffer.from('61:00 say;abc;0;Player;café\n'); @@ -72,7 +94,10 @@ describe('server chat API', () => { await appendFile(path, line.subarray(-2)); await utimes(path, new Date(at(60)), new Date(at(60))); const body = await (await get()).json(); - expect(body.entries.map((entry: any) => entry.message.message)).toEqual(['first', 'café']); + expect(body.entries.map((entry: ChatMessageEntry) => entry.message.message)).toEqual([ + 'first', + 'café' + ]); expect(body.entries[1].context.realTime).toBe(at(60)); expect((await (await get()).json()).entries).toEqual(body.entries); }); @@ -86,15 +111,21 @@ describe('server chat API', () => { await appendFile(path, '61:00 say;abc;0;Player;retry me\n'); await utimes(path, new Date(at(60)), new Date(at(60))); const { db } = await import('../../src/lib/server/db/index.js'); - const handle = db(), original = handle.prepare.bind(handle); + const handle = db(), + original = handle.prepare.bind(handle); const prepare = vi.spyOn(handle, 'prepare').mockImplementation((sql: string) => { - if (sql.includes('INSERT INTO match_chat')) throw new Error('Temporary database failure'); + if (sql.includes('INSERT INTO match_chat')) + throw new Error('Temporary database failure'); return original(sql); }); vi.spyOn(console, 'error').mockImplementation(() => {}); expect((await get()).status).toBe(500); prepare.mockRestore(); - expect((await (await get()).json()).entries.map((entry: any) => entry.message.message)).toEqual(['first', 'retry me']); + expect( + (await (await get()).json()).entries.map( + (entry: ChatMessageEntry) => entry.message.message + ) + ).toEqual(['first', 'retry me']); }); it('skips empty, stopped and offline servers and collects one last time after players leave', async () => { @@ -114,13 +145,19 @@ describe('server chat API', () => { await collectFleetChat(); await collectFleetChat(); expect(read).toHaveBeenCalledTimes(2); - db().prepare("UPDATE servers SET last_players=1,container_state='stopped' WHERE name=?").run(SERVER); + db() + .prepare("UPDATE servers SET last_players=1,container_state='stopped' WHERE name=?") + .run(SERVER); await collectFleetChat(); expect(read).toHaveBeenCalledTimes(2); const host = createHost({ name: 'Offline' }); db().prepare("UPDATE servers SET container_state='running' WHERE name=?").run(SERVER); - db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) - VALUES('offline-instance',?,?,'library','image',28960,?)`).run(SERVER, host.id, at(0)); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('offline-instance',?,?,'library','image',28960,?)` + ) + .run(SERVER, host.id, at(0)); await collectFleetChat(); expect(read).toHaveBeenCalledTimes(2); }); @@ -134,10 +171,17 @@ describe('server chat API', () => { const body = await response.json(); expect(response.status).toBe(200); expect(response.headers.get('cache-control')).toBe('no-store'); - expect(body).toMatchObject({ ok: true, missing: false, entries: [ - { type: 'message', message: { player: 'Player', message: 'hello' }, - context: { realTime: at(50), matchElapsedMs: 50_000, round: 4 } } - ] }); + expect(body).toMatchObject({ + ok: true, + missing: false, + entries: [ + { + type: 'message', + message: { player: 'Player', message: 'hello' }, + context: { realTime: at(50), matchElapsedMs: 50_000, round: 4 } + } + ] + }); expect(JSON.stringify(body)).not.toContain('headshots'); vi.spyOn(Date, 'now').mockReturnValue(baseline + 900_000); expect(await (await get()).json()).toEqual(body); @@ -172,7 +216,9 @@ describe('server chat API', () => { it('reports read failures without returning server filesystem paths', async () => { const logs = await import('../../src/lib/server/fleet/logs.js'); - vi.spyOn(logs, 'readLogChunk').mockRejectedValue(new Error('read failed at /private/server/log')); + vi.spyOn(logs, 'readLogChunk').mockRejectedValue( + new Error('read failed at /private/server/log') + ); vi.spyOn(console, 'error').mockImplementation(() => {}); const response = await get(); expect(response.status).toBe(500); @@ -185,7 +231,11 @@ describe('server chat API', () => { await log(['60:50 say;abc;0;Player;first round', '61:00 J;abc;0;Player']); const body = await (await get()).json(); expect(body.entries).toHaveLength(1); - expect(body.entries[0].context).toEqual({ realTime: at(50), matchElapsedMs: 50_000, round: 1 }); + expect(body.entries[0].context).toEqual({ + realTime: at(50), + matchElapsedMs: 50_000, + round: 1 + }); }); it('does not reuse round diagnostics after their retention window', async () => { @@ -194,7 +244,11 @@ describe('server chat API', () => { await log(['60:50 say;abc;0;Player;older message', '61:00 J;abc;0;Player']); vi.spyOn(Date, 'now').mockReturnValue(baseline + 8 * 86_400_000); const body = await (await get()).json(); - expect(body.entries[0].context).toEqual({ realTime: at(50), matchElapsedMs: 50_000, round: null }); + expect(body.entries[0].context).toEqual({ + realTime: at(50), + matchElapsedMs: 50_000, + round: null + }); }); it('does not associate retained messages with a deleted server incarnation', async () => { @@ -202,16 +256,27 @@ describe('server chat API', () => { await report('heartbeat', 40, 8); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('DELETE FROM servers WHERE name = ?').run(SERVER); - db().prepare(`INSERT INTO servers (name, game, map, created_at) - VALUES (?, 't5', 'zombie_theater', ?)`).run(SERVER, at(70)); + db() + .prepare( + `INSERT INTO servers (name, game, map, created_at) + VALUES (?, 't5', 'zombie_theater', ?)` + ) + .run(SERVER, at(70)); await report('game_start', 80); await report('heartbeat', 90, 2); - await log(['60:50 say;abc;0;Player;previous server', '61:40 say;abc;0;Player;current server'], 100); + await log( + ['60:50 say;abc;0;Player;previous server', '61:40 say;abc;0;Player;current server'], + 100 + ); const body = await (await get()).json(); const messages = body.entries.filter((entry: { type: string }) => entry.type === 'message'); expect(messages).toHaveLength(1); expect(messages[0].message.message).toBe('current server'); - expect(messages[0].context).toEqual({ realTime: at(100), matchElapsedMs: 20_000, round: 2 }); + expect(messages[0].context).toEqual({ + realTime: at(100), + matchElapsedMs: 20_000, + round: 2 + }); }); it('clears at a new match before it writes to the log and keeps the old chat with its match', async () => { @@ -225,13 +290,24 @@ describe('server chat API', () => { await report('heartbeat', 90, 2); await log(['60:30 say;abc;0;Player;old match', '61:40 say;abc;0;Player;new match'], 100); const body = await (await get()).json(); - expect(body.entries.map((entry: any) => entry.message.message)).toEqual(['new match']); + expect(body.entries.map((entry: ChatMessageEntry) => entry.message.message)).toEqual([ + 'new match' + ]); const { getMatchDetail } = await import('../../src/lib/server/fleet/match-history.js'); - expect(getMatchDetail(1)!.chat.map(entry => entry.type === 'message' && entry.message.message)).toEqual(['old match']); + expect( + getMatchDetail(1)!.chat.map( + (entry) => entry.type === 'message' && entry.message.message + ) + ).toEqual(['old match']); }); it('clears on log boundaries and clock resets before the next telemetry report', async () => { - for (const boundary of ['61:00 InitGame:', '0:00 InitGame:', '0:00 J;abc;0;Player', 'InitGame:']) { + for (const boundary of [ + '61:00 InitGame:', + '0:00 InitGame:', + '0:00 J;abc;0;Player', + 'InitGame:' + ]) { await report('game_start', 0); await report('heartbeat', 20, 4); await log(['60:30 say;abc;0;Player;old match'], 30); @@ -242,12 +318,26 @@ describe('server chat API', () => { }); it('shows only the latest log session without telemetry', async () => { - await log(['80:00 say;abc;0;Player;old match', '0:00 InitGame:', '0:30 say;abc;0;Player;new match'], 60); + await log( + [ + '80:00 say;abc;0;Player;old match', + '0:00 InitGame:', + '0:30 say;abc;0;Player;new match' + ], + 60 + ); const entries = (await (await get()).json()).entries; - expect(entries.filter((entry: any) => entry.type === 'message').map((entry: any) => entry.message.message)) - .toEqual(['new match']); + expect( + entries + .filter((entry: ContextualChatEntry) => entry.type === 'message') + .map((entry: ChatMessageEntry) => entry.message.message) + ).toEqual(['new match']); await log(['80:00 say;abc;0;Player;old match', '0:00 InitGame:'], 60); - expect((await (await get()).json()).entries.some((entry: any) => entry.type === 'message')).toBe(false); + expect( + (await (await get()).json()).entries.some( + (entry: ContextualChatEntry) => entry.type === 'message' + ) + ).toBe(false); }); it('collects without a viewer and retains repeated messages after log rotation, restart and server deletion', async () => { @@ -265,7 +355,10 @@ describe('server chat API', () => { const { getMatchDetail } = await import('../../src/lib/server/fleet/match-history.js'); const saved = getMatchDetail(1)!.chat; expect(saved).toHaveLength(2); - expect(saved[0]).toMatchObject({ message: { message: 'hello' }, context: { realTime: at(30), round: 4 } }); + expect(saved[0]).toMatchObject({ + message: { message: 'hello' }, + context: { realTime: at(30), round: 4 } + }); await rm(join(fixture.t5, SERVER, 'logs'), { recursive: true }); closeDb(); expect((await (await get()).json()).entries).toEqual(saved); diff --git a/tests/unit/server-detail-api.test.ts b/tests/unit/server-detail-api.test.ts index 636406f..9335b2f 100644 --- a/tests/unit/server-detail-api.test.ts +++ b/tests/unit/server-detail-api.test.ts @@ -3,8 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); describe('server detail data', () => { it('filters history before the global limit and includes the matching campaign members', async () => { @@ -12,58 +17,124 @@ describe('server detail data', () => { const handle = db(); handle.exec("INSERT INTO servers (name) VALUES ('target'), ('other')"); handle.transaction(() => { - const deployment = handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES (?, 'promote', ?)"); - const member = handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'done')"); + const deployment = handle.prepare( + "INSERT INTO deployments (id, action, started_at) VALUES (?, 'promote', ?)" + ); + const member = handle.prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'done')" + ); deployment.run('older-target', '2026-01-01'); - member.run('older-target', 'target'); member.run('older-target', 'other'); - for (let n = 0; n < 501; n++) { deployment.run('other-' + n, '2026-09-18'); member.run('other-' + n, 'other'); } + member.run('older-target', 'target'); + member.run('older-target', 'other'); + for (let n = 0; n < 501; n++) { + deployment.run('other-' + n, '2026-09-18'); + member.run('other-' + n, 'other'); + } })(); const { GET } = await import('../../src/routes/api/campaigns/+server.js'); - const all = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=500') })).json(); + const all = await ( + await GET({ url: new URL('http://localhost/api/campaigns?limit=500') }) + ).json(); expect(all.deployments).toHaveLength(500); - const first = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=25') })).json(); - const second = await (await GET({ url: new URL('http://localhost/api/campaigns?limit=25&offset=25') })).json(); + const first = await ( + await GET({ url: new URL('http://localhost/api/campaigns?limit=25') }) + ).json(); + const second = await ( + await GET({ url: new URL('http://localhost/api/campaigns?limit=25&offset=25') }) + ).json(); expect(first.total).toBe(502); expect(first.deployments).toHaveLength(25); - expect(new Set([...first.deployments, ...second.deployments].map(row => row.id)).size).toBe(50); - const filtered = await (await GET({ url: new URL('http://localhost/api/campaigns?server=target') })).json(); + expect( + new Set([...first.deployments, ...second.deployments].map((row) => row.id)).size + ).toBe(50); + const filtered = await ( + await GET({ url: new URL('http://localhost/api/campaigns?server=target') }) + ).json(); expect(filtered.deployments.map((row: { id: string }) => row.id)).toEqual(['older-target']); expect(filtered.deployments[0].servers).toHaveLength(2); - expect((await (await GET({ url: new URL('http://localhost/api/campaigns?server=unknown') })).json()).deployments).toEqual([]); + expect( + ( + await ( + await GET({ url: new URL('http://localhost/api/campaigns?server=unknown') }) + ).json() + ).deployments + ).toEqual([]); }); it('identifies plugins, scripts, composed configs and unknown inventory files', async () => { const { db } = await import('../../src/lib/server/db/index.js'); db().exec("INSERT INTO servers (name) VALUES ('target')"); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { script, version } = createScript({ name:'Plugin', kind:'plugin', releaseVersion:'v1.0.0', targetRelPath:'native.bin', content:Buffer.from('plugin') }); - recordScript('target','native.bin',version.id,version.sha256,6,'beta'); - recordScript('target','dedicated_zm.cfg',null,'hash',10,'compose'); - recordScript('target','main.gsc',null,'hash',10,'beta'); - recordScript('target','unknown.dat',null,'hash',10,'manual'); - const { GET } = await import('../../src/routes/api/servers/[name]/scripts/+server.js'); - const body = await (await GET({ params: {name:'target'} })).json(); - expect(Object.fromEntries(body.scripts.map((file: {relPath:string;kind:string}) => [file.relPath,file.kind]))).toEqual({ - 'dedicated_zm.cfg':'config','main.gsc':'script','native.bin':'plugin','unknown.dat':'file' + const { script, version } = createScript({ + name: 'Plugin', + kind: 'plugin', + releaseVersion: 'v1.0.0', + targetRelPath: 'native.bin', + content: Buffer.from('plugin') }); - expect(body.scripts.find((file: { relPath: string }) => file.relPath === 'native.bin')).toMatchObject({ - artifactId: script.id, version: version.version, releaseVersion: 'v1.0.0', sha256: version.sha256 + recordScript('target', 'native.bin', version.id, version.sha256, 6, 'beta'); + recordScript('target', 'dedicated_zm.cfg', null, 'hash', 10, 'compose'); + recordScript('target', 'main.gsc', null, 'hash', 10, 'beta'); + recordScript('target', 'unknown.dat', null, 'hash', 10, 'manual'); + const { GET } = await import('../../src/routes/api/servers/[name]/scripts/+server.js'); + const body = await (await GET({ params: { name: 'target' } })).json(); + expect( + Object.fromEntries( + body.scripts.map((file: { relPath: string; kind: string }) => [ + file.relPath, + file.kind + ]) + ) + ).toEqual({ + 'dedicated_zm.cfg': 'config', + 'main.gsc': 'script', + 'native.bin': 'plugin', + 'unknown.dat': 'file' + }); + expect( + body.scripts.find((file: { relPath: string }) => file.relPath === 'native.bin') + ).toMatchObject({ + artifactId: script.id, + version: version.version, + releaseVersion: 'v1.0.0', + sha256: version.sha256 }); }); - it.each([['native.bin', 'file'], ['native.dll', 'plugin']])('omits saved artifact metadata for modified %s and falls back to its extension', async (path, kind) => { - const { db } = await import('../../src/lib/server/db/index.js'); - db().exec("INSERT INTO servers (name) VALUES ('target')"); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { version } = createScript({ name:'Plugin', kind:'plugin', releaseVersion:'v1.0.0', targetRelPath:path, content:Buffer.from('plugin') }); - const content = Buffer.from('modified plugin'); - const sha256 = createHash('sha256').update(content).digest('hex'); - recordScript('target',path,version.id,sha256,content.length,'manual'); - const { GET } = await import('../../src/routes/api/servers/[name]/scripts/+server.js'); - const body = await (await GET({ params: {name:'target'} })).json(); - expect(body.scripts).toEqual([expect.objectContaining({ - relPath: path, kind, artifactId: null, version: null, releaseVersion: null, - scriptVersionId: version.id, sha256, size: content.length, source: 'manual' - })]); - }); + it.each([ + ['native.bin', 'file'], + ['native.dll', 'plugin'] + ])( + 'omits saved artifact metadata for modified %s and falls back to its extension', + async (path, kind) => { + const { db } = await import('../../src/lib/server/db/index.js'); + db().exec("INSERT INTO servers (name) VALUES ('target')"); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); + const { version } = createScript({ + name: 'Plugin', + kind: 'plugin', + releaseVersion: 'v1.0.0', + targetRelPath: path, + content: Buffer.from('plugin') + }); + const content = Buffer.from('modified plugin'); + const sha256 = createHash('sha256').update(content).digest('hex'); + recordScript('target', path, version.id, sha256, content.length, 'manual'); + const { GET } = await import('../../src/routes/api/servers/[name]/scripts/+server.js'); + const body = await (await GET({ params: { name: 'target' } })).json(); + expect(body.scripts).toEqual([ + expect.objectContaining({ + relPath: path, + kind, + artifactId: null, + version: null, + releaseVersion: null, + scriptVersionId: version.id, + sha256, + size: content.length, + source: 'manual' + }) + ]); + } + ); }); diff --git a/tests/unit/server-freshness.test.ts b/tests/unit/server-freshness.test.ts index db78078..97c18ed 100644 --- a/tests/unit/server-freshness.test.ts +++ b/tests/unit/server-freshness.test.ts @@ -9,13 +9,25 @@ it('keeps healthy reports online across polling and delivery delays, then expire vi.useFakeTimers(); const reportAt = Date.parse('2026-09-18T12:00:00Z'); const timestamp = new Date(reportAt).toISOString(); - for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 60_000, 90_000, 90_001, 120_000]) { + for (const elapsed of [ + -999, + 0, + STATUS_REFRESH_MS, + STATUS_REFRESH_MS + 4000, + 45_000, + 60_000, + 90_000, + 90_001, + 120_000 + ]) { const now = reportAt + elapsed; vi.setSystemTime(now); const expected = elapsed <= 90_000; const health = classifyHealth(timestamp, null, 'running'); expect(health).toBe(expected ? 'online' : 'stale'); - expect(freshServer({ health, last_seen_at: timestamp, last_error: null } as FleetServer, now)).toBe(expected); + expect( + freshServer({ health, last_seen_at: timestamp, last_error: null } as FleetServer, now) + ).toBe(expected); } vi.setSystemTime(reportAt + 120_001); expect(classifyHealth(timestamp, null, 'running')).toBe('offline'); @@ -29,7 +41,12 @@ it('does not hide failures or accept missing, invalid, or far-future timestamps' expect(classifyHealth(timestamp, 'timeout', 'running')).toBe('stale'); expect(classifyHealth(timestamp, null, 'stopped')).toBe('offline'); expect(classifyHealth(timestamp, null, 'restarting')).toBe('starting'); - expect(freshServer({ health: 'online', last_seen_at: timestamp, last_error: 'timeout' } as FleetServer, now)).toBe(false); + expect( + freshServer( + { health: 'online', last_seen_at: timestamp, last_error: 'timeout' } as FleetServer, + now + ) + ).toBe(false); for (const invalid of [null, 'invalid', new Date(now + 1001).toISOString()]) { expect(freshStatusTimestamp(invalid, now)).toBe(false); expect(classifyHealth(invalid, null, 'running')).toBe('offline'); diff --git a/tests/unit/server-history-storage.test.ts b/tests/unit/server-history-storage.test.ts index 8c7d4e4..06382e9 100644 --- a/tests/unit/server-history-storage.test.ts +++ b/tests/unit/server-history-storage.test.ts @@ -12,7 +12,12 @@ function database(legacy = false) { if (legacy) { sql = sql.replace(/^\s*server_deleted_at TEXT,\n/gm, ''); for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) { - sql = sql.replace(new RegExp(`(CREATE TABLE IF NOT EXISTS ${name} \\([\\s\\S]*?server\\s+TEXT NOT NULL)(,)`), '$1 REFERENCES servers(name)$2'); + sql = sql.replace( + new RegExp( + `(CREATE TABLE IF NOT EXISTS ${name} \\([\\s\\S]*?server\\s+TEXT NOT NULL)(,)` + ), + '$1 REFERENCES servers(name)$2' + ); } } handle.exec(sql); @@ -47,18 +52,30 @@ describe('durable server history', () => { seedHistory(db); const events = db.prepare('SELECT * FROM beta_events ORDER BY id').all(); const runs = db.prepare('SELECT rowid, * FROM beta_test_runs').all(); - const servers = db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all(); + const servers = db + .prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers') + .all(); migrateServerHistory(db); migrateServerHistory(db); expect(db.prepare('SELECT rowid, * FROM beta_test_runs').all()).toEqual(runs); - expect(db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all()).toEqual(servers); + expect( + db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all() + ).toEqual(servers); expect(db.prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(events); db.prepare("UPDATE beta_servers SET stage = 'failed', error = 'historical failure'").run(); - expect(db.prepare('SELECT message FROM beta_events ORDER BY id DESC LIMIT 1').get()).toEqual({ message: 'server: Failed — historical failure' }); + expect( + db.prepare('SELECT message FROM beta_events ORDER BY id DESC LIMIT 1').get() + ).toEqual({ message: 'server: Failed — historical failure' }); db.prepare("DELETE FROM servers WHERE name = 'server'").run(); - expect(db.prepare('SELECT metrics FROM beta_test_runs').get()).toEqual({ metrics: '{"rounds":25}' }); - expect(db.prepare('SELECT payload FROM beta_test_evidence').get()).toEqual({ payload: '{"round":25}' }); - expect(db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get()).toEqual({ had_original: 0, prior_version_id: null }); + expect(db.prepare('SELECT metrics FROM beta_test_runs').get()).toEqual({ + metrics: '{"rounds":25}' + }); + expect(db.prepare('SELECT payload FROM beta_test_evidence').get()).toEqual({ + payload: '{"round":25}' + }); + expect( + db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get() + ).toEqual({ had_original: 0, prior_version_id: null }); expect(db.pragma('foreign_keys', { simple: true })).toBe(1); expect(db.pragma('foreign_key_check')).toEqual([]); }); @@ -67,25 +84,62 @@ describe('durable server history', () => { const db = database(); migrateServerHistory(db); seedHistory(db); - db.exec(`INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('collection', 'Collection', 'before', '["server"]');`); + db.exec( + `INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('collection', 'Collection', 'before', '["server"]');` + ); db.prepare("DELETE FROM servers WHERE name = 'server'").run(); - for (const table of ['beta_servers', 'deployment_servers', 'server_deployments', 'maintenance_jobs', 'watchdog_incidents', 'leaderboard_matches']) { - expect(db.prepare(`SELECT server, server_deleted_at FROM ${table}`).get()).toEqual({ server: 'server', server_deleted_at: expect.any(String) }); + for (const table of [ + 'beta_servers', + 'deployment_servers', + 'server_deployments', + 'maintenance_jobs', + 'watchdog_incidents', + 'leaderboard_matches' + ]) { + expect(db.prepare(`SELECT server, server_deleted_at FROM ${table}`).get()).toEqual({ + server: 'server', + server_deleted_at: expect.any(String) + }); } const previous = db.prepare('SELECT server_deleted_at FROM beta_servers').get(); - const retained = db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get() as { retained_server_deletions: string }; - expect(JSON.parse(retained.retained_server_deletions)).toEqual({ server: expect.any(String) }); - expect(db.prepare('SELECT stage, healthy_at FROM beta_servers').get()).toEqual({ stage: 'ready', healthy_at: 'healthy' }); - expect(db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()).toEqual({ restart_state: 'done', restarted_at: 'restarted' }); - expect(db.prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') }); - expect(db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) }); + const retained = db + .prepare('SELECT retained_server_deletions FROM server_launch_collections') + .get() as { retained_server_deletions: string }; + expect(JSON.parse(retained.retained_server_deletions)).toEqual({ + server: expect.any(String) + }); + expect(db.prepare('SELECT stage, healthy_at FROM beta_servers').get()).toEqual({ + stage: 'ready', + healthy_at: 'healthy' + }); + expect( + db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get() + ).toEqual({ restart_state: 'done', restarted_at: 'restarted' }); + expect( + db + .prepare( + 'SELECT (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files' + ) + .get() + ).toEqual({ content_after: Buffer.from('test') }); + expect( + db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get() + ).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) }); db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960); INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new'); INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('replacement', 'server', 'health', 'open', 'New incident', 'new', 'new');`); migrateServerHistory(db); expect(db.prepare('SELECT server_deleted_at FROM beta_servers').get()).toEqual(previous); - expect(db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get()).toEqual(retained); - expect(db.prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'replacement'").get()).toEqual({ server_deleted_at: null }); + expect( + db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get() + ).toEqual(retained); + expect( + db + .prepare( + "SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'replacement'" + ) + .get() + ).toEqual({ server_deleted_at: null }); }); it('cancels old pending work without rewriting completed outcomes', () => { @@ -96,12 +150,34 @@ describe('durable server history', () => { INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('pending', 'server', 'restart', 'waiting', 'Waiting for empty', 'pending', 'ui'); INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('removal', 'server', 'destroy', 'running', 'Removing server files and registry', 'pending', 'ui'); DELETE FROM servers WHERE name = 'server';`); - expect(db.prepare("SELECT state, stage, error FROM server_deployments WHERE id = 'pending'").get()) - .toEqual({ state: 'cancelled', stage: 'Waiting for empty', error: 'Server deleted' }); - expect(db.prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'pending'").get()).toEqual({ message: 'Cancelled: server deleted' }); - expect(db.prepare('SELECT state, detail FROM maintenance_jobs ORDER BY id').all()).toEqual([{ state: 'done', detail: 'Updated' }, { state: 'cancelled', detail: 'Server deleted' }]); - expect(db.prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'created'").get()).toEqual({ state: 'done', finished_at: 'after' }); - expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'removal'").get()).toEqual({ state: 'running', server_deleted_at: expect.any(String) }); + expect( + db + .prepare("SELECT state, stage, error FROM server_deployments WHERE id = 'pending'") + .get() + ).toEqual({ state: 'cancelled', stage: 'Waiting for empty', error: 'Server deleted' }); + expect( + db + .prepare( + "SELECT message FROM server_deployment_events WHERE deployment_id = 'pending'" + ) + .get() + ).toEqual({ message: 'Cancelled: server deleted' }); + expect(db.prepare('SELECT state, detail FROM maintenance_jobs ORDER BY id').all()).toEqual([ + { state: 'done', detail: 'Updated' }, + { state: 'cancelled', detail: 'Server deleted' } + ]); + expect( + db + .prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'created'") + .get() + ).toEqual({ state: 'done', finished_at: 'after' }); + expect( + db + .prepare( + "SELECT state, server_deleted_at FROM server_deployments WHERE id = 'removal'" + ) + .get() + ).toEqual({ state: 'running', server_deleted_at: expect.any(String) }); }); it('keeps only the current rebuild and its observing campaign and incident attached', () => { @@ -114,15 +190,47 @@ describe('durable server history', () => { INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('watchdog', 'server', 1, 'pending'); INSERT INTO watchdog_actions (id, incident_id, action, operation_id, campaign_id, created_at) VALUES ('action', 'incident', 'rebuild', 'rebuild', 'watchdog', 'now'); DELETE FROM servers WHERE name = 'server';`); - expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'rebuild'").get()).toEqual({ state: 'running', server_deleted_at: null }); - expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'").get()).toEqual({ server_deleted_at: null }); - expect(db.prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'incident'").get()).toEqual({ server_deleted_at: null }); - expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'campaign'").get()).toEqual({ server_deleted_at: expect.any(String) }); + expect( + db + .prepare( + "SELECT state, server_deleted_at FROM server_deployments WHERE id = 'rebuild'" + ) + .get() + ).toEqual({ state: 'running', server_deleted_at: null }); + expect( + db + .prepare( + "SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'" + ) + .get() + ).toEqual({ server_deleted_at: null }); + expect( + db + .prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'incident'") + .get() + ).toEqual({ server_deleted_at: null }); + expect( + db + .prepare( + "SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'campaign'" + ) + .get() + ).toEqual({ server_deleted_at: expect.any(String) }); db.exec(`UPDATE server_rebuild_jobs SET removed_at = 'now'; UPDATE server_deployments SET state = 'queued' WHERE id = 'rebuild'; DELETE FROM settings WHERE key = 'server_history_v1';`); migrateServerHistory(db); - expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'rebuild'").get()).toEqual({ server_deleted_at: null }); - expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'").get()).toEqual({ server_deleted_at: null }); + expect( + db + .prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'rebuild'") + .get() + ).toEqual({ server_deleted_at: null }); + expect( + db + .prepare( + "SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'" + ) + .get() + ).toEqual({ server_deleted_at: null }); }); it('backfills missing historical servers while excluding initial launches with no deletion evidence', () => { @@ -139,13 +247,47 @@ describe('durable server history', () => { INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('gone', 'queued', 'old'), ('tombstone', 'done', 'old'); INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('old', 'Old collection', 'old', '["tombstone","gone","new"]');`); migrateServerHistory(db); - expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'old'").get()).toEqual({ server_deleted_at: expect.any(String) }); - for (const id of ['pending', 'failed']) expect(db.prepare('SELECT server_deleted_at FROM server_deployments WHERE id = ?').get(id)).toEqual({ server_deleted_at: null }); - expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'interrupted-create'").get()).toEqual({ server_deleted_at: expect.any(String) }); - expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'superseded'").get()).toEqual({ state: 'failed', server_deleted_at: expect.any(String) }); - expect(db.prepare("SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'gone'").get()).toEqual({ state: 'cancelled', server_deleted_at: expect.any(String) }); - expect(db.prepare("SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'tombstone'").get()).toEqual({ state: 'done', server_deleted_at: expect.any(String) }); - const collection = db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get() as { retained_server_deletions: string }; - expect(JSON.parse(collection.retained_server_deletions)).toEqual({ tombstone: expect.any(String), gone: expect.any(String) }); + expect( + db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'old'").get() + ).toEqual({ server_deleted_at: expect.any(String) }); + for (const id of ['pending', 'failed']) + expect( + db.prepare('SELECT server_deleted_at FROM server_deployments WHERE id = ?').get(id) + ).toEqual({ server_deleted_at: null }); + expect( + db + .prepare( + "SELECT server_deleted_at FROM server_deployments WHERE id = 'interrupted-create'" + ) + .get() + ).toEqual({ server_deleted_at: expect.any(String) }); + expect( + db + .prepare( + "SELECT state, server_deleted_at FROM server_deployments WHERE id = 'superseded'" + ) + .get() + ).toEqual({ state: 'failed', server_deleted_at: expect.any(String) }); + expect( + db + .prepare( + "SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'gone'" + ) + .get() + ).toEqual({ state: 'cancelled', server_deleted_at: expect.any(String) }); + expect( + db + .prepare( + "SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'tombstone'" + ) + .get() + ).toEqual({ state: 'done', server_deleted_at: expect.any(String) }); + const collection = db + .prepare('SELECT retained_server_deletions FROM server_launch_collections') + .get() as { retained_server_deletions: string }; + expect(JSON.parse(collection.retained_server_deletions)).toEqual({ + tombstone: expect.any(String), + gone: expect.any(String) + }); }); }); diff --git a/tests/unit/server-history-views.test.ts b/tests/unit/server-history-views.test.ts index c7329f4..f88aedb 100644 --- a/tests/unit/server-history-views.test.ts +++ b/tests/unit/server-history-views.test.ts @@ -16,43 +16,105 @@ afterEach(async () => fixture.cleanup()); it('exposes retained campaign and provisioning history without rebinding links to a reused name', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); - const { beginDeployment, finishDeployment, getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js'); + const { beginDeployment, finishDeployment, getDeployment, listDeployments } = + await import('../../src/lib/server/provisioning/history.js'); const operation = beginDeployment(server, 'restart'); finishDeployment(operation, { ok: true }); - db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('history-campaign', 'promote', ?, ?)").run(at, at); - db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('history-campaign', ?, 1, 'done')").run(server); - db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before_hash, content_after_hash) VALUES ('history-campaign', ?, 'updated', 'script.gsc', ?, ?)") - .run(server, putFileBlob(Buffer.from('before'), db()), putFileBlob(Buffer.from('after'), db())); + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('history-campaign', 'promote', ?, ?)" + ) + .run(at, at); + db() + .prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('history-campaign', ?, 1, 'done')" + ) + .run(server); + db() + .prepare( + "INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before_hash, content_after_hash) VALUES ('history-campaign', ?, 'updated', 'script.gsc', ?, ?)" + ) + .run( + server, + putFileBlob(Buffer.from('before'), db()), + putFileBlob(Buffer.from('after'), db()) + ); db().prepare('DELETE FROM servers WHERE name = ?').run(server); await seedServerRegistry(); closeDb(); const { GET: campaignGET } = await import('../../src/routes/api/campaigns/[id]/+server.js'); const detail = await (await campaignGET({ params: { id: 'history-campaign' } })).json(); - expect(detail.servers).toEqual([expect.objectContaining({ server, restart_state: 'done', server_deleted_at: expect.any(String) })]); - expect(detail.files).toEqual([expect.objectContaining({ server, rel_path: 'script.gsc', size_before: 6, size_after: 5 })]); + expect(detail.servers).toEqual([ + expect.objectContaining({ + server, + restart_state: 'done', + server_deleted_at: expect.any(String) + }) + ]); + expect(detail.files).toEqual([ + expect.objectContaining({ server, rel_path: 'script.gsc', size_before: 6, size_after: 5 }) + ]); const { GET: listGET } = await import('../../src/routes/api/campaigns/+server.js'); const list = await (await listGET({ url: new URL('http://localhost/api/campaigns') })).json(); expect(list.deployments[0].servers[0].server_deleted_at).toEqual(expect.any(String)); - expect(getDeployment(operation)?.deployment).toMatchObject({ server, state: 'done', server_exists: 0, server_deleted_at: expect.any(String) }); - expect(listDeployments()).toContainEqual(expect.objectContaining({ id: operation, server_exists: 0 })); + expect(getDeployment(operation)?.deployment).toMatchObject({ + server, + state: 'done', + server_exists: 0, + server_deleted_at: expect.any(String) + }); + expect(listDeployments()).toContainEqual( + expect.objectContaining({ id: operation, server_exists: 0 }) + ); const replacement = beginDeployment(server, 'restart'); - expect(getDeployment(replacement)?.deployment).toMatchObject({ server_exists: 1, server_deleted_at: null }); + expect(getDeployment(replacement)?.deployment).toMatchObject({ + server_exists: 1, + server_deleted_at: null + }); }); it('rejects historical maintenance retries and allows a new incident for a reused server name', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); - const job = db().prepare("INSERT INTO maintenance_jobs (server, requested_at, state, detail) VALUES (?, ?, 'failed', 'Original result')").run(server, at).lastInsertRowid; - db().prepare("INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('old-incident', ?, 'health', 'attention', 'Original finding', ?, ?)") + const job = db() + .prepare( + "INSERT INTO maintenance_jobs (server, requested_at, state, detail) VALUES (?, ?, 'failed', 'Original result')" + ) + .run(server, at).lastInsertRowid; + db() + .prepare( + "INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('old-incident', ?, 'health', 'attention', 'Original finding', ?, ?)" + ) .run(server, at, at); db().prepare('DELETE FROM servers WHERE name = ?').run(server); await seedServerRegistry(); - const { retryMaintenance, plutoniumStatus } = await import('../../src/lib/server/maintenance/runtime.js'); - const { updateWatchdogIncident, watchdogDetail } = await import('../../src/lib/server/maintenance/recovery.js'); + const { retryMaintenance, plutoniumStatus } = + await import('../../src/lib/server/maintenance/runtime.js'); + const { updateWatchdogIncident, watchdogDetail } = + await import('../../src/lib/server/maintenance/recovery.js'); expect(() => retryMaintenance(Number(job))).toThrow('deleted'); - expect(plutoniumStatus().jobs).toContainEqual(expect.objectContaining({ id: Number(job), state: 'failed', detail: 'Original result', server_deleted_at: expect.any(String) })); - expect(updateWatchdogIncident('old-incident', 'retry')).toMatchObject({ ok: false, status: 409 }); - expect(watchdogDetail('old-incident')?.incident).toMatchObject({ state: 'attention', summary: 'Original finding', server_deleted_at: expect.any(String) }); - expect(() => db().prepare("INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('new-incident', ?, 'health', 'open', 'New finding', ?, ?)") - .run(server, at, at)).not.toThrow(); + expect(plutoniumStatus().jobs).toContainEqual( + expect.objectContaining({ + id: Number(job), + state: 'failed', + detail: 'Original result', + server_deleted_at: expect.any(String) + }) + ); + expect(updateWatchdogIncident('old-incident', 'retry')).toMatchObject({ + ok: false, + status: 409 + }); + expect(watchdogDetail('old-incident')?.incident).toMatchObject({ + state: 'attention', + summary: 'Original finding', + server_deleted_at: expect.any(String) + }); + expect(() => + db() + .prepare( + "INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('new-incident', ?, 'health', 'open', 'New finding', ?, ?)" + ) + .run(server, at, at) + ).not.toThrow(); }); diff --git a/tests/unit/server-key-migration.test.ts b/tests/unit/server-key-migration.test.ts index 518353a..aabf725 100644 --- a/tests/unit/server-key-migration.test.ts +++ b/tests/unit/server-key-migration.test.ts @@ -4,8 +4,13 @@ import schema from '../../src/lib/server/db/schema.sql?raw'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); it('migrates idle, occupied and duplicate legacy keys without releasing unresolved assignments', async () => { const previous = new Database(fixture.dbPath); @@ -24,7 +29,10 @@ it('migrates idle, occupied and duplicate legacy keys without releasing unresolv ['t5_zombie_moon_2', 'idle-key'], ['t6_zombie_town_1', 't6-key'] ]; - for (const [server, credential] of cases) previous.prepare('INSERT INTO server_keys(server_id,pluto_key,updated_at) VALUES(?,?,?)').run(server, credential, '2026-01-01'); + for (const [server, credential] of cases) + previous + .prepare('INSERT INTO server_keys(server_id,pluto_key,updated_at) VALUES(?,?,?)') + .run(server, credential, '2026-01-01'); previous.exec(`INSERT INTO servers(name,game,desired_state) VALUES('t5_zombie_theater_1','t5','running'),('t5_zombie_pentagon_1','t5','stopped'); INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('native','t5_zombie_cosmodrome_1','host','library','image',28960,'2026-01-01'); @@ -47,7 +55,7 @@ it('migrates idle, occupied and duplicate legacy keys without releasing unresolv expect(db().pragma('foreign_key_check')).toEqual([]); closeDb(); } - const duplicate = pool.listServerKeys().find(key => key.assigned_servers.length === 2)!; + const duplicate = pool.listServerKeys().find((key) => key.assigned_servers.length === 2)!; expect(duplicate.assigned_servers).toEqual(['t5_zombie_pentagon_1', 't5_zombie_theater_1']); expect(duplicate.updated_at).toBe('2026-01-01'); expect(pool.reserveServerKey('t5_zombie_theater_2', 't5')).toBe('idle-key'); @@ -75,18 +83,54 @@ it('recovers actual launch credentials and holds both sides of an unresolved leg ('kino','t5_zombie_theater_1','host',2,'library','image',28960,'2026-01-01'), ('five','t5_zombie_pentagon_1','host',1,'library','image',28961,'2026-01-01'), ('queued','t5_zombie_cosmodrome_1','host',1,'library','image',28962,'2026-01-01');`); - function operation(id: string, instance: string, generation: number, kind: string, state: string, key: string, staged = false, nextGeneration?: number) { - const payload = JSON.stringify({ game: 't5', env: { PLUTO_SERVER_KEY: key }, nextGeneration }); - previous.prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,state,payload,created_at,updated_at) - VALUES(?,'host',?,?,?,?,?,'2026-01-01','2026-01-01')`).run(id, instance, generation, kind, state, staged ? '{}' : payload); - if (staged) previous.prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)').run(id, payload); + function operation( + id: string, + instance: string, + generation: number, + kind: string, + state: string, + key: string, + staged = false, + nextGeneration?: number + ) { + const payload = JSON.stringify({ + game: 't5', + env: { PLUTO_SERVER_KEY: key }, + nextGeneration + }); + previous + .prepare( + `INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,state,payload,created_at,updated_at) + VALUES(?,'host',?,?,?,?,?,'2026-01-01','2026-01-01')` + ) + .run(id, instance, generation, kind, state, staged ? '{}' : payload); + if (staged) + previous + .prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)') + .run(id, payload); } operation('obsolete-kino', 'kino', 1, 'instance.create', 'completed', 'obsolete-key'); operation('actual-kino', 'kino', 1, 'instance.rebuild', 'completed', 'actual-key', true, 2); operation('original-five', 'five', 1, 'instance.create', 'completed', 'original-key'); operation('unknown-five', 'five', 1, 'instance.rebuild', 'unknown', 'pending-key', true, 2); - operation('unknown-creation', 'queued', 1, 'instance.create', 'unknown', 'unknown-creation-key'); - operation('failed-new-key', 'kino', 2, 'instance.rebuild', 'failed', 'never-applied-key', true, 3); + operation( + 'unknown-creation', + 'queued', + 1, + 'instance.create', + 'unknown', + 'unknown-creation-key' + ); + operation( + 'failed-new-key', + 'kino', + 2, + 'instance.rebuild', + 'failed', + 'never-applied-key', + true, + 3 + ); previous.close(); const pool = await import('../../src/lib/server/fleet/server-keys.js'); @@ -95,11 +139,21 @@ it('recovers actual launch credentials and holds both sides of an unresolved leg expect(pool.getServerKey('t5_zombie_cosmodrome_1')).toBe('unknown-creation-key'); expect(pool.serverKeyPool('t5')).toEqual({ total: 5, available: 1 }); expect(pool.serverKeyPool('t5', 't5_zombie_pentagon_1')).toEqual({ total: 5, available: 2 }); - expect(() => pool.reserveServerKey('t5_zombie_moon_2', 't5', 'original-key')).toThrow('already reserved'); + expect(() => pool.reserveServerKey('t5_zombie_moon_2', 't5', 'original-key')).toThrow( + 'already reserved' + ); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare(`SELECT k.pluto_key,l.is_current FROM server_key_leases l JOIN server_keys k ON k.id=l.key_id - WHERE l.server_id='t5_zombie_pentagon_1' ORDER BY l.is_current`).all()) - .toEqual([{ pluto_key: 'original-key', is_current: 0 }, { pluto_key: 'pending-key', is_current: 1 }]); + expect( + db() + .prepare( + `SELECT k.pluto_key,l.is_current FROM server_key_leases l JOIN server_keys k ON k.id=l.key_id + WHERE l.server_id='t5_zombie_pentagon_1' ORDER BY l.is_current` + ) + .all() + ).toEqual([ + { pluto_key: 'original-key', is_current: 0 }, + { pluto_key: 'pending-key', is_current: 1 } + ]); expect(db().pragma('foreign_key_check')).toEqual([]); closeDb(); expect(pool.serverKeyPool('t5')).toEqual({ total: 5, available: 1 }); diff --git a/tests/unit/server-key-pool.test.ts b/tests/unit/server-key-pool.test.ts index 24b7776..c94aa6a 100644 --- a/tests/unit/server-key-pool.test.ts +++ b/tests/unit/server-key-pool.test.ts @@ -2,14 +2,21 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); const kino = 't5_zombie_theater_1'; const five = 't5_zombie_pentagon_1'; const town = 't6_zombie_town_1'; -async function keys() { return import('../../src/lib/server/fleet/server-keys.js'); } +async function keys() { + return import('../../src/lib/server/fleet/server-keys.js'); +} describe('per-game key pools', () => { it('shares keys across maps only after their previous reservation is released', async () => { @@ -26,7 +33,10 @@ describe('per-game key pools', () => { expect(pool.getServerKey(five)).toBeUndefined(); expect(pool.releaseServerKey(kino)).toBe(true); expect(pool.reserveServerKey(five, 't5')).toBe('secret-t5'); - expect(pool.listServerKeys().find(key => key.id === id)).toMatchObject({ label: 'Shared T5 key', assigned_servers: [five] }); + expect(pool.listServerKeys().find((key) => key.id === id)).toMatchObject({ + label: 'Shared T5 key', + assigned_servers: [five] + }); pool.releaseServerKey(five); expect(pool.removeServerKey(id)).toBe(true); expect(pool.removeServerKey(id)).toBe(false); @@ -38,9 +48,22 @@ describe('per-game key pools', () => { expect(pool.saveServerKey('t5', 'secret', 'Renamed')).toBe(id); expect(pool.saveServerKey('t5', 'secret')).toBe(id); expect(pool.serverKeyPool('t5')).toEqual({ total: 1, available: 1 }); - expect(pool.listServerKeys()).toEqual([{ id, game: 't5', label: 'Renamed', updated_at: expect.any(String), assigned_servers: [] }]); + expect(pool.listServerKeys()).toEqual([ + { + id, + game: 't5', + label: 'Renamed', + updated_at: expect.any(String), + assigned_servers: [] + } + ]); expect(JSON.stringify(pool.listServerKeys())).not.toContain('secret'); - for (const [game, credential, label] of [['t7', 'valid', ''], ['t5', 'two words', ''], ['t5', '', ''], ['t5', 'valid', 'x'.repeat(161)]]) { + for (const [game, credential, label] of [ + ['t7', 'valid', ''], + ['t5', 'two words', ''], + ['t5', '', ''], + ['t5', 'valid', 'x'.repeat(161)] + ]) { expect(() => pool.saveServerKey(game, credential, label)).toThrow(); } }); @@ -52,7 +75,9 @@ describe('per-game key pools', () => { expect(() => pool.reserveServerKey(five, 't5', 'manual-key')).toThrow('already reserved'); expect(pool.getServerKey(five)).toBeUndefined(); expect(pool.availableServerKeyCount('t5')).toBe(1); - expect(() => pool.reserveServerKey(kino, 't5', 'replacement')).toThrow('already has a reserved key'); + expect(() => pool.reserveServerKey(kino, 't5', 'replacement')).toThrow( + 'already has a reserved key' + ); expect(() => pool.reserveServerKey(kino, 't6')).toThrow('another game'); expect(pool.reserveServerKey(five, 't5', '')).toBe('spare-key'); }); @@ -60,10 +85,12 @@ describe('per-game key pools', () => { it('rolls back reservation and imported credentials when launch admission fails', async () => { const pool = await keys(); const { db } = await import('../../src/lib/server/db/index.js'); - expect(() => db().transaction(() => { - pool.reserveServerKey(kino, 't5', 'temporary-key'); - throw new Error('Launch validation failed'); - })()).toThrow('Launch validation failed'); + expect(() => + db().transaction(() => { + pool.reserveServerKey(kino, 't5', 'temporary-key'); + throw new Error('Launch validation failed'); + })() + ).toThrow('Launch validation failed'); expect(pool.listServerKeys()).toEqual([]); expect(pool.getServerKey(kino)).toBeUndefined(); }); @@ -72,7 +99,9 @@ describe('per-game key pools', () => { const pool = await keys(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); pool.reserveServerKey(kino, 't5', 'retained-key'); - db().prepare("INSERT INTO servers(name,game,desired_state) VALUES(?,'t5','stopped')").run(kino); + db() + .prepare("INSERT INTO servers(name,game,desired_state) VALUES(?,'t5','stopped')") + .run(kino); closeDb(); expect(pool.getServerKey(kino)).toBe('retained-key'); expect(pool.availableServerKeyCount('t5')).toBe(0); @@ -85,7 +114,9 @@ describe('per-game key pools', () => { pool.reserveServerKey(kino, 't5', 'same-secret'); const duplicate = pool.saveServerKey('t6', 'same-secret'); expect(pool.serverKeyPool('t6')).toEqual({ total: 1, available: 0 }); - expect(pool.listServerKeys().find(key => key.id === duplicate)?.assigned_servers).toEqual([kino]); + expect(pool.listServerKeys().find((key) => key.id === duplicate)?.assigned_servers).toEqual( + [kino] + ); expect(() => pool.removeServerKey(duplicate)).toThrow('reserved'); expect(() => pool.reserveServerKey(town, 't6')).toThrow('No available T6'); }); diff --git a/tests/unit/server-keys-api.test.ts b/tests/unit/server-keys-api.test.ts index 4c80b05..defedc4 100644 --- a/tests/unit/server-keys-api.test.ts +++ b/tests/unit/server-keys-api.test.ts @@ -2,20 +2,36 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); function request(method: string, body: unknown) { - return { request: new Request('http://localhost/api/server-keys', { - method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) - }) }; + return { + request: new Request('http://localhost/api/server-keys', { + method, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + }; } describe('game key pool API', () => { it('returns pool metadata and assigned servers without credentials', async () => { const route = await import('../../src/routes/api/server-keys/+server.js'); const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const saved = await route.PUT(request('PUT', { game: 't5', label: 'Cherry hostname 5', plutoKey: 'private-t5-credential' })); + const saved = await route.PUT( + request('PUT', { + game: 't5', + label: 'Cherry hostname 5', + plutoKey: 'private-t5-credential' + }) + ); expect(saved.status).toBe(200); const { id } = await saved.json(); reserveServerKey('t5_zombie_theater_2', 't5'); @@ -25,9 +41,17 @@ describe('game key pool API', () => { expect(text).not.toContain('private-t5-credential'); expect(text).not.toContain('pluto_key'); expect(text).not.toContain('plutoKey'); - expect(JSON.parse(text)).toEqual({ keys: [{ - id, game: 't5', label: 'Cherry hostname 5', assigned_servers: ['t5_zombie_theater_2'], updated_at: expect.any(String) - }] }); + expect(JSON.parse(text)).toEqual({ + keys: [ + { + id, + game: 't5', + label: 'Cherry hostname 5', + assigned_servers: ['t5_zombie_theater_2'], + updated_at: expect.any(String) + } + ] + }); }); it.each([ @@ -35,7 +59,7 @@ describe('game key pool API', () => { { serverId: 't5_zombie_theater_2', plutoKey: 'private-key' }, { game: 't5', plutoKey: 'contains whitespace' }, { game: 't5', plutoKey: 'private-key', label: 'x'.repeat(161) } - ])('rejects invalid pool input without saving it: %j', async body => { + ])('rejects invalid pool input without saving it: %j', async (body) => { const route = await import('../../src/routes/api/server-keys/+server.js'); const response = await route.PUT(request('PUT', body)); expect(response.status).toBe(400); @@ -45,23 +69,35 @@ describe('game key pool API', () => { it('deduplicates a credential in its game pool', async () => { const route = await import('../../src/routes/api/server-keys/+server.js'); - const first = await route.PUT(request('PUT', { game: 't6', plutoKey: 'private-key', label: 'First label' })); - const repeated = await route.PUT(request('PUT', { game: 't6', plutoKey: ' private-key ', label: 'Updated label' })); + const first = await route.PUT( + request('PUT', { game: 't6', plutoKey: 'private-key', label: 'First label' }) + ); + const repeated = await route.PUT( + request('PUT', { game: 't6', plutoKey: ' private-key ', label: 'Updated label' }) + ); expect(await repeated.json()).toEqual(await first.json()); - expect((await route.GET().json()).keys).toEqual([expect.objectContaining({ - game: 't6', label: 'Updated label', assigned_servers: [] - })]); + expect((await route.GET().json()).keys).toEqual([ + expect.objectContaining({ + game: 't6', + label: 'Updated label', + assigned_servers: [] + }) + ]); }); it('rejects removal while assigned and permits it after the key is released', async () => { const route = await import('../../src/routes/api/server-keys/+server.js'); - const { reserveServerKey, releaseServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); + const { reserveServerKey, releaseServerKey, getServerKey } = + await import('../../src/lib/server/fleet/server-keys.js'); const saved = await route.PUT(request('PUT', { game: 't5', plutoKey: 'private-key' })); const { id } = await saved.json(); reserveServerKey('t5_zombie_moon_1', 't5'); const blocked = await route.DELETE(request('DELETE', { id })); expect(blocked.status).toBe(400); - expect(await blocked.json()).toMatchObject({ ok: false, error: expect.stringMatching(/reserved|assigned|in use/i) }); + expect(await blocked.json()).toMatchObject({ + ok: false, + error: expect.stringMatching(/reserved|assigned|in use/i) + }); expect(getServerKey('t5_zombie_moon_1')).toBe('private-key'); releaseServerKey('t5_zombie_moon_1'); expect(await (await route.DELETE(request('DELETE', { id }))).json()).toEqual({ ok: true }); diff --git a/tests/unit/server-observation.test.ts b/tests/unit/server-observation.test.ts index 78f0baf..e5ed175 100644 --- a/tests/unit/server-observation.test.ts +++ b/tests/unit/server-observation.test.ts @@ -1,14 +1,31 @@ import { describe, expect, it } from 'vitest'; -import { freshTelemetry, selectServerObservation, type ServerObservation } from '../../src/lib/server-observation.js'; +import { + freshTelemetry, + selectServerObservation, + type ServerObservation +} from '../../src/lib/server-observation.js'; import { gameEvent } from '../helpers/telemetry.js'; const now = Date.parse('2026-09-27T12:00:00Z'); const at = (offset = 0) => new Date(now + offset).toISOString(); -const report = (offset = 0, players = 3) => ({ ...gameEvent('heartbeat', 15, players, 'managed'), receivedAt: at(offset) }); +const report = (offset = 0, players = 3) => ({ + ...gameEvent('heartbeat', 15, players, 'managed'), + receivedAt: at(offset) +}); function server(changes: Partial = {}): ServerObservation { - return { name: 'managed', health: 'online', container_state: 'running', container_checked_at: at(), - last_players: 4, last_map: 'zombie_theater', last_gametype: 'zom', last_seen_at: at(), - last_error: null, roster: [], ...changes }; + return { + name: 'managed', + health: 'online', + container_state: 'running', + container_checked_at: at(), + last_players: 4, + last_map: 'zombie_theater', + last_gametype: 'zom', + last_seen_at: at(), + last_error: null, + roster: [], + ...changes + }; } describe('shared server observation', () => { @@ -16,22 +33,52 @@ describe('shared server observation', () => { for (const reason of ['timeout', 'send-error']) { const raw = { ...server(), last_error: `${reason}:no reply`, rcon_failures: 1 }; const selected = selectServerObservation(server({ rcon_status: raw }), undefined, now); - expect(selected).toMatchObject({ health: 'online', last_error: null, rcon_status: raw }); - expect(selectServerObservation(selected, undefined, now + 90_000).health).toBe('online'); + expect(selected).toMatchObject({ + health: 'online', + last_error: null, + rcon_status: raw + }); + expect(selectServerObservation(selected, undefined, now + 90_000).health).toBe( + 'online' + ); expect(selectServerObservation(selected, undefined, now + 90_001).health).toBe('stale'); - expect(selectServerObservation(selected, undefined, now + 120_001).health).toBe('offline'); - expect(selectServerObservation(server({ rcon_status: { ...raw, rcon_failures: 2 } }), undefined, now)) - .toMatchObject({ health: 'stale', last_error: raw.last_error }); + expect(selectServerObservation(selected, undefined, now + 120_001).health).toBe( + 'offline' + ); + expect( + selectServerObservation( + server({ rcon_status: { ...raw, rcon_failures: 2 } }), + undefined, + now + ) + ).toMatchObject({ health: 'stale', last_error: raw.last_error }); } for (const reason of ['no-config', 'no-port', 'bad-reply', 'no-players']) { - expect(selectServerObservation(server({ last_error: `${reason}:failed`, rcon_failures: 1 }), undefined, now).health).toBe('stale'); + expect( + selectServerObservation( + server({ last_error: `${reason}:failed`, rcon_failures: 1 }), + undefined, + now + ).health + ).toBe('stale'); } - expect(selectServerObservation(server({ last_error: 'timeout:legacy failure' }), undefined, now).health).toBe('stale'); + expect( + selectServerObservation( + server({ last_error: 'timeout:legacy failure' }), + undefined, + now + ).health + ).toBe('stale'); }); it('accepts telemetry and container evidence through 90 seconds, then expires each independently', () => { - expect(selectServerObservation(server(), report(), now + 90_000)).toMatchObject({ health: 'online', status_source: 'telemetry' }); - expect(selectServerObservation(server(), report(90_001), now + 90_001).health).toBe('stale'); + expect(selectServerObservation(server(), report(), now + 90_000)).toMatchObject({ + health: 'online', + status_source: 'telemetry' + }); + expect(selectServerObservation(server(), report(90_001), now + 90_001).health).toBe( + 'stale' + ); const current = server({ container_checked_at: at(90_001) }); expect(selectServerObservation(current, report(), now + 90_001).health).toBe('stale'); expect(freshTelemetry(report(-90_000), now)).toBe(true); @@ -41,7 +88,12 @@ describe('shared server observation', () => { it('selects the telemetry count and roster together and keeps raw RCON evidence', () => { const raw = server({ last_error: 'timeout' }); const selected = selectServerObservation(server({ rcon_status: raw }), report(), now); - expect(selected).toMatchObject({ health: 'online', status_source: 'telemetry', last_players: 3, last_error: null }); + expect(selected).toMatchObject({ + health: 'online', + status_source: 'telemetry', + last_players: 3, + last_error: null + }); expect(selected.roster).toHaveLength(3); expect(selected.roster?.[0]).toMatchObject({ ping: null, score: 5000 }); expect(selected.rcon_status).toBe(raw); @@ -49,157 +101,333 @@ describe('shared server observation', () => { it('supplements telemetry members with GUID-matched pings without changing scores or names', () => { const telemetry = report(); - const raw = server({ roster: [ - { ...telemetry.players[1], num: 2, score: 10, ping: 0, name: 'Old name' }, - { ...telemetry.players[0], num: 3, score: 20, ping: 999 }, - { ...telemetry.players[2], num: 4, score: 30, ping: 45, guid: 'different-player', name: 'Different player' } - ] }); + const raw = server({ + roster: [ + { ...telemetry.players[1], num: 2, score: 10, ping: 0, name: 'Old name' }, + { ...telemetry.players[0], num: 3, score: 20, ping: 999 }, + { + ...telemetry.players[2], + num: 4, + score: 30, + ping: 45, + guid: 'different-player', + name: 'Different player' + } + ] + }); const selected = selectServerObservation(server({ rcon_status: raw }), telemetry, now); - expect(selected.roster).toEqual(telemetry.players.map((player, i) => ({ - name: player.name, guid: player.guid, num: null, score: 5000, ping: [999, 0, null][i] - }))); + expect(selected.roster).toEqual( + telemetry.players.map((player, i) => ({ + name: player.name, + guid: player.guid, + num: null, + score: 5000, + ping: [999, 0, null][i] + })) + ); expect(selected.last_players).toBe(3); expect(selected.rcon_status).toBe(raw); // A new telemetry report must not renew the RCON sample's age. - const expired = selectServerObservation({ ...selected, container_checked_at: at(90_001) }, report(90_001), now + 90_001); + const expired = selectServerObservation( + { ...selected, container_checked_at: at(90_001) }, + report(90_001), + now + 90_001 + ); expect(expired.health).toBe('online'); - expect(expired.roster?.map(player => player.ping)).toEqual([null, null, null]); + expect(expired.roster?.map((player) => player.ping)).toEqual([null, null, null]); }); it('prefers telemetry ping, including zero, and fills absent or null readings from RCON', () => { const telemetry = report(); telemetry.players[0].ping = 0; telemetry.players[1].ping = null; - const raw = server({ roster: telemetry.players.map(player => ({ ...player, num: 0, score: 10, ping: 42 })) }); - expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.map(player => player.ping)).toEqual([0, 42, 42]); + const raw = server({ + roster: telemetry.players.map((player) => ({ ...player, num: 0, score: 10, ping: 42 })) + }); + expect( + selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.map( + (player) => player.ping + ) + ).toEqual([0, 42, 42]); }); it.each([ - { last_seen_at: at(-90_001) }, { last_seen_at: at(2000) }, { last_seen_at: null }, - { last_error: 'timeout', rcon_failures: 2 }, { last_map: 'other-map' } - ])('does not supplement pings from unusable RCON evidence: %j', changes => { + { last_seen_at: at(-90_001) }, + { last_seen_at: at(2000) }, + { last_seen_at: null }, + { last_error: 'timeout', rcon_failures: 2 }, + { last_map: 'other-map' } + ])('does not supplement pings from unusable RCON evidence: %j', (changes) => { const telemetry = report(); - const raw = server({ ...changes, roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] }); - expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBeNull(); + const raw = server({ + ...changes, + roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] + }); + expect( + selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping + ).toBeNull(); }); it('requires RCON samples after the latest lifecycle and tolerates one transient failure', () => { const telemetry = report(); - const raw = server({ last_seen_at: at(-90_000), roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] }); - expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBe(42); - expect(selectServerObservation(server({ rcon_status: raw, telemetry_after: at(-90_000) }), telemetry, now).roster?.[0].ping).toBeNull(); - expect(selectServerObservation(server({ rcon_status: { ...raw, last_error: 'timeout', rcon_failures: 1 } }), telemetry, now).roster?.[0].ping).toBe(42); + const raw = server({ + last_seen_at: at(-90_000), + roster: [{ ...telemetry.players[0], num: 0, score: 100, ping: 42 }] + }); + expect( + selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping + ).toBe(42); + expect( + selectServerObservation( + server({ rcon_status: raw, telemetry_after: at(-90_000) }), + telemetry, + now + ).roster?.[0].ping + ).toBeNull(); + expect( + selectServerObservation( + server({ rcon_status: { ...raw, last_error: 'timeout', rcon_failures: 1 } }), + telemetry, + now + ).roster?.[0].ping + ).toBe(42); }); - it.each(['duplicate', 'null', 'negative', 'invalid'])('rejects %s ping matches', mode => { + it.each(['duplicate', 'null', 'negative', 'invalid'])('rejects %s ping matches', (mode) => { const telemetry = report(); - const player = { ...telemetry.players[0], num: 0, score: 100, ping: 42 as number | null, guid: telemetry.players[0].guid as string | null }; + const player = { + ...telemetry.players[0], + num: 0, + score: 100, + ping: 42 as number | null, + guid: telemetry.players[0].guid as string | null + }; if (mode === 'null') player.ping = null; if (mode === 'negative') player.ping = -1; if (mode === 'invalid') player.ping = NaN; - const raw = server({ roster: mode === 'duplicate' ? [player, { ...player, ping: 50 }] : [player] }); - expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping).toBeNull(); + const raw = server({ + roster: mode === 'duplicate' ? [player, { ...player, ping: 50 }] : [player] + }); + expect( + selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.[0].ping + ).toBeNull(); }); - it.each(['t5', 't6'])('does not match %s players by name when GUIDs differ or are missing', game => { - const telemetry = report(0, 2); - telemetry.players[0].name = 'EyelessGreem'; - telemetry.players[1].name = '_frxn.3'; - const raw = server({ roster: [ - { num: 0, score: 940, ping: 91, guid: '6C4C20', name: 'EyelessGreem' }, - { num: 1, score: 150, ping: 151, guid: null, name: '_frxn.3' } - ] }); - const selected = selectServerObservation(server({ game, rcon_status: raw }), telemetry, now); - expect(selected.roster?.map(player => player.ping)).toEqual([null, null]); - }); + it.each(['t5', 't6'])( + 'does not match %s players by name when GUIDs differ or are missing', + (game) => { + const telemetry = report(0, 2); + telemetry.players[0].name = 'EyelessGreem'; + telemetry.players[1].name = '_frxn.3'; + const raw = server({ + roster: [ + { num: 0, score: 940, ping: 91, guid: '6C4C20', name: 'EyelessGreem' }, + { num: 1, score: 150, ping: 151, guid: null, name: '_frxn.3' } + ] + }); + const selected = selectServerObservation( + server({ game, rcon_status: raw }), + telemetry, + now + ); + expect(selected.roster?.map((player) => player.ping)).toEqual([null, null]); + } + ); it.each([ - ['6C4C20', '7097376'], ['6c45dd', '7095773'], ['00006C4C20', '0007097376'], - ['0x6C45DD', '7095773'], ['10', '16'], ['20000000000001', '9007199254740993'] + ['6C4C20', '7097376'], + ['6c45dd', '7095773'], + ['00006C4C20', '0007097376'], + ['0x6C45DD', '7095773'], + ['10', '16'], + ['20000000000001', '9007199254740993'] ])('matches T6 hex GUID %s to decimal %s despite different names', (hex, decimal) => { const telemetry = report(0, 1); telemetry.players[0].guid = decimal; - const raw = server({ roster: [{ num: 0, score: 100, ping: 91, guid: hex, name: 'Previous name' }] }); - const selected = selectServerObservation(server({ game: 't6', rcon_status: raw }), telemetry, now); - expect(selected.roster?.[0]).toMatchObject({ guid: decimal, name: 'Player 1', ping: 91, score: 5000 }); + const raw = server({ + roster: [{ num: 0, score: 100, ping: 91, guid: hex, name: 'Previous name' }] + }); + const selected = selectServerObservation( + server({ game: 't6', rcon_status: raw }), + telemetry, + now + ); + expect(selected.roster?.[0]).toMatchObject({ + guid: decimal, + name: 'Player 1', + ping: 91, + score: 5000 + }); expect(selected.rcon_status?.roster?.[0].guid).toBe(hex); }); - it.each(['t5', undefined])('keeps decimal-looking GUIDs literal for game %s', game => { + it.each(['t5', undefined])('keeps decimal-looking GUIDs literal for game %s', (game) => { const telemetry = report(0, 2); telemetry.players[0].guid = '10'; telemetry.players[1].guid = '16'; - const raw = server({ roster: [{ num: 0, score: 100, ping: 91, guid: '10', name: 'Previous name' }] }); - expect(selectServerObservation(server({ game, rcon_status: raw }), telemetry, now).roster?.map(player => player.ping)).toEqual([91, null]); + const raw = server({ + roster: [{ num: 0, score: 100, ping: 91, guid: '10', name: 'Previous name' }] + }); + expect( + selectServerObservation(server({ game, rcon_status: raw }), telemetry, now).roster?.map( + (player) => player.ping + ) + ).toEqual([91, null]); }); it('treats digit-only T6 GUIDs as hexadecimal regardless of player names', () => { const telemetry = report(0, 2); telemetry.players[0].guid = '10'; telemetry.players[1].guid = '16'; - const raw = server({ roster: [{ num: 0, score: 100, ping: 91, guid: '10', name: 'Player 1' }] }); - expect(selectServerObservation(server({ game: 't6', rcon_status: raw }), telemetry, now).roster?.map(player => player.ping)).toEqual([null, 91]); + const raw = server({ + roster: [{ num: 0, score: 100, ping: 91, guid: '10', name: 'Player 1' }] + }); + expect( + selectServerObservation( + server({ game: 't6', rcon_status: raw }), + telemetry, + now + ).roster?.map((player) => player.ping) + ).toEqual([null, 91]); }); it('rejects duplicate RCON GUIDs after normalization', () => { const telemetry = report(0, 1); telemetry.players[0].guid = '7097376'; - const raw = server({ roster: [ - { num: 0, score: 100, ping: 91, guid: '6C4C20', name: 'Player 1' }, - { num: 1, score: 100, ping: 42, guid: '006c4c20', name: 'Other player' } - ] }); - expect(selectServerObservation(server({ game: 't6', rcon_status: raw }), telemetry, now).roster?.[0].ping).toBeNull(); + const raw = server({ + roster: [ + { num: 0, score: 100, ping: 91, guid: '6C4C20', name: 'Player 1' }, + { num: 1, score: 100, ping: 42, guid: '006c4c20', name: 'Other player' } + ] + }); + expect( + selectServerObservation(server({ game: 't6', rcon_status: raw }), telemetry, now) + .roster?.[0].ping + ).toBeNull(); }); it('matches GUIDs regardless of conflicting names', () => { const telemetry = report(0, 2); - const raw = server({ roster: [ - { num: 0, score: 100, ping: 42, guid: telemetry.players[1].guid, name: telemetry.players[0].name }, - { num: 1, score: 100, ping: 99, guid: 'another-guid', name: telemetry.players[1].name } - ] }); - expect(selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.map(player => player.ping)).toEqual([null, 42]); + const raw = server({ + roster: [ + { + num: 0, + score: 100, + ping: 42, + guid: telemetry.players[1].guid, + name: telemetry.players[0].name + }, + { + num: 1, + score: 100, + ping: 99, + guid: 'another-guid', + name: telemetry.players[1].name + } + ] + }); + expect( + selectServerObservation(server({ rcon_status: raw }), telemetry, now).roster?.map( + (player) => player.ping + ) + ).toEqual([null, 42]); }); it('uses the latest HTTP or streamed telemetry and falls back without extending freshness', () => { const original = server(); - const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now); + const selected = selectServerObservation( + server({ rcon_status: original, telemetry: report(0, 2) }), + report(-1000), + now + ); expect(selected.last_players).toBe(2); - const fallback = selectServerObservation({ ...selected, container_checked_at: at(91_000), - rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 91_000); - expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) }); + const fallback = selectServerObservation( + { + ...selected, + container_checked_at: at(91_000), + rcon_status: { ...original, last_seen_at: at(40_000) } + }, + undefined, + now + 91_000 + ); + expect(fallback).toMatchObject({ + status_source: 'rcon', + health: 'online', + last_players: 4, + last_seen_at: at(40_000) + }); }); it('displays telemetry pings and clears missing readings on subsequent reports', () => { const telemetry = report(0, 4); - telemetry.players = telemetry.players.map((player, i) => ({ ...player, ping: [0, 42, 999, null][i] })); + telemetry.players = telemetry.players.map((player, i) => ({ + ...player, + ping: [0, 42, 999, null][i] + })); const selected = selectServerObservation(server(), telemetry, now); - expect(selected.roster?.map(player => player.ping)).toEqual([0, 42, 999, null]); - expect(selectServerObservation(selected, report(1, 4), now + 1).roster?.map(player => player.ping)).toEqual([null, null, null, null]); - const start = { ...gameEvent('game_start', 0, 1, 'managed'), event: 'game_start' as const, receivedAt: at(), - players: [{ name: 'Player 1', guid: '123', ping: 25 }] }; - expect(selectServerObservation(server(), start, now).roster?.[0]).toMatchObject({ score: null, ping: 25 }); + expect(selected.roster?.map((player) => player.ping)).toEqual([0, 42, 999, null]); + expect( + selectServerObservation(selected, report(1, 4), now + 1).roster?.map( + (player) => player.ping + ) + ).toEqual([null, null, null, null]); + const start = { + ...gameEvent('game_start', 0, 1, 'managed'), + event: 'game_start' as const, + receivedAt: at(), + players: [{ name: 'Player 1', guid: '123', ping: 25 }] + }; + expect(selectServerObservation(server(), start, now).roster?.[0]).toMatchObject({ + score: null, + ping: 25 + }); }); - it.each(['starting', 'restarting', 'unhealthy', 'exited'])('telemetry cannot override container state %s', state => { - const selected = selectServerObservation(server({ health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', container_state: state }), report(), now); - expect(selected.health).not.toBe('online'); - expect(selected.last_players).toBeNull(); - expect(selected.roster).toBeNull(); - }); + it.each(['starting', 'restarting', 'unhealthy', 'exited'])( + 'telemetry cannot override container state %s', + (state) => { + const selected = selectServerObservation( + server({ + health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', + container_state: state + }), + report(), + now + ); + expect(selected.health).not.toBe('online'); + expect(selected.last_players).toBeNull(); + expect(selected.roster).toBeNull(); + } + ); it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => { const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) }); - expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' }); + expect(selectServerObservation(raw, report(), now)).toMatchObject({ + health: 'online', + last_players: 4, + status_source: 'rcon' + }); expect(selectServerObservation(raw, report(91_000), now + 91_000).health).toBe('stale'); expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline'); - expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline'); - expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline'); + expect( + selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health + ).toBe('offline'); + expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe( + 'offline' + ); }); it('requires fresh container evidence and reports after the latest lifecycle submission', () => { - expect(selectServerObservation(server({ container_checked_at: at(-91_000) }), report(), now).health).not.toBe('online'); - expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning'); - expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline'); + expect( + selectServerObservation(server({ container_checked_at: at(-91_000) }), report(), now) + .health + ).not.toBe('online'); + expect( + selectServerObservation(server({ health: 'provisioning' }), report(), now).health + ).toBe('provisioning'); + expect( + selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health + ).toBe('offline'); expect(freshTelemetry(report(), now, 'invalid')).toBe(false); expect(freshTelemetry(report(2000), now)).toBe(false); expect(freshTelemetry(report(-91_000), now)).toBe(false); @@ -211,7 +439,13 @@ describe('shared server observation', () => { const active = selectServerObservation(server(), start, now); expect(active.roster?.[0]).toMatchObject({ score: null, ping: null }); expect(active.game_started_at).toBe(at()); - expect(selectServerObservation({ ...active, game_ended_at: at() }, report(1, 0), now + 1)) - .toMatchObject({ last_players: 0, roster: [], game_started_at: null, game_ended_at: null }); + expect( + selectServerObservation({ ...active, game_ended_at: at() }, report(1, 0), now + 1) + ).toMatchObject({ + last_players: 0, + roster: [], + game_started_at: null, + game_ended_at: null + }); }); }); diff --git a/tests/unit/server-sections.test.ts b/tests/unit/server-sections.test.ts index ab5dcbb..1013bf9 100644 --- a/tests/unit/server-sections.test.ts +++ b/tests/unit/server-sections.test.ts @@ -3,9 +3,14 @@ import { serverSection, serverSectionAnchor } from '../../src/lib/server-section describe('server detail bookmarks', () => { it.each([ - ['scoreboard', 'live', null], ['chat', 'live', 'chat'], ['matches', 'matches', null], - ['config', 'configuration', null], ['inventory', 'configuration', 'inventory'], - ['logs', 'diagnostics', 'logs'], ['diagnostics', 'diagnostics', null], ['history', 'history', null] + ['scoreboard', 'live', null], + ['chat', 'live', 'chat'], + ['matches', 'matches', null], + ['config', 'configuration', null], + ['inventory', 'configuration', 'inventory'], + ['logs', 'diagnostics', 'logs'], + ['diagnostics', 'diagnostics', null], + ['history', 'history', null] ])('preserves the destination of tab=%s', (tab, section, anchor) => { expect(serverSection(tab)).toBe(section); expect(serverSectionAnchor(tab)).toBe(anchor); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts index d7be027..38e1384 100644 --- a/tests/unit/server-status.test.ts +++ b/tests/unit/server-status.test.ts @@ -5,7 +5,8 @@ import { join } from 'node:path'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn() + rconStatusResult: vi.fn(), + rconGetPlayerCount: vi.fn() })); let fixture: Fixture; @@ -14,17 +15,28 @@ beforeEach(async () => { vi.clearAllMocks(); fixture = await newFixture(['unregistered-folder']); }); -afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); +afterEach(async () => { + vi.useRealTimers(); + await fixture.cleanup(); +}); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers (name, last_error) VALUES ('managed', 'timeout:old process')").run(); + db() + .prepare("INSERT INTO servers (name, last_error) VALUES ('managed', 'timeout:old process')") + .run(); vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - return { db: db(), inspect: vi.mocked(nativeHealth), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer }; + return { + db: db(), + inspect: vi.mocked(nativeHealth), + rcon: vi.mocked(rconStatusResult), + fanoutStatus, + getServer + }; } describe('database fleet and container readiness', () => { @@ -34,13 +46,21 @@ describe('database fleet and container readiness', () => { const at = Date.parse('2026-10-08T12:00:00Z'); vi.setSystemTime(at); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'bad-reply', detail: 'unrecognized status', - rawReply: 'player 192.0.2.123:28960 invalid reply' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'bad-reply', + detail: 'unrecognized status', + rawReply: 'player 192.0.2.123:28960 invalid reply' + }); await fanoutStatus(); expect(getServer('managed')?.last_reply_snippet).toBe('player [redacted] invalid reply'); vi.setSystemTime(at + 30_000); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'agent', port: 1 }, - status: { players: 0, map: null, gametype: null, roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'agent', port: 1 }, + status: { players: 0, map: null, gametype: null, roster: [] } + }); await fanoutStatus(); expect(getServer('managed')?.last_reply_snippet).toBeNull(); }); @@ -52,20 +72,42 @@ describe('database fleet and container readiness', () => { const at = Date.parse('2026-10-07T12:00:00Z'); vi.setSystemTime(at); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 3, capacity: 4, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 3, capacity: 4, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); const events: unknown[] = []; - const stop = subscribe(event => { if (event.type === 'server-status') events.push(event); }); + const stop = subscribe((event) => { + if (event.type === 'server-status') events.push(event); + }); try { await fanoutStatus(); - expect(events.at(-1)).toMatchObject({ entries: [{ capacity: 4, capacityObservedAt: new Date(at).toISOString() }] }); - expect((await GET().json()).servers[0]).toMatchObject({ capacity: 4, capacity_observed_at: new Date(at).toISOString() }); + expect(events.at(-1)).toMatchObject({ + entries: [{ capacity: 4, capacityObservedAt: new Date(at).toISOString() }] + }); + expect((await GET().json()).servers[0]).toMatchObject({ + capacity: 4, + capacity_observed_at: new Date(at).toISOString() + }); vi.setSystemTime(at + 30_000); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'No reply' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'No reply' + }); await fanoutStatus(); - expect(events.at(-1)).toMatchObject({ entries: [{ capacity: null, capacityObservedAt: null }] }); - expect((await GET().json()).servers[0]).toMatchObject({ capacity: null, capacity_observed_at: null }); - } finally { stop(); } + expect(events.at(-1)).toMatchObject({ + entries: [{ capacity: null, capacityObservedAt: null }] + }); + expect((await GET().json()).servers[0]).toMatchObject({ + capacity: null, + capacity_observed_at: null + }); + } finally { + stop(); + } }); it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => { @@ -77,44 +119,87 @@ describe('database fleet and container readiness', () => { const at = Date.parse('2026-10-05T12:00:00Z'); vi.setSystemTime(at); inspect.mockResolvedValue({ ok: true, state: 'running' }); - const success = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }; + const success = { + ok: true as const, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } + }; rcon.mockResolvedValue(success); await fanoutStatus(); const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); + const unsubscribe = subscribe((event) => events.push(event)); try { - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'no reply' + }); vi.setSystemTime(at + 30_000); await fanoutStatus(); - expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:no reply' }); - expect(await snapshot()).toMatchObject({ health: 'online', last_error: null, - rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } }); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ - health: 'online', lastError: null, rconStatus: expect.objectContaining({ rcon_failures: 1, last_error: 'timeout:no reply' }) - })] })); + expect(getServer('managed')).toMatchObject({ + rcon_failures: 1, + last_error: 'timeout:no reply' + }); + expect(await snapshot()).toMatchObject({ + health: 'online', + last_error: null, + rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } + }); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: [ + expect.objectContaining({ + health: 'online', + lastError: null, + rconStatus: expect.objectContaining({ + rcon_failures: 1, + last_error: 'timeout:no reply' + }) + }) + ] + }) + ); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); - expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } }); + expect(await snapshot()).toMatchObject({ + health: 'online', + rcon_status: { rcon_failures: 1 } + }); // Reprocessing a sample with the same timestamp cannot count it twice. await fanoutStatus(); expect(getServer('managed')?.rcon_failures).toBe(1); vi.setSystemTime(at + 60_000); await fanoutStatus(); - expect(await snapshot()).toMatchObject({ health: 'stale', last_error: 'timeout:no reply', rcon_status: { rcon_failures: 2 } }); + expect(await snapshot()).toMatchObject({ + health: 'stale', + last_error: 'timeout:no reply', + rcon_status: { rcon_failures: 2 } + }); rcon.mockResolvedValue(success); vi.setSystemTime(at + 90_000); await fanoutStatus(); expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'send-error', detail: 'unreachable' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'send-error', + detail: 'unreachable' + }); vi.setSystemTime(at + 120_000); await fanoutStatus(); - expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } }); + expect(await snapshot()).toMatchObject({ + health: 'online', + rcon_status: { rcon_failures: 1 } + }); vi.setSystemTime(at + 180_001); expect(await snapshot()).toMatchObject({ health: 'stale' }); vi.setSystemTime(at + 210_001); expect(await snapshot()).toMatchObject({ health: 'offline' }); - } finally { unsubscribe(); } + } finally { + unsubscribe(); + } }); it('adds the failure counter to an existing database without suppressing legacy errors', async () => { @@ -122,7 +207,10 @@ describe('database fleet and container readiness', () => { const { closeDb } = await import('../../src/lib/server/db/index.js'); db.exec('ALTER TABLE servers DROP COLUMN rcon_failures'); closeDb(); - expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: 'timeout:old process' }); + expect(getServer('managed')).toMatchObject({ + rcon_failures: 0, + last_error: 'timeout:old process' + }); }); it('does not carry a failure streak across a new lifecycle submission', async () => { @@ -130,40 +218,67 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); const at = Date.parse('2026-10-05T12:00:00Z'); vi.setSystemTime(at); - db.prepare("UPDATE servers SET rcon_failures=1,last_checked_at=?").run(new Date(at - 30_000).toISOString()); - db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) - VALUES('new-start','managed','restart','done','done',?,?,'test')`).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString()); + db.prepare('UPDATE servers SET rcon_failures=1,last_checked_at=?').run( + new Date(at - 30_000).toISOString() + ); + db.prepare( + `INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by) + VALUES('new-start','managed','restart','done','done',?,?,'test')` + ).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString()); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'new process' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'new process' + }); await fanoutStatus(); - expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:new process' }); + expect(getServer('managed')).toMatchObject({ + rcon_failures: 1, + last_error: 'timeout:new process' + }); }); it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); - const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js'); + const { inspectForMonitoring, rconForMonitoring } = + await import('../../src/lib/server/fleet/health-observations.js'); vi.useFakeTimers({ toFake: ['Date'] }); const at = Date.parse('2026-10-05T12:00:00Z'); vi.setSystemTime(at); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await inspectForMonitoring('managed'); await rconForMonitoring('managed'); vi.setSystemTime(at + 10_000); await fanoutStatus(); expect(inspect).toHaveBeenCalledTimes(1); expect(rcon).toHaveBeenCalledTimes(1); - expect(getServer('managed')).toMatchObject({ container_checked_at: new Date(at).toISOString(), last_seen_at: new Date(at).toISOString() }); + expect(getServer('managed')).toMatchObject({ + container_checked_at: new Date(at).toISOString(), + last_seen_at: new Date(at).toISOString() + }); const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'manual failure' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'manual failure' + }); await POST({ params: { name: 'managed' } } as never); expect(rcon).toHaveBeenCalledTimes(2); vi.setSystemTime(at + 20_000); await fanoutStatus(); expect(rcon).toHaveBeenCalledTimes(3); - expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: new Date(at).toISOString() }); + expect(getServer('managed')).toMatchObject({ + last_error: 'timeout:manual failure', + last_seen_at: new Date(at).toISOString() + }); }); it('does not overwrite a newer manual observation with an earlier background reply', async () => { @@ -173,15 +288,29 @@ describe('database fleet and container readiness', () => { vi.setSystemTime(at); inspect.mockResolvedValue({ ok: true, state: 'running' }); let release!: (result: Awaited>) => void; - rcon.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + rcon.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = fanoutStatus(); await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1)); vi.setSystemTime(at + 10_000); - db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'").run(new Date().toISOString()); - release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + db.prepare( + "UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'" + ).run(new Date().toISOString()); + release({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await pending; - expect(getServer('managed')).toMatchObject({ last_error: 'timeout:manual failure', last_seen_at: null, last_checked_at: new Date(at + 10_000).toISOString() }); + expect(getServer('managed')).toMatchObject({ + last_error: 'timeout:manual failure', + last_seen_at: null, + last_checked_at: new Date(at + 10_000).toISOString() + }); }); it('refreshes a ten-server host while one check is slow, without duplicating or exceeding two checks', async () => { @@ -190,34 +319,51 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); const now = Date.parse('2026-10-02T12:00:00Z'); vi.setSystemTime(now); - for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name) VALUES (?)').run(`fast-${i}`); + for (let i = 0; i < 9; i++) + db.prepare('INSERT INTO servers (name) VALUES (?)').run(`fast-${i}`); const { prepareNativeHost } = await import('../helpers/native-host.js'); await prepareNativeHost(['managed', ...Array.from({ length: 9 }, (_, i) => `fast-${i}`)]); let release!: (result: { ok: true; state: string }) => void; let active = 0; let peak = 0; - inspect.mockImplementation(async name => { + inspect.mockImplementation(async (name) => { peak = Math.max(peak, ++active); try { await Promise.resolve(); return name === 'managed' - ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) + ? await new Promise<{ ok: true; state: string }>((resolve) => { + release = resolve; + }) : { ok: true as const, state: 'running' }; - } finally { active--; } + } finally { + active--; + } + }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); + const unsubscribe = subscribe((event) => events.push(event)); const first = fanoutStatus(); try { await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9)); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([ - expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 }) - ]) })); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: expect.arrayContaining([ + expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 }) + ]) + }) + ); vi.setSystemTime(now + 30_000); const second = fanoutStatus(); - await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000)); + await vi.waitFor(() => + expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual( + now + 30_000 + ) + ); expect(rcon).toHaveBeenCalledTimes(18); expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1); expect(peak).toBe(2); @@ -240,29 +386,39 @@ describe('database fleet and container readiness', () => { hosts.enrollHost(hosts.createEnrollment(registered.id).token, { protocolVersion: 1 }); for (let instance = 0; instance < 4; instance++) { const name = `host-${host}-${instance}`; - db.prepare('INSERT INTO servers (name) VALUES (?)') - .run(name); - db.prepare(`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at) - VALUES (?, ?, ?, 'library', 'image', ?, ?)`).run(name, name, registered.id, 28960 + instance, new Date().toISOString()); + db.prepare('INSERT INTO servers (name) VALUES (?)').run(name); + db.prepare( + `INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at) + VALUES (?, ?, ?, 'library', 'image', ?, ?)` + ).run(name, name, registered.id, 28960 + instance, new Date().toISOString()); } } let releaseSlow!: () => void; let releaseFast!: () => void; - const slow = new Promise(resolve => { releaseSlow = resolve; }); - const fast = new Promise(resolve => { releaseFast = resolve; }); + const slow = new Promise((resolve) => { + releaseSlow = resolve; + }); + const fast = new Promise((resolve) => { + releaseFast = resolve; + }); const active = new Map(); const peak = new Map(); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockImplementation(async name => { + rcon.mockImplementation(async (name) => { const host = name.slice(0, -2); active.set(host, (active.get(host) ?? 0) + 1); peak.set(host, Math.max(peak.get(host) ?? 0, active.get(host)!)); try { await (host === 'host-0' ? slow : fast); if (name === 'host-0-0') throw new Error('simulated failed probe'); - return { ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }; - } finally { active.set(host, active.get(host)! - 1); } + return { + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } + }; + } finally { + active.set(host, active.get(host)! - 1); + } }); const passes = [fanoutStatus()]; try { @@ -279,7 +435,8 @@ describe('database fleet and container readiness', () => { expect(getServer('host-0-2')?.container_checked_at).toBeNull(); expect(rcon).toHaveBeenCalledTimes(10); } finally { - releaseSlow(); releaseFast(); + releaseSlow(); + releaseFast(); await Promise.all(passes); } expect([...peak.values()]).toEqual([2, 2, 2]); @@ -296,11 +453,18 @@ describe('database fleet and container readiness', () => { vi.setSystemTime(now); db.prepare("INSERT INTO servers (name) VALUES ('slow')").run(); let release!: (result: { ok: true; state: string }) => void; - inspect.mockImplementation(async name => name === 'slow' - ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) - : { ok: true as const, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + inspect.mockImplementation(async (name) => + name === 'slow' + ? await new Promise<{ ok: true; state: string }>((resolve) => { + release = resolve; + }) + : { ok: true as const, state: 'running' } + ); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); const pending = fanoutStatus(); try { await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1)); @@ -309,8 +473,11 @@ describe('database fleet and container readiness', () => { release({ ok: true, state: 'running' }); await pending; } - expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([ - { server: 'managed', players: null }, { server: 'slow', players: 2 } + expect( + db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all() + ).toEqual([ + { server: 'managed', players: null }, + { server: 'slow', players: 2 } ]); }); @@ -318,69 +485,137 @@ describe('database fleet and container readiness', () => { const { db, rcon, getServer } = await setup(); const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); let release!: (result: Awaited>) => void; - rcon.mockImplementation(() => new Promise(resolve => { release = resolve; })); + rcon.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }) + ); const pending = POST({ params: { name: 'managed' } } as never); - db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); - release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run( + new Date().toISOString() + ); + release({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); expect(await (await pending).json()).toMatchObject({ ok: true }); - expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' }); + expect(getServer('managed')).toMatchObject({ + last_seen_at: null, + last_error: 'timeout:old process' + }); }); - it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => { - const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); - let release!: () => void; - const blocked = new Promise(resolve => { release = resolve; }); - inspect.mockImplementation(async () => { - if (phase === 'container') await blocked; - return { ok: true, state: 'running' }; - }); - rcon.mockImplementation(async () => { - if (phase === 'rcon') await blocked; - return { ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }; - }); - const pending = fanoutStatus(); - try { - if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled()); - db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run(); - } finally { release(); await pending; } - expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null }); - }); + it.each(['container', 'rcon'])( + 'ignores an old registration whose %s check finishes after replacement', + async (phase) => { + const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + inspect.mockImplementation(async () => { + if (phase === 'container') await blocked; + return { ok: true, state: 'running' }; + }); + rcon.mockImplementation(async () => { + if (phase === 'rcon') await blocked; + return { + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }; + }); + const pending = fanoutStatus(); + try { + if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled()); + db.prepare( + "UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'" + ).run(); + } finally { + release(); + await pending; + } + expect(getServer('managed')).toMatchObject({ + container_state: null, + last_seen_at: null, + last_error: null + }); + } + ); - it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => { - const { db, inspect, rcon, fanoutStatus } = await setup(); - const { GET } = await import('../../src/routes/api/servers/+server.js'); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - db.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')`).run(state, new Date().toISOString()); - const snapshot = async () => (await (await GET()).json()).servers[0]; - expect(await snapshot()).toMatchObject({ health: 'provisioning', last_error: null, last_players: null }); - const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); - try { await fanoutStatus(); } finally { unsubscribe(); } - expect(inspect).not.toHaveBeenCalled(); - expect(rcon).not.toHaveBeenCalled(); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ - health: 'provisioning', lastError: null, reason: null - })] })); - db.prepare("UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'").run(); - inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); - await fanoutStatus(); - expect(await snapshot()).toMatchObject({ health: 'offline', last_error: 'timeout:no reply' }); - db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run(); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); - await fanoutStatus(); - expect(await snapshot()).toMatchObject({ health: 'online', last_error: null }); - }); + it.each(['queued', 'running', 'awaiting-health'])( + 'shows provisioning during %s creation and resumes health reporting afterward', + async (state) => { + const { db, inspect, rcon, fanoutStatus } = await setup(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + db.prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')` + ).run(state, new Date().toISOString()); + const snapshot = async () => (await (await GET()).json()).servers[0]; + expect(await snapshot()).toMatchObject({ + health: 'provisioning', + last_error: null, + last_players: null + }); + const events: unknown[] = []; + const unsubscribe = subscribe((event) => events.push(event)); + try { + await fanoutStatus(); + } finally { + unsubscribe(); + } + expect(inspect).not.toHaveBeenCalled(); + expect(rcon).not.toHaveBeenCalled(); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: [ + expect.objectContaining({ + health: 'provisioning', + lastError: null, + reason: null + }) + ] + }) + ); + db.prepare( + "UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'" + ).run(); + inspect.mockResolvedValue({ ok: true, state: 'running' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'no reply' + }); + await fanoutStatus(); + expect(await snapshot()).toMatchObject({ + health: 'offline', + last_error: 'timeout:no reply' + }); + db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run(); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); + await fanoutStatus(); + expect(await snapshot()).toMatchObject({ health: 'online', last_error: null }); + } + ); it('keeps game_end through occupied polls until game_start or an empty lobby', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); inspect.mockResolvedValue({ ok: true, state: 'running' }); - const reply = (players: number) => ({ ok: true as const, endpoint: { host: 'localhost', port: 28960 }, - status: { players, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + const reply = (players: number) => ({ + ok: true as const, + endpoint: { host: 'localhost', port: 28960 }, + status: { players, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); rcon.mockResolvedValue(reply(1)); await postTelemetry(gameEvent('game_start', 0, 1, 'managed')); await postTelemetry(gameEvent('game_end', 10, 1, 'managed')); @@ -412,31 +647,61 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); expect((await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).status).toBe(200); - expect(getServer('managed')).toMatchObject({ active_since: at(0), inactive_since: null, game_started_at: at(0) }); + expect(getServer('managed')).toMatchObject({ + active_since: at(0), + inactive_since: null, + game_started_at: at(0) + }); vi.setSystemTime(startedAt + 19_000); - const heartbeat = await (await postTelemetry(gameEvent('heartbeat', 2, 1, 'managed'))).json(); + const heartbeat = await ( + await postTelemetry(gameEvent('heartbeat', 2, 1, 'managed')) + ).json(); expect(heartbeat.telemetry.timing.server).toEqual({ - activeSince: at(0), inactiveSince: null, gameStartedAt: at(0), gameEndedAt: null + activeSince: at(0), + inactiveSince: null, + gameStartedAt: at(0), + gameEndedAt: null }); vi.setSystemTime(startedAt + 30_000); const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); - try { await fanoutStatus(); } finally { unsubscribe(); } + const unsubscribe = subscribe((event) => events.push(event)); + try { + await fanoutStatus(); + } finally { + unsubscribe(); + } expect(rcon).toHaveBeenCalledOnce(); expect((await (await GET()).json()).servers[0]).toMatchObject({ - active_since: at(0), game_started_at: at(0), last_seen_at: at(19) + active_since: at(0), + game_started_at: at(0), + last_seen_at: at(19) }); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ - activeSince: at(0), gameStartedAt: at(0), lastSeenAt: at(19) - })] })); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: [ + expect.objectContaining({ + activeSince: at(0), + gameStartedAt: at(0), + lastSeenAt: at(19) + }) + ] + }) + ); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); - expect((await (await GET()).json()).servers[0]).toMatchObject({ active_since: at(0), game_started_at: at(0) }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + active_since: at(0), + game_started_at: at(0) + }); }); it('preserves occupancy observed by RCON before game-start telemetry', async () => { @@ -446,17 +711,30 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt - 20_000); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await fanoutStatus(); - expect(getServer('managed')).toMatchObject({ active_since: at(-20), game_started_at: null }); + expect(getServer('managed')).toMatchObject({ + active_since: at(-20), + game_started_at: null + }); vi.setSystemTime(startedAt); const start = await (await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).json(); expect(start.telemetry.timing.server).toEqual({ - activeSince: at(-20), inactiveSince: null, gameStartedAt: at(0), gameEndedAt: null + activeSince: at(-20), + inactiveSince: null, + gameStartedAt: at(0), + gameEndedAt: null + }); + expect(getServer('managed')).toMatchObject({ + active_since: at(-20), + inactive_since: null, + game_started_at: at(0) }); - expect(getServer('managed')).toMatchObject({ active_since: at(-20), inactive_since: null, game_started_at: at(0) }); }); it('records an empty lobby and rejoin between status polls without restoring the old match', async () => { @@ -473,24 +751,39 @@ describe('database fleet and container readiness', () => { vi.setSystemTime(startedAt + 11_000); const empty = await (await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'))).json(); expect(empty.telemetry.timing.server).toEqual({ - activeSince: null, inactiveSince: at(11), gameStartedAt: null, gameEndedAt: null + activeSince: null, + inactiveSince: at(11), + gameStartedAt: null, + gameEndedAt: null }); expect(getServer('managed')).toMatchObject({ - active_since: null, inactive_since: at(11), game_started_at: null, game_ended_at: null + active_since: null, + inactive_since: at(11), + game_started_at: null, + game_ended_at: null }); vi.setSystemTime(startedAt + 15_000); const rejoin = await (await postTelemetry(gameEvent('heartbeat', 1, 1, 'managed'))).json(); expect(rejoin.telemetry.timing.server).toEqual({ - activeSince: at(15), inactiveSince: null, gameStartedAt: null, gameEndedAt: null + activeSince: at(15), + inactiveSince: null, + gameStartedAt: null, + gameEndedAt: null }); expect(getServer('managed')).toMatchObject({ - active_since: at(15), inactive_since: null, game_started_at: null, game_ended_at: null + active_since: at(15), + inactive_since: null, + game_started_at: null, + game_ended_at: null }); vi.setSystemTime(startedAt + 30_000); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ - active_since: at(15), inactive_since: null, game_started_at: null, game_ended_at: null + active_since: at(15), + inactive_since: null, + game_started_at: null, + game_ended_at: null }); }); @@ -501,10 +794,16 @@ describe('database fleet and container readiness', () => { await mkdir(logs, { recursive: true }); await writeFile(join(logs, 'games_sp.log'), '0:00 InitGame:\n0:01 J;123;0;Player\n'); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await fanoutStatus(); - expect(getServer('managed')).toMatchObject({ active_since: expect.any(String), game_started_at: null }); + expect(getServer('managed')).toMatchObject({ + active_since: expect.any(String), + game_started_at: null + }); for (const event of ['round_change', 'heartbeat'] as const) { expect((await postTelemetry(gameEvent(event, 1, 1, 'managed'))).status).toBe(200); @@ -517,7 +816,8 @@ describe('database fleet and container readiness', () => { const { telemetry } = await response.json(); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ - game_started_at: telemetry.receivedAt, game_ended_at: null + game_started_at: telemetry.receivedAt, + game_ended_at: null }); }); @@ -530,50 +830,117 @@ describe('database fleet and container readiness', () => { expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now()); }); - it.each(['t5', 't6'])('supplements %s telemetry with RCON pings consistently in REST and status events', async game => { - const { db, inspect, rcon, fanoutStatus } = await setup(); - const { GET } = await import('../../src/routes/api/servers/+server.js'); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - const map = game === 't6' ? 'zm_transit' : 'zombie_theater'; - db.prepare("UPDATE servers SET game = ?, map = ?, last_players = 4, last_seen_at = ? WHERE name = 'managed'").run(game, map, new Date().toISOString()); - inspect.mockResolvedValue({ ok: true, state: 'running' }); - const telemetry = gameEvent('heartbeat', 15, 3, 'managed', map); - telemetry.players[0].ping = 42; - telemetry.players[1].guid = '7097376'; - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 4, map, gametype: 'zom', roster: [ - { num: 0, guid: telemetry.players[0].guid, name: 'Old name', score: 50, ping: 100 }, - { num: 1, guid: game === 't6' ? '6C4C20' : telemetry.players[1].guid, name: 'Another old name', score: 100, ping: 30 }, - { num: 2, guid: '6C45DD', name: telemetry.players[2].name, score: 150, ping: 151 } - ] } }); - await postTelemetry(telemetry); - const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); - try { await fanoutStatus(); } finally { unsubscribe(); } - expect(rcon).toHaveBeenCalledOnce(); - const server = (await (await GET()).json()).servers[0]; - expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null, - rcon_status: { last_players: 4, last_error: null } }); - expect(server.roster).toHaveLength(3); - expect(server.roster[0]).toMatchObject({ ping: 42, score: 5000 }); - expect(server.roster[1]).toMatchObject({ ping: 30, score: 5000, name: 'Player 2' }); - expect(server.roster[2]).toMatchObject({ ping: null, guid: telemetry.players[2].guid, name: 'Player 3', score: 5000 }); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ - health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at - })] })); - expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 }); - }); + it.each(['t5', 't6'])( + 'supplements %s telemetry with RCON pings consistently in REST and status events', + async (game) => { + const { db, inspect, rcon, fanoutStatus } = await setup(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const map = game === 't6' ? 'zm_transit' : 'zombie_theater'; + db.prepare( + "UPDATE servers SET game = ?, map = ?, last_players = 4, last_seen_at = ? WHERE name = 'managed'" + ).run(game, map, new Date().toISOString()); + inspect.mockResolvedValue({ ok: true, state: 'running' }); + const telemetry = gameEvent('heartbeat', 15, 3, 'managed', map); + telemetry.players[0].ping = 42; + telemetry.players[1].guid = '7097376'; + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { + players: 4, + map, + gametype: 'zom', + roster: [ + { + num: 0, + guid: telemetry.players[0].guid, + name: 'Old name', + score: 50, + ping: 100 + }, + { + num: 1, + guid: game === 't6' ? '6C4C20' : telemetry.players[1].guid, + name: 'Another old name', + score: 100, + ping: 30 + }, + { + num: 2, + guid: '6C45DD', + name: telemetry.players[2].name, + score: 150, + ping: 151 + } + ] + } + }); + await postTelemetry(telemetry); + const events: unknown[] = []; + const unsubscribe = subscribe((event) => events.push(event)); + try { + await fanoutStatus(); + } finally { + unsubscribe(); + } + expect(rcon).toHaveBeenCalledOnce(); + const server = (await (await GET()).json()).servers[0]; + expect(server).toMatchObject({ + health: 'online', + last_players: 3, + status_source: 'telemetry', + last_error: null, + rcon_status: { last_players: 4, last_error: null } + }); + expect(server.roster).toHaveLength(3); + expect(server.roster[0]).toMatchObject({ ping: 42, score: 5000 }); + expect(server.roster[1]).toMatchObject({ ping: 30, score: 5000, name: 'Player 2' }); + expect(server.roster[2]).toMatchObject({ + ping: null, + guid: telemetry.players[2].guid, + name: 'Player 3', + score: 5000 + }); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: [ + expect.objectContaining({ + health: 'online', + players: 3, + roster: server.roster, + lastSeenAt: server.last_seen_at + }) + ] + }) + ); + expect( + db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get() + ).toEqual({ players: 3 }); + } + ); it('keeps occupied telemetry online when supplementary RCON fails', async () => { const { inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'no reply' + }); await postTelemetry(gameEvent('heartbeat', 15, 1, 'managed')); await fanoutStatus(); expect(rcon).toHaveBeenCalledOnce(); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', status_source: 'telemetry', - last_players: 1, last_error: null, roster: [{ name: 'Player 1', score: 5000, ping: null }] }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'online', + status_source: 'telemetry', + last_players: 1, + last_error: null, + roster: [{ name: 'Player 1', score: 5000, ping: null }] + }); }); it('skips supplementary RCON for empty telemetry rosters', async () => { @@ -589,12 +956,21 @@ describe('database fleet and container readiness', () => { const { GET } = await import('../../src/routes/api/servers/+server.js'); inspect.mockResolvedValue({ ok: true, state: 'running' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); - db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 91_000).toISOString()); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + db.prepare('UPDATE server_telemetry SET received_at = ?').run( + new Date(Date.now() - 91_000).toISOString() + ); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await fanoutStatus(); expect(rcon).toHaveBeenCalledTimes(1); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 0, status_source: 'rcon' }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'online', + last_players: 0, + status_source: 'rcon' + }); }); it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => { @@ -608,68 +984,135 @@ describe('database fleet and container readiness', () => { it('skips startup RCON, clears old errors, and resumes polling only after the host agent reports readiness', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); - inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' }); + inspect.mockResolvedValue({ + ok: false, + state: 'starting', + reason: 'container state: starting' + }); await fanoutStatus(); await fanoutStatus(); expect(rcon).not.toHaveBeenCalled(); - expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null }); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null }); + expect(getServer('managed')).toMatchObject({ + container_state: 'starting', + last_error: null + }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'starting', + last_error: null, + last_players: null + }); inspect.mockResolvedValue({ ok: true, state: 'running' }); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'no reply' + }); await fanoutStatus(); expect(rcon).toHaveBeenCalledTimes(1); - expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' }); + expect(getServer('managed')).toMatchObject({ + container_state: 'running', + last_error: 'timeout:no reply' + }); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); await fanoutStatus(); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'online', + last_error: null + }); }); - it.each(['unhealthy', 'exited', 'stopped', 'missing'])('does not poll RCON with container state %s', async (state) => { - const { inspect, rcon, fanoutStatus } = await setup(); - inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' }); - await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); - await fanoutStatus(); - expect(rcon).not.toHaveBeenCalled(); - const { GET } = await import('../../src/routes/api/servers/+server.js'); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null }); - }); + it.each(['unhealthy', 'exited', 'stopped', 'missing'])( + 'does not poll RCON with container state %s', + async (state) => { + const { inspect, rcon, fanoutStatus } = await setup(); + inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' }); + await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); + await fanoutStatus(); + expect(rcon).not.toHaveBeenCalled(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'offline', + last_players: null + }); + } + ); - it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => { - const { inspect, rcon, fanoutStatus, getServer } = await setup(); - vi.useFakeTimers({ toFake: ['Date'] }); - const { GET } = await import('../../src/routes/api/servers/+server.js'); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out')); - else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' }); - await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); - rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, - status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); - const events: unknown[] = []; - const unsubscribe = subscribe(event => events.push(event)); - try { await fanoutStatus(); } finally { unsubscribe(); } - expect(rcon).toHaveBeenCalledOnce(); - expect(getServer('managed')?.container_state).toBe('unknown'); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 2, status_source: 'rcon' }); - expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ - health: 'online', players: 2, containerState: 'unknown' - })] })); + it.each(['unknown', undefined, 'throw'])( + 'automatically recovers through RCON when inspection returns %s', + async (state) => { + const { inspect, rcon, fanoutStatus, getServer } = await setup(); + vi.useFakeTimers({ toFake: ['Date'] }); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out')); + else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' }); + await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); + rcon.mockResolvedValue({ + ok: true, + endpoint: { host: 'localhost', port: 28960 }, + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } + }); + const events: unknown[] = []; + const unsubscribe = subscribe((event) => events.push(event)); + try { + await fanoutStatus(); + } finally { + unsubscribe(); + } + expect(rcon).toHaveBeenCalledOnce(); + expect(getServer('managed')?.container_state).toBe('unknown'); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'online', + last_players: 2, + status_source: 'rcon' + }); + expect(events).toContainEqual( + expect.objectContaining({ + type: 'server-status', + entries: [ + expect.objectContaining({ + health: 'online', + players: 2, + containerState: 'unknown' + }) + ] + }) + ); - rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); - vi.setSystemTime(Date.now() + 30_000); - await fanoutStatus(); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null, - rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } }); - vi.setSystemTime(Date.now() + 30_000); - await fanoutStatus(); - expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' }); - }); + rcon.mockResolvedValue({ + ok: false, + endpoint: null, + reason: 'timeout', + detail: 'no reply' + }); + vi.setSystemTime(Date.now() + 30_000); + await fanoutStatus(); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'online', + last_error: null, + rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } + }); + vi.setSystemTime(Date.now() + 30_000); + await fanoutStatus(); + expect((await (await GET()).json()).servers[0]).toMatchObject({ + health: 'stale', + last_error: 'timeout:no reply' + }); + } + ); it('waits for an EC restart request to settle before trusting the old healthy container', async () => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); - db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); + db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run( + new Date().toISOString() + ); await fanoutStatus(); expect(inspect).not.toHaveBeenCalled(); expect(rcon).not.toHaveBeenCalled(); diff --git a/tests/unit/slow-statements.test.ts b/tests/unit/slow-statements.test.ts index 4416113..9d0e860 100644 --- a/tests/unit/slow-statements.test.ts +++ b/tests/unit/slow-statements.test.ts @@ -7,17 +7,31 @@ afterEach(() => vi.restoreAllMocks()); it('preserves results and errors, logs slow query shapes without values', () => { const db = new Database(':memory:'); const log = vi.spyOn(console, 'warn').mockImplementation(() => {}); - vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValueOnce(21) - .mockReturnValueOnce(30).mockReturnValueOnce(31).mockReturnValueOnce(40).mockReturnValueOnce(62); + vi.spyOn(performance, 'now') + .mockReturnValueOnce(0) + .mockReturnValueOnce(21) + .mockReturnValueOnce(30) + .mockReturnValueOnce(31) + .mockReturnValueOnce(40) + .mockReturnValueOnce(62); try { - const query = timeStatement(db.prepare("SELECT ? AS value, 'private''literal' AS literal"), "SELECT ? AS value, 'private''literal' AS literal", 20); + const query = timeStatement( + db.prepare("SELECT ? AS value, 'private''literal' AS literal"), + "SELECT ? AS value, 'private''literal' AS literal", + 20 + ); expect(query.get('secret')).toEqual({ value: 'secret', literal: "private'literal" }); expect(query.all('other')).toHaveLength(1); expect(() => query.get()).toThrow(); expect(log).toHaveBeenCalledTimes(2); - expect(log.mock.calls[0]).toEqual(['[sql]', { ms: 21, method: 'get', sql: 'SELECT ? AS value, ? AS literal' }]); + expect(log.mock.calls[0]).toEqual([ + '[sql]', + { ms: 21, method: 'get', sql: 'SELECT ? AS value, ? AS literal' } + ]); expect(JSON.stringify(log.mock.calls)).not.toMatch(/secret|private|other/); - } finally { db.close(); } + } finally { + db.close(); + } }); it('does not wrap statements when disabled or invalid', () => { @@ -28,5 +42,7 @@ it('does not wrap statements when disabled or invalid', () => { for (const threshold of [0, -1, NaN, Infinity]) { expect(timeStatement(statement, 'SELECT 1', threshold).get).toBe(get); } - } finally { db.close(); } + } finally { + db.close(); + } }); diff --git a/tests/unit/staged-files.test.ts b/tests/unit/staged-files.test.ts index b99096b..d273ae6 100644 --- a/tests/unit/staged-files.test.ts +++ b/tests/unit/staged-files.test.ts @@ -1,7 +1,7 @@ import { putFileBlob } from '../../src/lib/server/db/file-blobs.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { existsSync, writeFileSync } from 'node:fs'; +import { chmod, readFile, stat, writeFile } from 'node:fs/promises'; +import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; @@ -11,7 +11,8 @@ vi.mock('node:fs', async (importOriginal) => { }); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() + rconGetPlayerCount: vi.fn(), + rconStatusResult: vi.fn() })); let fixture: Fixture; @@ -20,7 +21,8 @@ const relPath = 'staged.gsc'; const before = Buffer.from('main(){ /* current live script */ }'); const after = Buffer.from('main(){ /* reviewed replacement */ }'); type Source = 'campaign' | 'beta'; -const destination = (server = servers[0]) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath); +const destination = (server = servers[0]) => + join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath); beforeEach(async () => { vi.resetModules(); @@ -35,10 +37,12 @@ beforeEach(async () => { vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + const { rconGetPlayerCount, rconStatusResult } = + await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rconStatusResult).mockResolvedValue({ - ok: true, endpoint: { host: 'test', port: 28960 }, + ok: true, + endpoint: { host: 'test', port: 28960 }, status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); }); @@ -51,27 +55,48 @@ afterEach(async () => { async function queueChange(source: Source, targets = [servers[0]]) { const { db } = await import('../../src/lib/server/db/index.js'); - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const original = createScript({ name: 'Staged script', kind: 'script', targetRelPath: relPath, - scopeType: 'game', scopeValue: 't5', content: before }); + const { createScript, uploadNewVersion } = + await import('../../src/lib/server/fleet/scripts.js'); + const { recordScript, getServerScript } = + await import('../../src/lib/server/fleet/inventory.js'); + const original = createScript({ + name: 'Staged script', + kind: 'script', + targetRelPath: relPath, + scopeType: 'game', + scopeValue: 't5', + content: before + }); const replacement = uploadNewVersion(original.script.id, after); for (const server of targets) { await writeFile(destination(server), before); - recordScript(server, relPath, original.version.id, original.version.sha256, before.length, 'fixture'); + recordScript( + server, + relPath, + original.version.id, + original.version.sha256, + before.length, + 'fixture' + ); } const inventory = getServerScript(targets[0], relPath); let sourceId: string; if (source === 'campaign') { - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(replacement.id, original.script.id); + db() + .prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') + .run(replacement.id, original.script.id); const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const result = await runDriftDeployment({ servers: targets }); expect(result.deploymentId).toBeTruthy(); sourceId = result.deploymentId!; } else { const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const result = await openBeta({ scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }], - servers: targets, waitForHealth: false, waitForMatch: false }); + const result = await openBeta({ + scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }], + servers: targets, + waitForHealth: false, + waitForMatch: false + }); sourceId = String(result.betaId); } return { sourceId, inventory, replacement }; @@ -82,18 +107,24 @@ async function queueChange(source: Source, targets = [servers[0]]) { async function tick(source: Source) { const { db } = await import('../../src/lib/server/db/index.js'); const { activateFiles } = await import('../helpers/staged-files.js'); - const sources = db().prepare('SELECT DISTINCT source_id FROM staged_file_changes WHERE source_kind=? AND state=\'pending\'').all(source) as { source_id: string }[]; + const sources = db() + .prepare( + "SELECT DISTINCT source_id FROM staged_file_changes WHERE source_kind=? AND state='pending'" + ) + .all(source) as { source_id: string }[]; for (const row of sources) await activateFiles(source, row.source_id); } -async function expectLiveUnchanged(inventory: Awaited>['inventory'], server = servers[0]) { +async function expectLiveUnchanged( + inventory: Awaited>['inventory'], + server = servers[0] +) { const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(await readFile(destination(server))).toEqual(before); expect(getServerScript(server, relPath)).toEqual(inventory); } describe.each(['campaign', 'beta'])('%s staged file activation', (source) => { - it('preserves existing file permissions when activating staged bytes', async () => { await queueChange(source); await chmod(destination(), 0o640); @@ -105,7 +136,8 @@ describe.each(['campaign', 'beta'])('%s staged file activation', (source it('applies the exact queued bytes even if artifact storage changes while waiting', async () => { const { replacement } = await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE script_versions SET content_hash = ? WHERE id = ?') + db() + .prepare('UPDATE script_versions SET content_hash = ? WHERE id = ?') .run(putFileBlob(Buffer.from('changed after review'), db()), replacement.id); await tick(source); expect(await readFile(destination())).toEqual(after); diff --git a/tests/unit/sweep.test.ts b/tests/unit/sweep.test.ts index 989c1c7..3f53fcf 100644 --- a/tests/unit/sweep.test.ts +++ b/tests/unit/sweep.test.ts @@ -18,12 +18,14 @@ afterEach(async () => { describe('sweep', () => { it('leaves history cleanup to the retention requester', async () => { const retention = await import('../../src/lib/server/retention.js'); - const prune = vi.spyOn(retention,'pruneHistory'); + const prune = vi.spyOn(retention, 'pruneHistory'); try { const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); await sweep(); expect(prune).not.toHaveBeenCalled(); - } finally { prune.mockRestore(); } + } finally { + prune.mockRestore(); + } }); it('emits server state only when state, players, or error changes', async () => { @@ -32,12 +34,21 @@ describe('sweep', () => { const { db } = await import('../../src/lib/server/db/index.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); await seedServerRegistry(); - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('changes', 'test', ?, 'ui')") + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('changes', 'test', ?, 'ui')" + ) .run(new Date().toISOString()); - db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) - VALUES ('changes', 't5_zombie_theater_1', 1, 'pending')`).run(); + db() + .prepare( + `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) + VALUES ('changes', 't5_zombie_theater_1', 1, 'pending')` + ) + .run(); const events = vi.fn(); - const unsubscribe = subscribe(event => { if (event.type === 'server-state') events(event); }); + const unsubscribe = subscribe((event) => { + if (event.type === 'server-state') events(event); + }); try { await sweep(); expect(events).toHaveBeenCalledTimes(1); @@ -49,17 +60,26 @@ describe('sweep', () => { db().prepare('UPDATE server_deployments SET checked_at = ?').run(checkedAt); await sweep(); expect(events).toHaveBeenCalledTimes(1); - expect(db().prepare("SELECT checked_at, attempts FROM deployment_servers WHERE deployment_id = 'changes'").get()) - .toEqual({ checked_at: checkedAt, attempts: 1 }); + expect( + db() + .prepare( + "SELECT checked_at, attempts FROM deployment_servers WHERE deployment_id = 'changes'" + ) + .get() + ).toEqual({ checked_at: checkedAt, attempts: 1 }); db().prepare('UPDATE server_deployments SET players = 2').run(); await sweep(); expect(events).toHaveBeenCalledTimes(2); - expect(events).toHaveBeenLastCalledWith(expect.objectContaining({ state: 'pending', players: 2 })); + expect(events).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'pending', players: 2 }) + ); db().prepare("UPDATE server_deployments SET error = 'Waiting for players'").run(); await sweep(); expect(events).toHaveBeenCalledTimes(3); - expect(events).toHaveBeenLastCalledWith(expect.objectContaining({ error: 'Waiting for players' })); + expect(events).toHaveBeenLastCalledWith( + expect.objectContaining({ error: 'Waiting for players' }) + ); db().prepare("UPDATE server_deployments SET state = 'failed'").run(); await sweep(); expect(events).toHaveBeenCalledTimes(4); @@ -69,8 +89,12 @@ describe('sweep', () => { db().prepare("UPDATE server_deployments SET state = 'done'").run(); await sweep(); expect(events).toHaveBeenCalledTimes(5); - expect(events).toHaveBeenLastCalledWith(expect.objectContaining({ state: 'done', error: null })); - } finally { unsubscribe(); } + expect(events).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', error: null }) + ); + } finally { + unsubscribe(); + } }); it('queues pending rows in Provisioning even when integrations are unset', async () => { @@ -80,7 +104,10 @@ describe('sweep', () => { await seedServerRegistry(); const depId = 'dep-test-001'; - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'test', ?, 'ui')") + db() + .prepare( + "INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'test', ?, 'ui')" + ) .run(depId, new Date().toISOString()); db() .prepare( @@ -99,13 +126,17 @@ describe('sweep', () => { await sweep(); const rows = db() - .prepare("SELECT restart_state FROM deployment_servers WHERE deployment_id = ?") + .prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?') .all(depId) as Array<{ restart_state: string }>; expect(rows.length).toBe(2); for (const r of rows) expect(r.restart_state).toBe('pending'); const operations = db().prepare('SELECT state, stage FROM server_deployments').all(); expect(operations).toHaveLength(2); - for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting to verify the server is empty' }); + for (const operation of operations) + expect(operation).toEqual({ + state: 'waiting', + stage: 'Waiting to verify the server is empty' + }); }); it('is a no-op when nothing is pending', async () => { diff --git a/tests/unit/system-status.test.ts b/tests/unit/system-status.test.ts index 7a87567..99f5d83 100644 --- a/tests/unit/system-status.test.ts +++ b/tests/unit/system-status.test.ts @@ -7,14 +7,24 @@ beforeEach(async () => { fixture = await newFixture([]); vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret'); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); it('directs a fresh controller to host enrollment and verified game files', async () => { const { GET } = await import('../../src/routes/api/system-status/+server.js'); const response = await GET(); const body = await response.json(); expect(response.headers.get('cache-control')).toBe('no-store'); - expect(body.checks.map((check: { id: string }) => check.id)).toEqual(['hosts', 'runtime', 'game-files', 'rcon', 'keys', 'first-server']); + expect(body.checks.map((check: { id: string }) => check.id)).toEqual([ + 'hosts', + 'runtime', + 'game-files', + 'rcon', + 'keys', + 'first-server' + ]); expect(body.checks[0]).toMatchObject({ state: 'unconfigured', href: '/hosts' }); expect(body.checks[2].state).toBe('attention'); expect(body.readyServers).toBe(0); @@ -22,16 +32,22 @@ it('directs a fresh controller to host enrollment and verified game files', asyn }); it('reports an enrolled but disconnected host as needing attention without issuing commands', async () => { - const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + const { createHost, createEnrollment, enrollHost } = + await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const host = createHost({ name: 'Game host' }); const enrolled = enrollHost(createEnrollment(host.id).token, { protocolVersion: 1 }); - db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', host.id); + db() + .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') + .run('2000-01-01T00:00:00Z', host.id); const { GET } = await import('../../src/routes/api/system-status/+server.js'); const response = await GET(); const text = await response.text(); const body = JSON.parse(text); - expect(body.checks[0]).toMatchObject({ state: 'attention', detail: expect.stringContaining('0 connected') }); + expect(body.checks[0]).toMatchObject({ + state: 'attention', + detail: expect.stringContaining('0 connected') + }); expect(text).not.toContain(enrolled.credential); expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]); }); diff --git a/tests/unit/t6-config-flow.test.ts b/tests/unit/t6-config-flow.test.ts index 5c8dc45..fc86165 100644 --- a/tests/unit/t6-config-flow.test.ts +++ b/tests/unit/t6-config-flow.test.ts @@ -18,7 +18,8 @@ beforeEach(async () => { vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - for (const name of [town, farm]) await mkdir(join(fixture.t6, name, 'configs'), { recursive: true }); + for (const name of [town, farm]) + await mkdir(join(fixture.t6, name, 'configs'), { recursive: true }); }); afterEach(async () => { @@ -30,20 +31,35 @@ describe('T6 config lifecycle in a mixed fleet', () => { it('preserves variants through beta, scope promotion, drift repair, and campaign rollback', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); - const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const { scanServerDrift, runDriftDeployment } = + await import('../../src/lib/server/fleet/drift.js'); + const { rollbackPromoteDeployment } = + await import('../../src/lib/server/fleet/rollback.js'); const { db } = await import('../../src/lib/server/db/index.js'); const previous = '// prior operator settings\r\nsv_voice 0\r\n'; await writeFile(cfgPath(town), previous); - const { script, version } = createScript({ name: 'T6 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't6', - content: Buffer.from('sv_voice 1\nsv_maprotation "map zm_buried"\n') }); + const { script, version } = createScript({ + name: 'T6 baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't6', + content: Buffer.from('sv_voice 1\nsv_maprotation "map zm_buried"\n') + }); expect(script.target_rel_path).toBe('dedicated_zm.cfg'); - const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: [town], waitForHealth: false, waitForMatch: false }); + const { betaId } = await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: [town], + waitForHealth: false, + waitForMatch: false + }); expect(await readFile(cfgPath(town), 'utf8')).toBe(previous); await activateFiles('beta', betaId); - expect(await readFile(cfgPath(town), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); - db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId); + expect(await readFile(cfgPath(town), 'utf8')).toContain( + 'execgts zm_standard_town.cfg map zm_transit' + ); + db() + .prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?") + .run(new Date().toISOString(), betaId); db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId); const promoted = await promoteBeta(betaId); expect(new Set(promoted.landed)).toEqual(new Set([town, farm])); @@ -70,33 +86,48 @@ describe('T6 config lifecycle in a mixed fleet', () => { it('restores the exact pre-beta T6 config and preserves seeded rulesets during drift repair', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); + const { scanServerDrift, runDriftDeployment } = + await import('../../src/lib/server/fleet/drift.js'); const previous = 'sv_voice 1\n'; await writeFile(cfgPath(town), previous); const settings = join(fixture.t6, town, 'runtime/plutonium/storage/t6/gamesettings'); await mkdir(settings, { recursive: true }); await writeFile(join(settings, 'zm_standard_town.cfg'), 'ui_zm_mapstartlocation town\n'); await writeFile(join(fixture.t6, town, 'configs/dedicated.cfg'), 'sv_voice 1\n'); - const { script, version } = createScript({ name: 'T6 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't6', - content: Buffer.from('sv_voice 0\n') }); - const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], - servers: [town], waitForHealth: false, waitForMatch: false }); + const { script, version } = createScript({ + name: 'T6 baseline', + kind: 'baseline', + scopeType: 'game', + scopeValue: 't6', + content: Buffer.from('sv_voice 0\n') + }); + const { betaId } = await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: [town], + waitForHealth: false, + waitForMatch: false + }); await activateFiles('beta', betaId); const rollback = await rollbackBeta(betaId); await activateFiles('campaign', rollback.deploymentId!); expect(await readFile(cfgPath(town), 'utf8')).toBe(previous); expect(await scanServerDrift(town)).toEqual([]); await runDriftDeployment({ servers: [town] }); - expect(await readFile(join(settings, 'zm_standard_town.cfg'), 'utf8')).toBe('ui_zm_mapstartlocation town\n'); + expect(await readFile(join(settings, 'zm_standard_town.cfg'), 'utf8')).toBe( + 'ui_zm_mapstartlocation town\n' + ); }); it('reads the T6 game log and keeps log path traversal checks', async () => { - const { resolveLogPath, gameLogKey, tailLog } = await import('../../src/lib/server/fleet/logs.js'); + const { resolveLogPath, gameLogKey, tailLog } = + await import('../../src/lib/server/fleet/logs.js'); const logs = join(fixture.t6, town, 'logs'); await mkdir(logs, { recursive: true }); await writeFile(join(logs, 'games_zm.log'), '0:00 InitGame:\n0:01 J;123;0;Player\n'); expect(resolveLogPath(town, gameLogKey(town))).toBe(resolve(logs, 'games_zm.log')); - expect((await tailLog(resolveLogPath(town, gameLogKey(town))!, 10))?.lines).toContain('0:01 J;123;0;Player'); + expect((await tailLog(resolveLogPath(town, gameLogKey(town))!, 10))?.lines).toContain( + '0:01 J;123;0;Player' + ); expect(resolveLogPath('t6_/../../escape', 'games_zm')).toBeNull(); }); }); diff --git a/tests/unit/telemetry-auth.test.ts b/tests/unit/telemetry-auth.test.ts index 7d8f6a7..dd425f0 100644 --- a/tests/unit/telemetry-auth.test.ts +++ b/tests/unit/telemetry-auth.test.ts @@ -5,14 +5,27 @@ import { gameEvent } from '../helpers/telemetry.js'; const token = 'ec_telemetry_0123456789abcdef0123456789abcdef'; const otherToken = 'ec_telemetry_fedcba9876543210fedcba9876543210'; -const request = (values: Record = {}, url = 'http://localhost/api/telemetry') => { +const request = ( + values: Record = {}, + url = 'http://localhost/api/telemetry' +) => { const headers = new Headers({ 'Content-Type': 'application/json' }); - for (const [key, value] of Object.entries(values)) if (value !== undefined) headers.set(key, value); - return new Request(url, { method: 'POST', headers, body: JSON.stringify(gameEvent('heartbeat')) }); + for (const [key, value] of Object.entries(values)) + if (value !== undefined) headers.set(key, value); + return new Request(url, { + method: 'POST', + headers, + body: JSON.stringify(gameEvent('heartbeat')) + }); }; -beforeEach(() => { vi.stubEnv('EC_TELEMETRY_TOKEN', token); }); -afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +beforeEach(() => { + vi.stubEnv('EC_TELEMETRY_TOKEN', token); +}); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); describe('machine telemetry credentials', () => { it.each([ @@ -20,14 +33,19 @@ describe('machine telemetry credentials', () => { { authorization: `bearer ${token}` }, { 'X-EC-Telemetry-Token': token }, { Authorization: `Bearer ${token}`, 'X-EC-Telemetry-Token': token } - ])('accepts an explicit telemetry secret without a browser session: %j', headers => { + ])('accepts an explicit telemetry secret without a browser session: %j', (headers) => { expect(authenticateTelemetry(request(headers))).toEqual({ ok: true }); }); - it.each([undefined, '', 'too-short', 'x'.repeat(513), 'x'.repeat(40) + ' not-header-safe'])('fails closed when configuration is unusable: %s', configured => { - vi.stubEnv('EC_TELEMETRY_TOKEN', configured); - expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toMatchObject({ ok: false, status: 503 }); - }); + it.each([undefined, '', 'too-short', 'x'.repeat(513), 'x'.repeat(40) + ' not-header-safe'])( + 'fails closed when configuration is unusable: %s', + (configured) => { + vi.stubEnv('EC_TELEMETRY_TOKEN', configured); + expect( + authenticateTelemetry(request({ Authorization: `Bearer ${token}` })) + ).toMatchObject({ ok: false, status: 503 }); + } + ); it.each([ {}, @@ -42,7 +60,7 @@ describe('machine telemetry credentials', () => { { Authorization: 'Bearer short', 'X-EC-Telemetry-Token': token }, { Authorization: `Bearer ec_host_${'1'.repeat(43)}` }, { Authorization: `Bearer ec_enroll_${'1'.repeat(43)}` } - ])('rejects missing, wrong, ambiguous or unrelated credentials: %j', headers => { + ])('rejects missing, wrong, ambiguous or unrelated credentials: %j', (headers) => { const authentication = authenticateTelemetry(request(headers)); expect(authentication).toMatchObject({ ok: false, status: 401 }); expect(JSON.stringify(authentication)).not.toContain(token); @@ -50,19 +68,37 @@ describe('machine telemetry credentials', () => { }); it('does not accept tokens placed in query parameters', () => { - expect(authenticateTelemetry(request({}, `http://localhost/api/telemetry?token=${token}`))).toMatchObject({ ok: false, status: 401 }); + expect( + authenticateTelemetry(request({}, `http://localhost/api/telemetry?token=${token}`)) + ).toMatchObject({ ok: false, status: 401 }); }); it('retains existing transport behavior for authenticated local machine relays', () => { - expect(authenticateTelemetry(request({ 'X-EC-Telemetry-Token': token }, 'http://controller.internal/api/telemetry'))).toEqual({ ok: true }); + expect( + authenticateTelemetry( + request( + { 'X-EC-Telemetry-Token': token }, + 'http://controller.internal/api/telemetry' + ) + ) + ).toEqual({ ok: true }); }); it('uses the current configured token after rotation and safely rejects differing token lengths', () => { - expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toEqual({ ok: true }); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toEqual({ + ok: true + }); vi.stubEnv('EC_TELEMETRY_TOKEN', otherToken); - expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toMatchObject({ ok: false, status: 401 }); - expect(authenticateTelemetry(request({ Authorization: `Bearer ${otherToken}` }))).toEqual({ ok: true }); - expect(authenticateTelemetry(request({ Authorization: `Bearer ${'x'.repeat(32)}` }))).toMatchObject({ ok: false, status: 401 }); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${token}` }))).toMatchObject({ + ok: false, + status: 401 + }); + expect(authenticateTelemetry(request({ Authorization: `Bearer ${otherToken}` }))).toEqual({ + ok: true + }); + expect( + authenticateTelemetry(request({ Authorization: `Bearer ${'x'.repeat(32)}` })) + ).toMatchObject({ ok: false, status: 401 }); }); }); @@ -74,32 +110,40 @@ describe('telemetry endpoint authentication boundary', () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); - afterEach(async () => { await fixture.cleanup(); }); - - it.each([401, 503])('returns %s before parsing or storing untrusted evidence', async status => { - if (status === 503) vi.stubEnv('EC_TELEMETRY_TOKEN', undefined); - const { POST } = await import('../../src/routes/api/telemetry/+server.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - const incoming = request({ Cookie: 'ec_session=browser-user' }); - const parse = vi.spyOn(incoming, 'json'); - const listener = vi.fn(); - const unsubscribe = subscribe(listener); - const response = await POST({ request: incoming }); - unsubscribe(); - expect(response.status).toBe(status); - expect(response.headers.get('cache-control')).toBe('no-store'); - expect(parse).not.toHaveBeenCalled(); - expect(incoming.bodyUsed).toBe(false); - expect(listTelemetry()).toEqual([]); - expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 0 }); - expect(listener).not.toHaveBeenCalled(); + afterEach(async () => { + await fixture.cleanup(); }); + it.each([401, 503])( + 'returns %s before parsing or storing untrusted evidence', + async (status) => { + if (status === 503) vi.stubEnv('EC_TELEMETRY_TOKEN', undefined); + const { POST } = await import('../../src/routes/api/telemetry/+server.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const incoming = request({ Cookie: 'ec_session=browser-user' }); + const parse = vi.spyOn(incoming, 'json'); + const listener = vi.fn(); + const unsubscribe = subscribe(listener); + const response = await POST({ request: incoming }); + unsubscribe(); + expect(response.status).toBe(status); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect(parse).not.toHaveBeenCalled(); + expect(incoming.bodyUsed).toBe(false); + expect(listTelemetry()).toEqual([]); + expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ + count: 0 + }); + expect(listener).not.toHaveBeenCalled(); + } + ); + it('accepts authenticated automation with the unchanged event payload and excludes headers from diagnostics', async () => { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); - const { listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js'); + const { listTelemetryRequests } = + await import('../../src/lib/server/fleet/telemetry-requests.js'); const payload = gameEvent('heartbeat'); const response = await POST({ request: request({ 'X-EC-Telemetry-Token': token }) }); expect(response.status).toBe(200); diff --git a/tests/unit/telemetry-diagnostics.test.ts b/tests/unit/telemetry-diagnostics.test.ts index 57abbcf..095bcdd 100644 --- a/tests/unit/telemetry-diagnostics.test.ts +++ b/tests/unit/telemetry-diagnostics.test.ts @@ -18,7 +18,8 @@ afterEach(async () => { }); async function diagnostics(name = server) { - const { GET } = await import('../../src/routes/api/servers/[name]/telemetry-requests/+server.js'); + const { GET } = + await import('../../src/routes/api/servers/[name]/telemetry-requests/+server.js'); return GET({ params: { name } }); } @@ -27,7 +28,9 @@ it('returns uncached, empty diagnostics for a registered server before any reque expect(response.status).toBe(200); expect(response.headers.get('cache-control')).toContain('no-store'); expect(await response.json()).toEqual({ - ok: true, requests: [], retention: { maxPerServer: 200, maxAgeDays: 7, maxTotal: 10000 } + ok: true, + requests: [], + retention: { maxPerServer: 200, maxAgeDays: 7, maxTotal: 10000 } }); expect((await diagnostics('unknown')).status).toBe(404); }); @@ -44,9 +47,18 @@ it('persists the received payload and response, isolates each server, and lists const body = await (await diagnostics()).json(); expect(body.requests).toHaveLength(2); expect(body.requests[0]).toEqual({ - id: expect.any(Number), server, receivedAt: expect.any(String), status: 200, error: null, - event: 'round_change', map: 'zombie_theater', round: 3, playerCount: 4, - contentType: 'application/json', payload: JSON.stringify(latest), payloadTruncated: false + id: expect.any(Number), + server, + receivedAt: expect.any(String), + status: 200, + error: null, + event: 'round_change', + map: 'zombie_theater', + round: 3, + playerCount: 4, + contentType: 'application/json', + payload: JSON.stringify(latest), + payloadTruncated: false }); expect(body.requests[0].id).toBeGreaterThan(body.requests[1].id); expect(body.requests[1].payload).toBe(JSON.stringify(first)); @@ -62,10 +74,18 @@ it('records rejected player data with the same validation reason returned to the const result = await response.json(); expect(result.error).toContain('players[0].kills'); const { requests } = await (await diagnostics()).json(); - expect(requests).toEqual([expect.objectContaining({ - server, status: 400, error: result.error, event: 'round_change', round: 5, - playerCount: 1, payload: JSON.stringify(invalid), payloadTruncated: false - })]); + expect(requests).toEqual([ + expect.objectContaining({ + server, + status: 400, + error: result.error, + event: 'round_change', + round: 5, + playerCount: 1, + payload: JSON.stringify(invalid), + payloadTruncated: false + }) + ]); const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); expect(listTelemetry()).toEqual([]); }); @@ -76,14 +96,24 @@ it('records identifiable JSON sent with an unsupported or missing Content-Type', const { POST } = await import('../../src/routes/api/telemetry/+server.js'); // A byte body avoids Request automatically assigning text/plain to a string body. const request = new Request('http://localhost/api/telemetry', { - method: 'POST', headers: telemetryAuthHeaders(), body: new TextEncoder().encode(JSON.stringify(payload)) + method: 'POST', + headers: telemetryAuthHeaders(), + body: new TextEncoder().encode(JSON.stringify(payload)) }); expect(request.headers.get('content-type')).toBeNull(); expect((await POST({ request })).status).toBe(415); const { requests } = await (await diagnostics()).json(); expect(requests).toEqual([ - expect.objectContaining({ status: 415, contentType: null, error: 'Content-Type must be application/json' }), - expect.objectContaining({ status: 415, contentType: 'text/plain', payload: JSON.stringify(payload) }) + expect.objectContaining({ + status: 415, + contentType: null, + error: 'Content-Type must be application/json' + }), + expect.objectContaining({ + status: 415, + contentType: 'text/plain', + payload: JSON.stringify(payload) + }) ]); }); @@ -93,7 +123,13 @@ it('distinguishes a zero round and an empty scoreboard from absent or invalid fi await post({ server, players: 4 }); const { requests } = await (await diagnostics()).json(); expect(requests).toEqual([ - expect.objectContaining({ status: 400, event: null, map: null, round: null, playerCount: null }), + expect.objectContaining({ + status: 400, + event: null, + map: null, + round: null, + playerCount: null + }), expect.objectContaining({ status: 200, event: 'game_start', round: null, playerCount: 0 }), expect.objectContaining({ status: 200, event: 'round_change', round: 0, playerCount: 0 }) ]); @@ -109,14 +145,23 @@ it('keeps unknown and destroyed server rejections and matches the submitted serv setServerField(server, 'destroyed_at', new Date().toISOString()); expect((await post(gameEvent('game_end', 1))).status).toBe(404); - const { listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js'); - expect(listTelemetryRequests(unknown)).toEqual([expect.objectContaining({ server: unknown, status: 404 })]); - expect(listTelemetryRequests(spaced)).toEqual([expect.objectContaining({ server: spaced, status: 404 })]); + const { listTelemetryRequests } = + await import('../../src/lib/server/fleet/telemetry-requests.js'); + expect(listTelemetryRequests(unknown)).toEqual([ + expect.objectContaining({ server: unknown, status: 404 }) + ]); + expect(listTelemetryRequests(spaced)).toEqual([ + expect.objectContaining({ server: spaced, status: 404 }) + ]); expect((await diagnostics(unknown)).status).toBe(404); const response = await diagnostics(); expect(response.status).toBe(200); expect((await response.json()).requests).toEqual([ - expect.objectContaining({ server, status: 404, error: 'Server is not registered or has been destroyed' }), + expect.objectContaining({ + server, + status: 404, + error: 'Server is not registered or has been destroyed' + }), expect.objectContaining({ server, status: 200 }) ]); }); @@ -124,21 +169,30 @@ it('keeps unknown and destroyed server rejections and matches the submitted serv it('does not attribute malformed JSON or unusable server names to a server', async () => { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); const malformed = new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, body: `{"server":"${server}",` + method: 'POST', + headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, + body: `{"server":"${server}",` }); expect((await POST({ request: malformed })).status).toBe(400); for (const value of [undefined, null, '', ' ', 123, 's'.repeat(256)]) { expect((await post({ ...gameEvent('game_start'), server: value })).status).toBe(400); } const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 0 }); + expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ + count: 0 + }); }); it('bounds stored payloads and marks truncation without rejecting a valid event', async () => { const payload = { ...gameEvent('round_change', 4), diagnostic: 'x'.repeat(20_000) }; expect((await post(payload)).status).toBe(200); const { requests } = await (await diagnostics()).json(); - expect(requests[0]).toMatchObject({ status: 200, round: 4, playerCount: 4, payloadTruncated: true }); + expect(requests[0]).toMatchObject({ + status: 200, + round: 4, + playerCount: 4, + payloadTruncated: true + }); expect(requests[0].payload).toBe(JSON.stringify(payload).slice(0, 16 * 1024)); }); @@ -154,7 +208,9 @@ it('records processing failures after rollback without changing the scoreboard, const matches = db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all(); const listener = vi.fn(); const unsubscribe = subscribe(listener); - vi.spyOn(beta, 'recordGameEvent').mockImplementationOnce(() => { throw new Error('Injected beta processing failure'); }); + vi.spyOn(beta, 'recordGameEvent').mockImplementationOnce(() => { + throw new Error('Injected beta processing failure'); + }); const response = await post(gameEvent('game_end', 99)); unsubscribe(); @@ -165,7 +221,12 @@ it('records processing failures after rollback without changing the scoreboard, expect(db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all()).toEqual(matches); expect(listener).not.toHaveBeenCalled(); expect((await (await diagnostics()).json()).requests).toEqual([ - expect.objectContaining({ status: 500, event: 'game_end', round: 99, error: 'Could not process telemetry: Injected beta processing failure' }), + expect.objectContaining({ + status: 500, + event: 'game_end', + round: 99, + error: 'Could not process telemetry: Injected beta processing failure' + }), expect.objectContaining({ status: 200, round: 3 }) ]); }); @@ -183,15 +244,19 @@ it('still accepts and broadcasts telemetry if storing diagnostics fails', async expect(response.status).toBe(200); const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - expect(listTelemetry()).toEqual([expect.objectContaining({ ...payload, receivedAt: expect.any(String) })]); + expect(listTelemetry()).toEqual([ + expect.objectContaining({ ...payload, receivedAt: expect.any(String) }) + ]); expect(listener).toHaveBeenCalledWith({ - type: 'server-telemetry', entries: [expect.objectContaining({ ...payload, receivedAt: expect.any(String) })] + type: 'server-telemetry', + entries: [expect.objectContaining({ ...payload, receivedAt: expect.any(String) })] }); }); it('retains only the latest 200 requests for a server and removes requests older than seven days', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { recordTelemetryRequest, listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js'); + const { recordTelemetryRequest, listTelemetryRequests } = + await import('../../src/lib/server/fleet/telemetry-requests.js'); const now = new Date().toISOString(); const old = new Date(Date.now() - 8 * 86_400_000).toISOString(); const insert = db().prepare(`INSERT INTO telemetry_requests @@ -201,27 +266,52 @@ it('retains only the latest 200 requests for a server and removes requests older insert.run(otherServer, now); for (let i = 0; i < 205; i++) insert.run(server, now); })(); - recordTelemetryRequest({ value: gameEvent('round_change', 9), receivedAt: now, status: 200, error: null, contentType: 'application/json' }); + recordTelemetryRequest({ + value: gameEvent('round_change', 9), + receivedAt: now, + status: 200, + error: null, + contentType: 'application/json' + }); const requests = listTelemetryRequests(server); expect(requests).toHaveLength(200); expect(requests[0]).toMatchObject({ round: 9 }); - expect(requests.map(entry => entry.id)).toEqual([...requests.map(entry => entry.id)].sort((a, b) => b - a)); + expect(requests.map((entry) => entry.id)).toEqual( + [...requests.map((entry) => entry.id)].sort((a, b) => b - a) + ); expect(listTelemetryRequests(otherServer)).toHaveLength(1); - expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests WHERE received_at = ?').get(old)).toEqual({ count: 0 }); + expect( + db() + .prepare('SELECT COUNT(*) AS count FROM telemetry_requests WHERE received_at = ?') + .get(old) + ).toEqual({ count: 0 }); }); it('caps global diagnostic history even when rejected requests name many unknown servers', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - const { recordTelemetryRequest } = await import('../../src/lib/server/fleet/telemetry-requests.js'); + const { recordTelemetryRequest } = + await import('../../src/lib/server/fleet/telemetry-requests.js'); const now = new Date().toISOString(); const insert = db().prepare(`INSERT INTO telemetry_requests (server, received_at, status, payload, payload_truncated) VALUES (?, ?, 404, '{}', 0)`); db().transaction(() => { for (let i = 0; i < 10_005; i++) insert.run(`unknown-${i}`, now); })(); - recordTelemetryRequest({ value: gameEvent('game_start'), receivedAt: now, status: 200, error: null, contentType: 'application/json' }); - expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 10_000 }); - expect(db().prepare('SELECT server FROM telemetry_requests ORDER BY id DESC LIMIT 1').get()).toEqual({ server }); - expect(db().prepare("SELECT id FROM telemetry_requests WHERE server = 'unknown-0'").get()).toBeUndefined(); + recordTelemetryRequest({ + value: gameEvent('game_start'), + receivedAt: now, + status: 200, + error: null, + contentType: 'application/json' + }); + expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ + count: 10_000 + }); + expect( + db().prepare('SELECT server FROM telemetry_requests ORDER BY id DESC LIMIT 1').get() + ).toEqual({ server }); + expect( + db().prepare("SELECT id FROM telemetry_requests WHERE server = 'unknown-0'").get() + ).toBeUndefined(); }); diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index e05793b..402de39 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -1,6 +1,11 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { gameEvent, postTelemetry as post, roster, telemetryAuthHeaders } from '../helpers/telemetry.js'; +import { + gameEvent, + postTelemetry as post, + roster, + telemetryAuthHeaders +} from '../helpers/telemetry.js'; let fixture: Fixture; beforeEach(async () => { @@ -17,50 +22,88 @@ it('does not let a request received before a lifecycle submission restore cleare const { db } = await import('../../src/lib/server/db/index.js'); const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const at = (seconds: number) => new Date(Date.parse('2026-09-20T12:00:00Z') + seconds * 1000).toISOString(); - db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)').run(payload.server, at(10)); + const at = (seconds: number) => + new Date(Date.parse('2026-09-20T12:00:00Z') + seconds * 1000).toISOString(); + db() + .prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)') + .run(payload.server, at(10)); for (const seconds of [9, 10]) { saveTelemetry({ ...gameEvent('game_start'), receivedAt: at(seconds) }); - expect(getServer(payload.server)).toMatchObject({ active_since: null, game_started_at: null }); + expect(getServer(payload.server)).toMatchObject({ + active_since: null, + game_started_at: null + }); } const current = saveTelemetry({ ...gameEvent('heartbeat', 2), receivedAt: at(11) }); - expect(current.timing?.server).toEqual({ activeSince: at(11), inactiveSince: null, gameStartedAt: null, gameEndedAt: null }); + expect(current.timing?.server).toEqual({ + activeSince: at(11), + inactiveSince: null, + gameStartedAt: null, + gameEndedAt: null + }); }); it('generates timer anchors on the server instead of accepting client-supplied timing', async () => { - const response = await post({ ...gameEvent('game_start'), receivedAt: '2000-01-01T00:00:00Z', - timing: { server: { activeSince: '2000-01-01T00:00:00Z' } } }); + const response = await post({ + ...gameEvent('game_start'), + receivedAt: '2000-01-01T00:00:00Z', + timing: { server: { activeSince: '2000-01-01T00:00:00Z' } } + }); const { telemetry } = await response.json(); expect(response.status).toBe(200); expect(telemetry.receivedAt).not.toBe('2000-01-01T00:00:00Z'); - expect(telemetry.timing.server).toEqual({ activeSince: telemetry.receivedAt, inactiveSince: null, - gameStartedAt: telemetry.receivedAt, gameEndedAt: null }); + expect(telemetry.timing.server).toEqual({ + activeSince: telemetry.receivedAt, + inactiveSince: null, + gameStartedAt: telemetry.receivedAt, + gameEndedAt: null + }); }); it('persists player timing across matches and resets after absence or a telemetry gap', async () => { - const { saveTelemetry, listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + const { saveTelemetry, listTelemetry } = + await import('../../src/lib/server/fleet/telemetry.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); const guid = roster(1)[0].guid; - const at = (seconds: number) => new Date(Date.parse('2026-09-20T12:00:00Z') + seconds * 1000).toISOString(); - const save = (event: Parameters[0], seconds: number, round = 1, players = 1) => - saveTelemetry({ ...gameEvent(event, round, players), receivedAt: at(seconds) }); + const at = (seconds: number) => + new Date(Date.parse('2026-09-20T12:00:00Z') + seconds * 1000).toISOString(); + const save = ( + event: Parameters[0], + seconds: number, + round = 1, + players = 1 + ) => saveTelemetry({ ...gameEvent(event, round, players), receivedAt: at(seconds) }); save('game_start', 0); save('heartbeat', 60, 5); const end = save('game_end', 90, 6); - expect(end.timing?.players[guid]).toEqual({ serverJoinedAt: at(0), matchJoinedAt: at(0), roundStarted: 1 }); + expect(end.timing?.players[guid]).toEqual({ + serverJoinedAt: at(0), + matchJoinedAt: at(0), + roundStarted: 1 + }); const next = save('game_start', 100); - expect(next.timing?.players[guid]).toEqual({ serverJoinedAt: at(0), matchJoinedAt: at(100), roundStarted: 1 }); + expect(next.timing?.players[guid]).toEqual({ + serverJoinedAt: at(0), + matchJoinedAt: at(100), + roundStarted: 1 + }); closeDb(); expect(listTelemetry()[0].timing).toEqual(next.timing); save('heartbeat', 120, 2, 0); expect(save('heartbeat', 140, 3).timing?.players[guid]).toEqual({ - serverJoinedAt: at(140), matchJoinedAt: at(140), roundStarted: 3 + serverJoinedAt: at(140), + matchJoinedAt: at(140), + roundStarted: 3 }); expect(save('heartbeat', 400, 8).timing?.players[guid]).toEqual({ - serverJoinedAt: at(400), matchJoinedAt: at(400), roundStarted: 8 + serverJoinedAt: at(400), + matchJoinedAt: at(400), + roundStarted: 8 }); expect(save('heartbeat', 420, 1).timing?.players[guid]).toEqual({ - serverJoinedAt: at(400), matchJoinedAt: at(420), roundStarted: 1 + serverJoinedAt: at(400), + matchJoinedAt: at(420), + roundStarted: 1 }); }); @@ -72,18 +115,39 @@ it('timestamps initial telemetry and subsequent status and telemetry events with const reader = (await GET()).body!.getReader(); const nextEvent = async () => { const frame = new TextDecoder().decode((await reader.read()).value); - return JSON.parse(frame.split('\n').find(line => line.startsWith('data: '))!.slice(6)); + return JSON.parse( + frame + .split('\n') + .find((line) => line.startsWith('data: '))! + .slice(6) + ); }; try { await reader.read(); // Initial SSE comment. - expect(await nextEvent()).toMatchObject({ snapshot: true, serverNow: startedAt, entries: expect.any(Array) }); - expect(await nextEvent()).toMatchObject({ snapshot: true, entries: [], serverNow: startedAt }); + expect(await nextEvent()).toMatchObject({ + snapshot: true, + serverNow: startedAt, + entries: expect.any(Array) + }); + expect(await nextEvent()).toMatchObject({ + snapshot: true, + entries: [], + serverNow: startedAt + }); clock.mockReturnValue(startedAt + 30_000); emit({ type: 'server-status', entries: [] }); - expect(await nextEvent()).toEqual({ type: 'server-status', entries: [], serverNow: startedAt + 30_000 }); + expect(await nextEvent()).toEqual({ + type: 'server-status', + entries: [], + serverNow: startedAt + 30_000 + }); clock.mockReturnValue(startedAt + 31_000); emit({ type: 'server-telemetry', entries: [] }); - expect(await nextEvent()).toEqual({ type: 'server-telemetry', entries: [], serverNow: startedAt + 31_000 }); + expect(await nextEvent()).toEqual({ + type: 'server-telemetry', + entries: [], + serverNow: startedAt + 31_000 + }); } finally { await reader.cancel(); clock.mockRestore(); @@ -93,7 +157,9 @@ it('timestamps initial telemetry and subsequent status and telemetry events with it('serializes each live server event once across connections and releases cancelled subscribers', async () => { const { GET } = await import('../../src/routes/api/events/servers/+server.js'); const { emit } = await import('../../src/lib/server/pipeline/bus.js'); - const readers = await Promise.all([GET(), GET()].map(async response => (await response).body!.getReader())); + const readers = await Promise.all( + [GET(), GET()].map(async (response) => (await response).body!.getReader()) + ); const stringify = vi.spyOn(JSON, 'stringify'); try { for (const reader of readers) { @@ -103,16 +169,22 @@ it('serializes each live server event once across connections and releases cance } for (const event of [ { type: 'server-status' as const, delta: true, entries: [] }, - { type: 'server-telemetry' as const, entries: [{ ...payload, receivedAt: new Date().toISOString() }] } + { + type: 'server-telemetry' as const, + entries: [{ ...payload, receivedAt: new Date().toISOString() }] + } ]) { stringify.mockClear(); emit(event); expect(stringify).toHaveBeenCalledTimes(1); - const [first, second] = await Promise.all(readers.map(reader => reader.read())); + const [first, second] = await Promise.all(readers.map((reader) => reader.read())); expect(first.value).toBe(second.value); const frame = new TextDecoder().decode(first.value); expect(frame.startsWith(`event: ${event.type}\ndata: `)).toBe(true); - expect(JSON.parse(frame.split('\n')[1].slice(6))).toEqual({ ...event, serverNow: expect.any(Number) }); + expect(JSON.parse(frame.split('\n')[1].slice(6))).toEqual({ + ...event, + serverNow: expect.any(Number) + }); } await readers[0].cancel(); stringify.mockClear(); @@ -125,41 +197,54 @@ it('serializes each live server event once across connections and releases cance expect(stringify).not.toHaveBeenCalled(); } finally { stringify.mockRestore(); - await Promise.all(readers.map(reader => reader.cancel())); + await Promise.all(readers.map((reader) => reader.cancel())); } }); -it.each(['round_change', 'heartbeat'] as const)('persists %s stats, broadcasts updates, and leaves RCON safety data untouched', async (event) => { - const payload = gameEvent(event, 17); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - const listener = vi.fn(); - const unsubscribe = subscribe(listener); - expect(await (await post(payload)).json()).toEqual({ ok: true, telemetry: { ...payload, receivedAt: expect.any(String), timing: expect.any(Object) } }); - expect(listener).toHaveBeenCalledWith({ type: 'server-telemetry', entries: [{ ...payload, receivedAt: expect.any(String), timing: expect.any(Object) }] }); - unsubscribe(); - await post({ ...payload, round: 1 }); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); - const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - expect(listTelemetry()).toEqual([{ ...payload, round: 1, receivedAt: expect.any(String), timing: expect.any(Object) }]); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - expect(getServer(payload.server)?.last_players).toBeNull(); - const { GET } = await import('../../src/routes/api/events/servers/+server.js'); - const reader = (await GET()).body!.getReader(); - await reader.read(); - const snapshot = new TextDecoder().decode((await reader.read()).value); - expect(snapshot).toContain('"round":1'); - expect(snapshot).toContain('"headshots":50'); - expect(snapshot).toContain('"guid":"12345678901234560"'); - await reader.cancel(); -}); +it.each(['round_change', 'heartbeat'] as const)( + 'persists %s stats, broadcasts updates, and leaves RCON safety data untouched', + async (event) => { + const payload = gameEvent(event, 17); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const listener = vi.fn(); + const unsubscribe = subscribe(listener); + expect(await (await post(payload)).json()).toEqual({ + ok: true, + telemetry: { ...payload, receivedAt: expect.any(String), timing: expect.any(Object) } + }); + expect(listener).toHaveBeenCalledWith({ + type: 'server-telemetry', + entries: [{ ...payload, receivedAt: expect.any(String), timing: expect.any(Object) }] + }); + unsubscribe(); + await post({ ...payload, round: 1 }); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + expect(listTelemetry()).toEqual([ + { ...payload, round: 1, receivedAt: expect.any(String), timing: expect.any(Object) } + ]); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(payload.server)?.last_players).toBeNull(); + const { GET } = await import('../../src/routes/api/events/servers/+server.js'); + const reader = (await GET()).body!.getReader(); + await reader.read(); + const snapshot = new TextDecoder().decode((await reader.read()).value); + expect(snapshot).toContain('"round":1'); + expect(snapshot).toContain('"headshots":50'); + expect(snapshot).toContain('"guid":"12345678901234560"'); + await reader.cancel(); + } +); it('accepts all four events without an active beta and updates match lifecycle', async () => { const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); const start = gameEvent('game_start'); expect((await post(start)).status).toBe(200); - expect(listTelemetry()).toEqual([{ ...start, receivedAt: expect.any(String), timing: expect.any(Object) }]); + expect(listTelemetry()).toEqual([ + { ...start, receivedAt: expect.any(String), timing: expect.any(Object) } + ]); expect(getServer(start.server)?.game_started_at).toBeTruthy(); expect(getServer(start.server)?.game_ended_at).toBeNull(); expect((await post(payload)).status).toBe(200); @@ -180,38 +265,72 @@ it('accepts all four events without an active beta and updates match lifecycle', expect((await post(gameEvent('game_end', 0, 0))).status).toBe(200); }); -it.each(['game_start', 'round_change', 'heartbeat', 'game_end'] as const)('preserves %s player pings through ingestion, streaming and reload', async event => { - const payload = gameEvent(event, 17); - payload.players = payload.players.map((player, i) => ({ ...player, ping: [0, 42, 999, null][i] })); - const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); - const listener = vi.fn(); - const unsubscribe = subscribe(listener); - try { - const response = await post(payload); - expect(response.status).toBe(200); - expect((await response.json()).telemetry.players).toEqual(payload.players); - expect(listener).toHaveBeenCalledWith({ type: 'server-telemetry', entries: [expect.objectContaining({ players: payload.players })] }); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); - const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - const { telemetryReport } = await import('../../src/lib/server/fleet/status-evidence.js'); - expect(listTelemetry()[0].players).toEqual(payload.players); - expect(telemetryReport(payload.server)?.players).toEqual(payload.players); - } finally { unsubscribe(); } -}); - -it.each([-1, 1.5, '42', false, {}, [], Number.MAX_SAFE_INTEGER + 1])('rejects invalid player ping: %j', async ping => { - for (const event of ['game_start', 'heartbeat'] as const) { - const body = gameEvent(event, 17, 1); - expect((await post({ ...body, players: [{ ...body.players[0], ping }] })).status).toBe(400); +it.each(['game_start', 'round_change', 'heartbeat', 'game_end'] as const)( + 'preserves %s player pings through ingestion, streaming and reload', + async (event) => { + const payload = gameEvent(event, 17); + payload.players = payload.players.map((player, i) => ({ + ...player, + ping: [0, 42, 999, null][i] + })); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const listener = vi.fn(); + const unsubscribe = subscribe(listener); + try { + const response = await post(payload); + expect(response.status).toBe(200); + expect((await response.json()).telemetry.players).toEqual(payload.players); + expect(listener).toHaveBeenCalledWith({ + type: 'server-telemetry', + entries: [expect.objectContaining({ players: payload.players })] + }); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + const { telemetryReport } = + await import('../../src/lib/server/fleet/status-evidence.js'); + expect(listTelemetry()[0].players).toEqual(payload.players); + expect(telemetryReport(payload.server)?.players).toEqual(payload.players); + } finally { + unsubscribe(); + } } -}); +); -it.each([null, [], {}, ...['boot', 'session_start', 'match_end', 'feature', 'other'].map(event => ({ ...payload, event })), - { ...payload, map: '' }, { ...payload, map: undefined }, { ...payload, round: -1 }, - { ...payload, round: 1.5 }, { ...payload, round: '17' }, { ...payload, players: 4 }, { ...payload, players: null }, - { ...payload, players: [null] }, { ...payload, players: [{ name: 'Josh', guid: '123' }] }, - ...['name', 'guid', 'kills', 'headshots', 'downs', 'revives', 'score'].map(field => ({ ...payload, players: [{ ...roster(1)[0], [field]: null }] })), +it.each([-1, 1.5, '42', false, {}, [], Number.MAX_SAFE_INTEGER + 1])( + 'rejects invalid player ping: %j', + async (ping) => { + for (const event of ['game_start', 'heartbeat'] as const) { + const body = gameEvent(event, 17, 1); + expect((await post({ ...body, players: [{ ...body.players[0], ping }] })).status).toBe( + 400 + ); + } + } +); + +it.each([ + null, + [], + {}, + ...['boot', 'session_start', 'match_end', 'feature', 'other'].map((event) => ({ + ...payload, + event + })), + { ...payload, map: '' }, + { ...payload, map: undefined }, + { ...payload, round: -1 }, + { ...payload, round: 1.5 }, + { ...payload, round: '17' }, + { ...payload, players: 4 }, + { ...payload, players: null }, + { ...payload, players: [null] }, + { ...payload, players: [{ name: 'Josh', guid: '123' }] }, + ...['name', 'guid', 'kills', 'headshots', 'downs', 'revives', 'score'].map((field) => ({ + ...payload, + players: [{ ...roster(1)[0], [field]: null }] + })), + // eslint-disable-next-line no-loss-of-precision -- Deliberately reject GUIDs decoded as unsafe JSON numbers. { ...payload, players: [{ ...roster(1)[0], guid: 12345678901234567 }] }, { ...payload, players: [{ ...roster(1)[0], score: -1 }] } ])('rejects invalid telemetry: %j', async (body) => { @@ -237,15 +356,27 @@ it('rejects unknown and destroyed servers and non-JSON requests', async () => { it('rejects malformed JSON', async () => { const { POST } = await import('../../src/routes/api/telemetry/+server.js'); - expect((await POST({ request: new Request('http://localhost/api/telemetry', { - method: 'POST', headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, body: '{' - }) })).status).toBe(400); + expect( + ( + await POST({ + request: new Request('http://localhost/api/telemetry', { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...telemetryAuthHeaders() }, + body: '{' + }) + }) + ).status + ).toBe(400); }); it('migrates old round-only storage without fabricating player identities', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - db().exec('DROP TABLE server_telemetry; CREATE TABLE server_telemetry (server TEXT PRIMARY KEY, round INTEGER NOT NULL, players INTEGER NOT NULL, received_at TEXT NOT NULL)'); - db().prepare('INSERT INTO server_telemetry VALUES (?, ?, ?, ?)').run(payload.server, 99, 4, new Date().toISOString()); + db().exec( + 'DROP TABLE server_telemetry; CREATE TABLE server_telemetry (server TEXT PRIMARY KEY, round INTEGER NOT NULL, players INTEGER NOT NULL, received_at TEXT NOT NULL)' + ); + db() + .prepare('INSERT INTO server_telemetry VALUES (?, ?, ?, ?)') + .run(payload.server, 99, 4, new Date().toISOString()); closeDb(); const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); expect(listTelemetry()).toEqual([]); diff --git a/tests/unit/time-zone.test.ts b/tests/unit/time-zone.test.ts index 0f26129..531225e 100644 --- a/tests/unit/time-zone.test.ts +++ b/tests/unit/time-zone.test.ts @@ -1,11 +1,24 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { get } from 'svelte/store'; -import { availableTimeZones, createTimeFormatter, DEFAULT_TIME_ZONE, validTimeZone, timestampISO } from '../../src/lib/time-zone.js'; -import { startTimeZonePreference, timeFormat, timeZone, timeZonePersistenceError, TIME_ZONE_STORAGE_KEY } from '../../src/lib/ui/stores/time-zone.js'; +import { + availableTimeZones, + createTimeFormatter, + DEFAULT_TIME_ZONE, + validTimeZone, + timestampISO +} from '../../src/lib/time-zone.js'; +import { + startTimeZonePreference, + timeFormat, + timeZone, + timeZonePersistenceError, + TIME_ZONE_STORAGE_KEY +} from '../../src/lib/ui/stores/time-zone.js'; let stop: (() => void) | undefined; afterEach(() => { - stop?.(); stop = undefined; + stop?.(); + stop = undefined; timeZone.set(DEFAULT_TIME_ZONE); vi.unstubAllGlobals(); vi.restoreAllMocks(); @@ -16,7 +29,9 @@ describe('timezone formatting', () => { it('uses New York by default, including when a saved zone is invalid', () => { expect(get(timeZone)).toBe('America/New_York'); - expect(createTimeFormatter('invalid').dateTime('2026-01-15T12:00:00Z')).toBe('Jan 15, 2026, 7:00:00 AM EST'); + expect(createTimeFormatter('invalid').dateTime('2026-01-15T12:00:00Z')).toBe( + 'Jan 15, 2026, 7:00:00 AM EST' + ); expect(ny.dateTime('2026-07-15T12:00:00Z')).toBe('Jul 15, 2026, 8:00:00 AM EDT'); }); @@ -44,7 +59,12 @@ describe('timezone formatting', () => { it('interprets SQLite and unzoned ISO timestamps as UTC while preserving explicit offsets', () => { const expected = ny.dateTime('2026-01-15T12:00:00Z'); - for (const at of ['2026-01-15 12:00:00', '2026-01-15T12:00:00', '2026-01-15 12:00:00.123', '2026-01-15T07:00:00-05:00']) { + for (const at of [ + '2026-01-15 12:00:00', + '2026-01-15T12:00:00', + '2026-01-15 12:00:00.123', + '2026-01-15T07:00:00-05:00' + ]) { expect(ny.dateTime(at)).toBe(expected); } }); @@ -70,9 +90,18 @@ describe('timezone formatting', () => { }); it('uses timezone calendar dates for yesterday across DST and year boundaries', () => { - expect(ny.relative('2026-11-01T04:30:00Z', Date.parse('2026-11-02T05:15:00Z'))).toBe('Yesterday, 12:30 AM'); - expect(ny.relative('2025-12-31T05:00:00Z', Date.parse('2026-01-01T06:00:00Z'))).toBe('Yesterday, 12:00 AM'); - expect(createTimeFormatter('invalid').relative('2026-10-05T17:00:00Z', Date.parse('2026-10-07T18:00:00Z'))).toBe('Oct 5, 2026, 1:00 PM EDT'); + expect(ny.relative('2026-11-01T04:30:00Z', Date.parse('2026-11-02T05:15:00Z'))).toBe( + 'Yesterday, 12:30 AM' + ); + expect(ny.relative('2025-12-31T05:00:00Z', Date.parse('2026-01-01T06:00:00Z'))).toBe( + 'Yesterday, 12:00 AM' + ); + expect( + createTimeFormatter('invalid').relative( + '2026-10-05T17:00:00Z', + Date.parse('2026-10-07T18:00:00Z') + ) + ).toBe('Oct 5, 2026, 1:00 PM EDT'); }); it('offers valid zones including New York and UTC, with an older-browser fallback', () => { @@ -82,9 +111,13 @@ describe('timezone formatting', () => { for (const value of [null, '', 'invalid', 123]) expect(validTimeZone(value)).toBe(false); vi.spyOn(Intl, 'supportedValuesOf').mockImplementation(() => ['Europe/London']); expect(availableTimeZones()).toEqual(['America/New_York', 'Europe/London', 'UTC']); - vi.stubGlobal('Intl', new Proxy(Intl, { - get: (target, key) => key === 'supportedValuesOf' ? undefined : Reflect.get(target, key) - })); + vi.stubGlobal( + 'Intl', + new Proxy(Intl, { + get: (target, key) => + key === 'supportedValuesOf' ? undefined : Reflect.get(target, key) + }) + ); expect(availableTimeZones()).toContain('Asia/Kolkata'); expect(availableTimeZones().every(validTimeZone)).toBe(true); }); @@ -94,13 +127,17 @@ function browser(saved?: string) { const values = new Map(saved ? [[TIME_ZONE_STORAGE_KEY, saved]] : []); const storage = { getItem: vi.fn((key: string) => values.get(key) ?? null), - setItem: vi.fn((key: string, value: string) => { values.set(key, value); }) + setItem: vi.fn((key: string, value: string) => { + values.set(key, value); + }) }; const window = new EventTarget(); vi.stubGlobal('localStorage', storage); vi.stubGlobal('window', window); function change(newValue: string | null, key: string | null = TIME_ZONE_STORAGE_KEY) { - window.dispatchEvent(Object.assign(new Event('storage'), { storageArea: storage, key, newValue })); + window.dispatchEvent( + Object.assign(new Event('storage'), { storageArea: storage, key, newValue }) + ); } return { values, storage, change }; } @@ -110,21 +147,27 @@ describe('timezone preference', () => { const { values } = browser('UTC'); stop = startTimeZonePreference(); let shown = ''; - const unsubscribe = timeFormat.subscribe(format => { shown = format.time('2026-01-15T12:00:00Z'); }); + const unsubscribe = timeFormat.subscribe((format) => { + shown = format.time('2026-01-15T12:00:00Z'); + }); expect(shown).toBe('12:00:00 PM UTC'); timeZone.set('America/New_York'); expect(shown).toBe('7:00:00 AM EST'); expect(values.get(TIME_ZONE_STORAGE_KEY)).toBe('America/New_York'); - stop(); stop = startTimeZonePreference(); + stop(); + stop = startTimeZonePreference(); expect(get(timeZone)).toBe('America/New_York'); unsubscribe(); }); - it.each([undefined, 'invalid'])('defaults safely with a missing or corrupt preference (%s)', saved => { - browser(saved); - stop = startTimeZonePreference(); - expect(get(timeZone)).toBe(DEFAULT_TIME_ZONE); - }); + it.each([undefined, 'invalid'])( + 'defaults safely with a missing or corrupt preference (%s)', + (saved) => { + browser(saved); + stop = startTimeZonePreference(); + expect(get(timeZone)).toBe(DEFAULT_TIME_ZONE); + } + ); it('syncs changes from other tabs and removes listeners on cleanup', () => { const { change } = browser(); @@ -135,16 +178,23 @@ describe('timezone preference', () => { expect(get(timeZone)).toBe('Asia/Tokyo'); change(null, null); expect(get(timeZone)).toBe(DEFAULT_TIME_ZONE); - stop(); stop = undefined; + stop(); + stop = undefined; change('UTC'); expect(get(timeZone)).toBe(DEFAULT_TIME_ZONE); }); it('keeps the preference usable when browser storage is blocked', () => { const { storage } = browser(); - storage.getItem.mockImplementation(() => { throw new Error('Storage blocked'); }); - storage.setItem.mockImplementation(() => { throw new Error('Storage blocked'); }); - expect(() => { stop = startTimeZonePreference(); }).not.toThrow(); + storage.getItem.mockImplementation(() => { + throw new Error('Storage blocked'); + }); + storage.setItem.mockImplementation(() => { + throw new Error('Storage blocked'); + }); + expect(() => { + stop = startTimeZonePreference(); + }).not.toThrow(); timeZone.set('UTC'); expect(get(timeFormat).time('2026-01-15T12:00:00Z')).toBe('12:00:00 PM UTC'); expect(get(timeZonePersistenceError)).toBe(true); diff --git a/tests/unit/ui-activity.test.ts b/tests/unit/ui-activity.test.ts index a6486d0..e2ef155 100644 --- a/tests/unit/ui-activity.test.ts +++ b/tests/unit/ui-activity.test.ts @@ -1,59 +1,121 @@ import { beforeEach, afterEach, expect, it, vi } from 'vitest'; import { render } from 'svelte/server'; +import type { ComponentProps } from 'svelte'; import { writable, type Writable } from 'svelte/store'; import { page } from '$app/stores'; import ActivityLog from '../../src/lib/ui/components/ActivityLog.svelte'; import ActivityPage from '../../src/routes/activity/+page.svelte'; -import { activityApiHref, activityHistoryHref, activityQuery, activityTimeLabel } from '../../src/lib/ui/activity-history.js'; +import { + activityApiHref, + activityHistoryHref, + activityQuery, + activityTimeLabel +} from '../../src/lib/ui/activity-history.js'; import type { ActivityPage as ActivityReport, ActivityRow } from '../../src/lib/activity.js'; vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/activity') }) })); vi.mock('$app/navigation', () => ({ goto: vi.fn() })); -vi.mock('$ui/stores/fleet.js', () => ({ fleet: writable({ servers: [], operations: [], receivedAt: 0 }) })); +vi.mock('$ui/stores/fleet.js', () => ({ + fleet: writable({ servers: [], operations: [], receivedAt: 0 }) +})); vi.mock('$ui/stores/servers.js', () => ({ uiNow: writable(Date.parse('2026-10-07T18:00:00Z')) })); -const row: ActivityRow = { id: 'operation-1', at: '2026-10-07T17:59:45Z', type: 'restart', source: 'maintenance', sources: ['maintenance', 'campaign'], state: 'succeeded', rawState: 'done', title: 'Restart server', detail: 'Verified game readiness.', server: 't6_zombie_town_1', serverDeletedAt: null, hostId: 'host-a', hostName: 'Dallas 1', href: '/provisioning/operation-1' }; -const report = (): ActivityReport => ({ rows: [{ ...row }], total: 52, limit: 25, offset: 0, query: { limit: 25, offset: 0 } }); -function scope(path = '/activity') { (page as unknown as Writable<{ url: URL }>).set({ url: new URL(`http://localhost${path}`) }); } -beforeEach(() => { scope(); vi.useFakeTimers(); vi.setSystemTime(new Date('2026-10-07T18:00:00Z')); }); +const row: ActivityRow = { + id: 'operation-1', + at: '2026-10-07T17:59:45Z', + type: 'restart', + source: 'maintenance', + sources: ['maintenance', 'campaign'], + state: 'succeeded', + rawState: 'done', + title: 'Restart server', + detail: 'Verified game readiness.', + server: 't6_zombie_town_1', + serverDeletedAt: null, + hostId: 'host-a', + hostName: 'Dallas 1', + href: '/provisioning/operation-1' +}; +const report = (): ActivityReport => ({ + rows: [{ ...row }], + total: 52, + limit: 25, + offset: 0, + query: { limit: 25, offset: 0 } +}); +function scope(path = '/activity') { + (page as unknown as Writable<{ url: URL }>).set({ url: new URL(`http://localhost${path}`) }); +} +beforeEach(() => { + scope(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-10-07T18:00:00Z')); +}); afterEach(() => vi.useRealTimers()); it('sends only supported normalized filters and enforces scoped identities', () => { - const url = new URL('https://example.test/servers/town?tab=history&server=wrong&source=host&state=failed&offset=50&returnTo=%2F'); + const url = new URL( + 'https://example.test/servers/town?tab=history&server=wrong&source=host&state=failed&offset=50&returnTo=%2F' + ); const href = activityApiHref(url, { server: 't6_zombie_town_1', source: 'maintenance' }); const params = new URL(href, url).searchParams; - expect(params.get('server')).toBe('t6_zombie_town_1'); expect(params.get('source')).toBe('maintenance'); - expect(params.get('state')).toBe('failed'); expect(params.get('offset')).toBe('50'); expect(params.get('limit')).toBe('25'); - expect(params.has('tab')).toBe(false); expect(params.has('returnTo')).toBe(false); - expect(activityQuery(new URL('https://example.test/activity?type=unknown&days=0')).type).toBeUndefined(); + expect(params.get('server')).toBe('t6_zombie_town_1'); + expect(params.get('source')).toBe('maintenance'); + expect(params.get('state')).toBe('failed'); + expect(params.get('offset')).toBe('50'); + expect(params.get('limit')).toBe('25'); + expect(params.has('tab')).toBe(false); + expect(params.has('returnTo')).toBe(false); + expect( + activityQuery(new URL('https://example.test/activity?type=unknown&days=0')).type + ).toBeUndefined(); }); it('resets pagination when filtering and preserves the containing page, hash, and return context', () => { - const current = new URL('https://example.test/servers/town?tab=history&offset=50&source=campaign&returnTo=%2F%3Ffilter%3Dattention#history'); - const filtered = new URL(activityHistoryHref(current, { state: 'failed', q: 'Dallas lobby' }), current); - expect(filtered.searchParams.get('offset')).toBeNull(); expect(filtered.searchParams.get('tab')).toBe('history'); - expect(filtered.searchParams.get('source')).toBe('campaign'); expect(filtered.searchParams.get('returnTo')).toBe('/?filter=attention'); - expect(filtered.hash).toBe('#history'); expect(filtered.searchParams.get('q')).toBe('Dallas lobby'); - expect(new URL(activityHistoryHref(filtered, { offset: '25' }), current).searchParams.get('state')).toBe('failed'); + const current = new URL( + 'https://example.test/servers/town?tab=history&offset=50&source=campaign&returnTo=%2F%3Ffilter%3Dattention#history' + ); + const filtered = new URL( + activityHistoryHref(current, { state: 'failed', q: 'Dallas lobby' }), + current + ); + expect(filtered.searchParams.get('offset')).toBeNull(); + expect(filtered.searchParams.get('tab')).toBe('history'); + expect(filtered.searchParams.get('source')).toBe('campaign'); + expect(filtered.searchParams.get('returnTo')).toBe('/?filter=attention'); + expect(filtered.hash).toBe('#history'); + expect(filtered.searchParams.get('q')).toBe('Dallas lobby'); + expect( + new URL(activityHistoryHref(filtered, { offset: '25' }), current).searchParams.get('state') + ).toBe('failed'); }); it('keeps provisioning excluded from log polling after filtering or clearing filters', () => { const url = new URL('http://localhost/activity?type=create&excludeProvisioning=false'); const scope = { excludeProvisioning: true }; - expect(new URL(activityApiHref(url, scope), url).searchParams.get('excludeProvisioning')).toBe('true'); + expect(new URL(activityApiHref(url, scope), url).searchParams.get('excludeProvisioning')).toBe( + 'true' + ); const cleared = new URL(activityHistoryHref(url, { type: '', source: '', offset: '' }), url); expect(activityQuery(cleared, scope).excludeProvisioning).toBe(true); - expect(activityQuery(new URL('http://localhost/servers/town?tab=history')).excludeProvisioning).toBeUndefined(); + expect( + activityQuery(new URL('http://localhost/servers/town?tab=history')).excludeProvisioning + ).toBeUndefined(); }); it('keeps maintenance excluded from log polling while preserving the Maintenance history scope', () => { const url = new URL('http://localhost/activity?source=maintenance&excludeMaintenance=false'); const logScope = { excludeProvisioning: true, excludeMaintenance: true }; - expect(new URL(activityApiHref(url, logScope), url).searchParams.get('excludeMaintenance')).toBe('true'); + expect( + new URL(activityApiHref(url, logScope), url).searchParams.get('excludeMaintenance') + ).toBe('true'); const cleared = new URL(activityHistoryHref(url, { type: '', source: '', offset: '' }), url); expect(activityQuery(cleared, logScope).excludeMaintenance).toBe(true); const maintenance = new URL('http://localhost/maintenance?view=activity'); - const query = new URL(activityApiHref(maintenance, { source: 'maintenance', excludeMaintenance: false }), maintenance).searchParams; + const query = new URL( + activityApiHref(maintenance, { source: 'maintenance', excludeMaintenance: false }), + maintenance + ).searchParams; expect(query.get('source')).toBe('maintenance'); expect(query.has('excludeMaintenance')).toBe(false); }); @@ -63,56 +125,107 @@ it('shows relative recent timestamps and preserves accurate older dates and inva expect(activityTimeLabel('2026-10-07 17:59:45', 'America/New_York', now)).toBe('15 s ago'); expect(activityTimeLabel('2026-10-07T17:50:00Z', 'America/New_York', now)).toBe('10 min ago'); expect(activityTimeLabel('2026-10-07T16:00:00Z', 'America/New_York', now)).toBe('2 h ago'); - expect(activityTimeLabel('2026-10-06T17:00:00Z', 'America/New_York', now)).toBe('Yesterday, 1:00 PM'); + expect(activityTimeLabel('2026-10-06T17:00:00Z', 'America/New_York', now)).toBe( + 'Yesterday, 1:00 PM' + ); expect(activityTimeLabel('not a date', 'UTC', now)).toBe('Time unavailable'); - expect(activityTimeLabel('2026-10-08T18:00:00Z', 'America/New_York', now)).toBe('Oct 8, 2026, 2:00 PM EDT'); + expect(activityTimeLabel('2026-10-08T18:00:00Z', 'America/New_York', now)).toBe( + 'Oct 8, 2026, 2:00 PM EDT' + ); }); it('renders one page heading with provisioning management and filters removed', () => { - const body = render(ActivityPage, { props: { data: { activity: report(), user: { role: 'admin' }, setupRequired: false } as any } }).body; - expect(body.match(/]/g)).toHaveLength(1); expect(body).not.toContain('Manage servers'); expect(body).not.toContain('href="/provisioning"'); - expect(body).not.toContain('`); expect(body).toContain('Fleet actions'); expect(body).toContain('Select all eligible'); expect(body).toContain('0 selected'); - const submit = [...body.matchAll(/]*>[\s\S]*?<\/button>/g)].find(match => match[0].includes('Restart 0 servers'))?.[0]; + const submit = [...body.matchAll(/]*>[\s\S]*?<\/button>/g)].find((match) => + match[0].replace(/\s+/g, ' ').includes('Restart 0 servers') + )?.[0]; expect(submit).toContain('disabled'); }); }); diff --git a/tests/unit/ui-freshness.test.ts b/tests/unit/ui-freshness.test.ts index 125a63c..8d321ab 100644 --- a/tests/unit/ui-freshness.test.ts +++ b/tests/unit/ui-freshness.test.ts @@ -7,8 +7,14 @@ afterEach(() => vi.useRealTimers()); describe('quiet refresh feedback', () => { it('shows relative receipt time with an exact tooltip without announcing every tick', () => { - vi.useFakeTimers(); vi.setSystemTime('2026-10-07T12:03:00Z'); - const body = render(Freshness, { props: { updatedAt: Date.parse('2026-10-07T12:00:00Z'), intervalLabel: 'Updates every 5 seconds' } }).body; + vi.useFakeTimers(); + vi.setSystemTime('2026-10-07T12:03:00Z'); + const body = render(Freshness, { + props: { + updatedAt: Date.parse('2026-10-07T12:00:00Z'), + intervalLabel: 'Updates every 5 seconds' + } + }).body; expect(body).toContain('Live'); expect(body).toContain('3 min ago'); expect(body).toContain('datetime="2026-10-07T12:00:00.000Z"'); @@ -18,13 +24,23 @@ describe('quiet refresh feedback', () => { }); it('does not claim live updates or an automatic schedule while paused', () => { - const body = render(Freshness, { props: { paused: true, updatedAt: Date.now(), intervalLabel: 'Updates every 5 seconds' } }).body; + const body = render(Freshness, { + props: { paused: true, updatedAt: Date.now(), intervalLabel: 'Updates every 5 seconds' } + }).body; expect(body).toContain('Paused'); expect(body).not.toMatch(/Live|Updates every/); }); it('replaces live claims with a retry on failures and treats invalid timestamps as unknown', () => { - const body = render(Freshness, { props: { updatedAt: Date.now(), error: 'Log updates interrupted.', intervalLabel: 'Updates every 5 seconds', onRetry: () => {}, retryDisabled: true } }).body; + const body = render(Freshness, { + props: { + updatedAt: Date.now(), + error: 'Log updates interrupted.', + intervalLabel: 'Updates every 5 seconds', + onRetry: () => {}, + retryDisabled: true + } + }).body; expect(body).toContain('Log updates interrupted.'); expect(body).toMatch(/]+disabled[^>]*>Retry<\/button>/); expect(body).not.toMatch(/Live|Updates every/); @@ -37,14 +53,18 @@ describe('quiet refresh feedback', () => { describe('readable game logs', () => { it('strips digit colour directives while preserving log whitespace and other carets', () => { const raw = ' [12:30] ^1Red^7 player\t^2joined^0\r\npath ^name, exponent x^y, trailing ^'; - expect(stripGameColorCodes(raw)).toBe(' [12:30] Red player\tjoined\r\npath ^name, exponent x^y, trailing ^'); + expect(stripGameColorCodes(raw)).toBe( + ' [12:30] Red player\tjoined\r\npath ^name, exponent x^y, trailing ^' + ); expect(raw).toContain('^1Red^7'); }); it('allows searches across embedded colour changes without mutating downloadable source lines', () => { const raw = ['^1Con^2nection^7 accepted', ' Player ^3Jamie^7 joined ']; const displayed = raw.map(stripGameColorCodes); - expect(displayed.filter(line => line.toLowerCase().includes('connection'))).toEqual(['Connection accepted']); + expect(displayed.filter((line) => line.toLowerCase().includes('connection'))).toEqual([ + 'Connection accepted' + ]); expect(displayed.join('\n')).toBe('Connection accepted\n Player Jamie joined '); expect(raw.join('\n')).toBe('^1Con^2nection^7 accepted\n Player ^3Jamie^7 joined '); }); diff --git a/tests/unit/ui-game-file-transfer.test.ts b/tests/unit/ui-game-file-transfer.test.ts index a539ca1..332615c 100644 --- a/tests/unit/ui-game-file-transfer.test.ts +++ b/tests/unit/ui-game-file-transfer.test.ts @@ -5,10 +5,24 @@ import { gameFileTransferProgress } from '../../src/lib/ui/game-file-transfer.js const now = Date.parse('2026-10-04T16:00:00Z'); function cache(changes: Partial = {}): HostLibraryCache { return { - hostId: 'host-1', libraryVersionId: 't5-upload', status: 'downloading', bytesDownloaded: 200, bytesVerified: 0, bytesTotal: 1000, - manifestSha256: null, operationId: 'operation-1', error: null, updatedAt: new Date(now).toISOString(), - transfer: { phase: 'downloading', startedAt: new Date(now - 10_000).toISOString(), elapsedSeconds: 10, - phaseElapsedSeconds: 10, bytesPerSecond: 20, etaSeconds: 40 }, + hostId: 'host-1', + libraryVersionId: 't5-upload', + status: 'downloading', + bytesDownloaded: 200, + bytesVerified: 0, + bytesTotal: 1000, + manifestSha256: null, + operationId: 'operation-1', + error: null, + updatedAt: new Date(now).toISOString(), + transfer: { + phase: 'downloading', + startedAt: new Date(now - 10_000).toISOString(), + elapsedSeconds: 10, + phaseElapsedSeconds: 10, + bytesPerSecond: 20, + etaSeconds: 40 + }, ...changes }; } @@ -16,59 +30,137 @@ function cache(changes: Partial = {}): HostLibraryCache { describe('game file transfer presentation', () => { it('ticks elapsed time without inventing downloaded bytes or counting down an old estimate', () => { expect(gameFileTransferProgress(cache(), true, now + 5000)).toMatchObject({ - phase: 'downloading', phaseBytes: 200, downloaded: 200, total: 1000, - elapsedSeconds: 15, bytesPerSecond: 20, etaSeconds: 40, live: true + phase: 'downloading', + phaseBytes: 200, + downloaded: 200, + total: 1000, + elapsedSeconds: 15, + bytesPerSecond: 20, + etaSeconds: 40, + live: true }); }); it('switches progress and the estimate to verification without resetting total elapsed', () => { - const verifying = cache({ status: 'verifying', bytesDownloaded: 1000, bytesVerified: 100, - transfer: { phase: 'verifying', startedAt: new Date(now - 60_000).toISOString(), elapsedSeconds: 60, - phaseElapsedSeconds: 2, bytesPerSecond: 50, etaSeconds: 18 } }); + const verifying = cache({ + status: 'verifying', + bytesDownloaded: 1000, + bytesVerified: 100, + transfer: { + phase: 'verifying', + startedAt: new Date(now - 60_000).toISOString(), + elapsedSeconds: 60, + phaseElapsedSeconds: 2, + bytesPerSecond: 50, + etaSeconds: 18 + } + }); expect(gameFileTransferProgress(verifying, true, now + 1000)).toMatchObject({ - phase: 'verifying', phaseBytes: 100, downloaded: 1000, elapsedSeconds: 61, bytesPerSecond: 50, etaSeconds: 18 + phase: 'verifying', + phaseBytes: 100, + downloaded: 1000, + elapsedSeconds: 61, + bytesPerSecond: 50, + etaSeconds: 18 }); }); it('freezes at the reported duration and hides estimates while offline', () => { - expect(gameFileTransferProgress(cache(), false, now + 12_000)).toMatchObject({ elapsedSeconds: 10, etaSeconds: null, bytesPerSecond: null, live: false }); - }); - - it('stops the clock and removes the estimate when reports become stale', () => { - expect(gameFileTransferProgress(cache(), true, now + 30_001)).toMatchObject({ elapsedSeconds: 40, etaSeconds: null, bytesPerSecond: null, stale: true }); - expect(gameFileTransferProgress(cache(), true, now + 600_000).elapsedSeconds).toBe(40); - }); - - it.each(['ready', 'outdated', 'failed'] as const)('keeps a final duration for %s copies', status => { - expect(gameFileTransferProgress(cache({ status }), true, now + 600_000)).toMatchObject({ - elapsedSeconds: 10, bytesPerSecond: null, etaSeconds: null, active: false, stale: false + expect(gameFileTransferProgress(cache(), false, now + 12_000)).toMatchObject({ + elapsedSeconds: 10, + etaSeconds: null, + bytesPerSecond: null, + live: false }); }); - it('does not invent a timer for queued downloads or old agents', () => { - expect(gameFileTransferProgress(cache({ transfer: undefined }), true, now + 60_000)).toMatchObject({ elapsedSeconds: null, etaSeconds: null, phaseBytes: 200 }); - expect(gameFileTransferProgress(cache({ status: 'verifying', transfer: undefined, bytesVerified: undefined, bytesDownloaded: 1000 }), true, now)) - .toMatchObject({ elapsedSeconds: null, etaSeconds: null, phase: 'verifying', phaseBytes: null }); + it('stops the clock and removes the estimate when reports become stale', () => { + expect(gameFileTransferProgress(cache(), true, now + 30_001)).toMatchObject({ + elapsedSeconds: 40, + etaSeconds: null, + bytesPerSecond: null, + stale: true + }); + expect(gameFileTransferProgress(cache(), true, now + 600_000).elapsedSeconds).toBe(40); }); - it.each([0, null, Number.NaN, Number.POSITIVE_INFINITY, -1])('hides estimates for an unavailable transfer rate (%s)', bytesPerSecond => { - const current = cache(); - current.transfer!.bytesPerSecond = bytesPerSecond; - expect(gameFileTransferProgress(current, true, now)).toMatchObject({ etaSeconds: null, bytesPerSecond: null }); + it.each(['ready', 'outdated', 'failed'] as const)( + 'keeps a final duration for %s copies', + (status) => { + expect(gameFileTransferProgress(cache({ status }), true, now + 600_000)).toMatchObject({ + elapsedSeconds: 10, + bytesPerSecond: null, + etaSeconds: null, + active: false, + stale: false + }); + } + ); + + it('does not invent a timer for queued downloads or old agents', () => { + expect( + gameFileTransferProgress(cache({ transfer: undefined }), true, now + 60_000) + ).toMatchObject({ elapsedSeconds: null, etaSeconds: null, phaseBytes: 200 }); + expect( + gameFileTransferProgress( + cache({ + status: 'verifying', + transfer: undefined, + bytesVerified: undefined, + bytesDownloaded: 1000 + }), + true, + now + ) + ).toMatchObject({ + elapsedSeconds: null, + etaSeconds: null, + phase: 'verifying', + phaseBytes: null + }); }); + it.each([0, null, Number.NaN, Number.POSITIVE_INFINITY, -1])( + 'hides estimates for an unavailable transfer rate (%s)', + (bytesPerSecond) => { + const current = cache(); + current.transfer!.bytesPerSecond = bytesPerSecond; + expect(gameFileTransferProgress(current, true, now)).toMatchObject({ + etaSeconds: null, + bytesPerSecond: null + }); + } + ); + it('does not show a misleading estimate before any bytes arrive or after all bytes arrive', () => { for (const bytesDownloaded of [0, 1000]) { - expect(gameFileTransferProgress(cache({ bytesDownloaded }), true, now)).toMatchObject({ etaSeconds: null, bytesPerSecond: null }); + expect(gameFileTransferProgress(cache({ bytesDownloaded }), true, now)).toMatchObject({ + etaSeconds: null, + bytesPerSecond: null + }); } }); it('does not reuse a download estimate after verification starts', () => { - expect(gameFileTransferProgress(cache({ status: 'verifying', bytesDownloaded: 1000, bytesVerified: 50 }), true, now + 5000)) - .toMatchObject({ phase: 'verifying', phaseBytes: 50, etaSeconds: null, elapsedSeconds: 10 }); + expect( + gameFileTransferProgress( + cache({ status: 'verifying', bytesDownloaded: 1000, bytesVerified: 50 }), + true, + now + 5000 + ) + ).toMatchObject({ + phase: 'verifying', + phaseBytes: 50, + etaSeconds: null, + elapsedSeconds: 10 + }); }); it('handles an invalid report timestamp without advancing the timer or displaying estimates', () => { - expect(gameFileTransferProgress(cache({ updatedAt: 'invalid' }), true, now)).toMatchObject({ elapsedSeconds: 10, etaSeconds: null, stale: true }); + expect(gameFileTransferProgress(cache({ updatedAt: 'invalid' }), true, now)).toMatchObject({ + elapsedSeconds: 10, + etaSeconds: null, + stale: true + }); }); }); diff --git a/tests/unit/ui-host-operation-labels.test.ts b/tests/unit/ui-host-operation-labels.test.ts index 2745cf0..5128f0e 100644 --- a/tests/unit/ui-host-operation-labels.test.ts +++ b/tests/unit/ui-host-operation-labels.test.ts @@ -1,30 +1,58 @@ import { describe, expect, it, vi } from 'vitest'; import { render } from 'svelte/server'; +import type { ComponentProps } from 'svelte'; import { writable } from 'svelte/store'; import HostOperations from '../../src/routes/hosts/operations/+page.svelte'; import type { HostOperationSummary } from '../../src/lib/hosts.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/hosts/operations') }), navigating: writable(null) })); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/hosts/operations') }), + navigating: writable(null) +})); vi.mock('$app/navigation', () => ({ beforeNavigate: vi.fn() })); const operation: HostOperationSummary = { - id: 'retained-op', hostId: 'host-1', hostName: 'Dallas 1', kind: 'instance.inspect', - instanceId: 'container-1', generation: 1, state: 'completed', error: null, - serverName: 't6_zombie_town_1', targetPath: null, source: 'maintenance', routine: true, - createdAt: '2026-10-07 17:00:00', updatedAt: '2026-10-07 17:00:00' + id: 'retained-op', + hostId: 'host-1', + hostName: 'Dallas 1', + kind: 'instance.inspect', + instanceId: 'container-1', + generation: 1, + state: 'completed', + error: null, + serverName: 't6_zombie_town_1', + targetPath: null, + source: 'maintenance', + routine: true, + createdAt: '2026-10-07 17:00:00', + updatedAt: '2026-10-07 17:00:00' }; describe('host operation history presentation', () => { it('uses readable operation, server and state labels while keeping identifiers in technical detail', () => { - const body = render(HostOperations, { props: { data: { - history: { operations: [operation], page: 1, pageSize: 25, totalPages: 1, total: 1, filter: 'all' }, - scope: {}, hostName: null - } as any } }).body; + const body = render(HostOperations, { + props: { + data: { + history: { + operations: [operation], + page: 1, + pageSize: 25, + totalPages: 1, + total: 1, + filter: 'all' + }, + scope: {}, + hostName: null + } as ComponentProps['data'] + } + }).body; expect(body).toContain('Check server container'); expect(body).toContain('Town Survival · Lobby 1'); expect(body).toContain('Completed'); expect(body).toContain('Technical details'); - expect(body.indexOf('instance.inspect')).toBeGreaterThan(body.indexOf('Technical details')); + expect(body.indexOf('instance.inspect')).toBeGreaterThan( + body.indexOf('Technical details') + ); expect(body).toContain('retained-op'); expect(body).toContain('datetime="2026-10-07T17:00:00.000Z"'); expect(body).toContain('title="Oct 7, 2026, 1:00:00 PM EDT"'); diff --git a/tests/unit/ui-host-overview.test.ts b/tests/unit/ui-host-overview.test.ts index 09b1bd1..8c22500 100644 --- a/tests/unit/ui-host-overview.test.ts +++ b/tests/unit/ui-host-overview.test.ts @@ -1,16 +1,46 @@ import { expect, it } from 'vitest'; import { render } from 'svelte/server'; import type { Host } from '../../src/lib/hosts.js'; -import { hostListStatus, hostReadingsStale, hostCpuUsage, hostUsedSpace, stableHostOrder } from '../../src/lib/ui/host-overview.js'; +import { + hostListStatus, + hostReadingsStale, + hostCpuUsage, + hostUsedSpace, + stableHostOrder +} from '../../src/lib/ui/host-overview.js'; import HostList from '../../src/lib/ui/components/HostList.svelte'; const now = Date.parse('2026-10-06T12:00:00Z'); function host(id = 'ec-1', changes: Partial = {}): Host { - return { id, name: id, hostname: id, mode: 'active', portMin: 28960, portMax: 28980, - bindAddress: '0.0.0.0', publicAddress: '', minimumFreeBytes: 100, createdAt: new Date(now).toISOString(), - lastHeartbeat: new Date(now).toISOString(), connected: true, connectionState: 'online', readiness: 'ready', readinessReasons: [], - inventory: { cpuUsagePercent: 0, memoryTotal: 1024 ** 3, memoryAvailable: 512 * 1024 ** 2, storageTotal: 10 * 1024 ** 3, storageAvailable: 8 * 1024 ** 3 }, - enrolled: true, credentialActive: true, decommission: null, preparation: { state: 'ready', items: [] }, ...changes }; + return { + id, + name: id, + hostname: id, + mode: 'active', + portMin: 28960, + portMax: 28980, + bindAddress: '0.0.0.0', + publicAddress: '', + minimumFreeBytes: 100, + createdAt: new Date(now).toISOString(), + lastHeartbeat: new Date(now).toISOString(), + connected: true, + connectionState: 'online', + readiness: 'ready', + readinessReasons: [], + inventory: { + cpuUsagePercent: 0, + memoryTotal: 1024 ** 3, + memoryAvailable: 512 * 1024 ** 2, + storageTotal: 10 * 1024 ** 3, + storageAvailable: 8 * 1024 ** 3 + }, + enrolled: true, + credentialActive: true, + decommission: null, + preparation: { state: 'ready', items: [] }, + ...changes + }; } it('does not show an old Ready state as available when the heartbeat becomes stale', () => { @@ -18,45 +48,107 @@ it('does not show an old Ready state as available when the heartbeat becomes sta expect(hostReadingsStale(host(), now + 90_001)).toBe(true); expect(hostListStatus(host(), now + 90_001)).toMatchObject({ key: 'offline', priority: 0 }); expect(hostReadingsStale(host('bad-date', { lastHeartbeat: 'invalid' }), now)).toBe(true); - expect(hostListStatus(host('new', { enrolled: false, connected: false, lastHeartbeat: null }), now).label).toBe('Awaiting enrollment'); + expect( + hostListStatus(host('new', { enrolled: false, connected: false, lastHeartbeat: null }), now) + .label + ).toBe('Awaiting enrollment'); }); it('keeps deliberate disablement and deletion distinct from unexpected outages', () => { - expect(hostListStatus(host('disabled', { mode: 'disabled', connected: false }), now).label).toBe('Disabled'); + expect( + hostListStatus(host('disabled', { mode: 'disabled', connected: false }), now).label + ).toBe('Disabled'); expect(hostListStatus(host('draining', { mode: 'draining' }), now).label).toBe('Draining'); - const deleting = host('deleting', { connected: false, decommission: { operationId: 'delete', stage: 'cleaned', requestedAt: '', updatedAt: '', error: null, retryable: false } }); + const deleting = host('deleting', { + connected: false, + decommission: { + operationId: 'delete', + stage: 'cleaned', + requestedAt: '', + updatedAt: '', + error: null, + retryable: false + } + }); expect(hostListStatus(deleting, now).label).toBe('Deleting'); - deleting.decommission!.stage = 'failed'; deleting.decommission!.error = 'Cleanup did not finish'; - expect(hostListStatus(deleting, now)).toMatchObject({ key: 'attention', label: 'Cleanup failed', detail: 'Cleanup did not finish' }); + deleting.decommission!.stage = 'failed'; + deleting.decommission!.error = 'Cleanup did not finish'; + expect(hostListStatus(deleting, now)).toMatchObject({ + key: 'attention', + label: 'Cleanup failed', + detail: 'Cleanup did not finish' + }); }); it('surfaces setup and update failures without treating ordinary preparation as failure', () => { - const preparing = host('preparing', { preparation: { state: 'preparing', items: [{ kind: 'runtime', label: 'Runtime', state: 'preparing', detail: 'Downloading runtime' }] } }); - expect(hostListStatus(preparing, now)).toMatchObject({ key: 'preparing', detail: 'Downloading runtime' }); - preparing.preparation!.state = 'blocked'; preparing.preparation!.items[0].state = 'blocked'; + const preparing = host('preparing', { + preparation: { + state: 'preparing', + items: [ + { + kind: 'runtime', + label: 'Runtime', + state: 'preparing', + detail: 'Downloading runtime' + } + ] + } + }); + expect(hostListStatus(preparing, now)).toMatchObject({ + key: 'preparing', + detail: 'Downloading runtime' + }); + preparing.preparation!.state = 'blocked'; + preparing.preparation!.items[0].state = 'blocked'; expect(hostListStatus(preparing, now).key).toBe('attention'); - expect(hostListStatus(host('update', { inventory: { agentUpdate: { enabled: true, state: 'failed', currentVersion: '1.0', error: 'Invalid signature' } } }), now)).toMatchObject({ key: 'attention', detail: 'Agent update failed: Invalid signature' }); - expect(hostListStatus(host('revoked', { credentialActive: false }), now)).toMatchObject({ key: 'attention', tab: 'settings' }); + expect( + hostListStatus( + host('update', { + inventory: { + agentUpdate: { + enabled: true, + state: 'failed', + currentVersion: '1.0', + error: 'Invalid signature' + } + } + }), + now + ) + ).toMatchObject({ key: 'attention', detail: 'Agent update failed: Invalid signature' }); + expect(hostListStatus(host('revoked', { credentialActive: false }), now)).toMatchObject({ + key: 'attention', + tab: 'settings' + }); }); it('sorts issues first initially and preserves existing positions when polls change status', () => { - const healthy = host('a'); const offline = host('z', { connected: false }); + const healthy = host('a'); + const offline = host('z', { connected: false }); const order = stableHostOrder([], [healthy, offline], now); expect(order).toEqual(['z', 'a']); - expect(stableHostOrder(order, [host('z'), host('a', { connected: false }), host('new')], now)).toEqual(['z', 'a', 'new']); + expect( + stableHostOrder(order, [host('z'), host('a', { connected: false }), host('new')], now) + ).toEqual(['z', 'a', 'new']); expect(stableHostOrder(order, [healthy], now)).toEqual(['a']); }); it('distinguishes real zero usage from missing or invalid metrics', () => { expect(hostCpuUsage(0)).toBe('0.0%'); - for (const value of [undefined, null, NaN, Infinity, -1, 101]) expect(hostCpuUsage(value)).toBe('Not reported'); + for (const value of [undefined, null, NaN, Infinity, -1, 101]) + expect(hostCpuUsage(value)).toBe('Not reported'); expect(hostUsedSpace(1024, 1024)).toBe('0 B / 1.0 KiB'); expect(hostUsedSpace(1024, undefined)).toBe('Not reported'); expect(hostUsedSpace(1024, 2048)).toBe('Not reported'); }); it('renders compact host links and workload without duplicate identity, setup metadata or deletion actions', () => { - const body = render(HostList, { props: { hosts: [{ ...host(), serverCount: 3, playerCount: 12, unreportedInstances: 0 }], now } }).body; + const body = render(HostList, { + props: { + hosts: [{ ...host(), serverCount: 3, playerCount: 12, unreportedInstances: 0 }], + now + } + }).body; expect(body).toContain('3 assigned servers on ec-1'); expect(body).toContain('>Servers'); expect(body).toContain('>Players'); @@ -68,17 +160,37 @@ it('renders compact host links and workload without duplicate identity, setup me expect(body).not.toContain('Game ports'); expect(body).not.toContain('Automatic setup'); expect(body).not.toContain('Resource readings stale'); - const stale = render(HostList, { props: { hosts: [{ ...host(), serverCount: 3, playerCount: 12, unreportedInstances: 0 }], now: now + 90_001 } }).body; + const stale = render(HostList, { + props: { + hosts: [{ ...host(), serverCount: 3, playerCount: 12, unreportedInstances: 0 }], + now: now + 90_001 + } + }).body; expect(stale).toContain('Resource readings stale'); expect(stale).toContain('Offline'); }); it('marks partial player totals and distinguishes unavailable counts from empty hosts', () => { - const partial = render(HostList, { props: { hosts: [{ ...host(), serverCount: 3, playerCount: 4, unreportedInstances: 1 }], now } }).body; + const partial = render(HostList, { + props: { + hosts: [{ ...host(), serverCount: 3, playerCount: 4, unreportedInstances: 1 }], + now + } + }).body; expect(partial).toContain('>4+'); expect(partial).toContain('Player counts unavailable for 1 of 3 servers'); - const unknown = render(HostList, { props: { hosts: [{ ...host(), serverCount: 3, playerCount: 0, unreportedInstances: 3 }], now } }).body; + const unknown = render(HostList, { + props: { + hosts: [{ ...host(), serverCount: 3, playerCount: 0, unreportedInstances: 3 }], + now + } + }).body; expect(unknown).toContain('>—'); - const empty = render(HostList, { props: { hosts: [{ ...host(), serverCount: 0, playerCount: 0, unreportedInstances: 0 }], now } }).body; + const empty = render(HostList, { + props: { + hosts: [{ ...host(), serverCount: 0, playerCount: 0, unreportedInstances: 0 }], + now + } + }).body; expect(empty).toContain('>0'); }); diff --git a/tests/unit/ui-identity-status.test.ts b/tests/unit/ui-identity-status.test.ts index c4fa6b0..b5db4eb 100644 --- a/tests/unit/ui-identity-status.test.ts +++ b/tests/unit/ui-identity-status.test.ts @@ -4,9 +4,21 @@ import { statusPresentation } from '../../src/lib/ui/status.js'; describe('operator-facing identities and states', () => { it('preserves lobby numbers and distinguishes T6 variants sharing an engine map', () => { - expect(serverIdentity('t6_zombie_town_12')).toEqual({ map:'Town Survival', lobby:12, game:'Black Ops II' }); - expect(serverIdentity('t6_zombie_farm_2')).toEqual({ map:'Farm Survival', lobby:2, game:'Black Ops II' }); - expect(serverIdentity('custom-server')).toEqual({ map:'custom-server', lobby:null, game:'' }); + expect(serverIdentity('t6_zombie_town_12')).toEqual({ + map: 'Town Survival', + lobby: 12, + game: 'Black Ops II' + }); + expect(serverIdentity('t6_zombie_farm_2')).toEqual({ + map: 'Farm Survival', + lobby: 2, + game: 'Black Ops II' + }); + expect(serverIdentity('custom-server')).toEqual({ + map: 'custom-server', + lobby: null, + game: '' + }); }); it('keeps normal waiting neutral and distinguishes test success from promotion', () => { expect(statusPresentation('awaiting-empty').tone).toBe('neutral'); diff --git a/tests/unit/ui-language.test.ts b/tests/unit/ui-language.test.ts index 08cc8a2..940f6f1 100644 --- a/tests/unit/ui-language.test.ts +++ b/tests/unit/ui-language.test.ts @@ -13,14 +13,57 @@ import GameFileHosts from '../../src/lib/ui/components/GameFileHosts.svelte'; import type { Host } from '../../src/lib/hosts.js'; import type { PlayerMatch } from '../../src/lib/leaderboard.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/hosts'), data: { user: { id: 'admin', role: 'admin' } } }) })); -vi.mock('$app/navigation', () => ({ goto: vi.fn(), beforeNavigate: vi.fn(), afterNavigate: vi.fn(), invalidateAll: vi.fn() })); +vi.mock('$app/stores', () => ({ + page: writable({ + url: new URL('http://localhost/hosts'), + data: { user: { id: 'admin', role: 'admin' } } + }) +})); +vi.mock('$app/navigation', () => ({ + goto: vi.fn(), + beforeNavigate: vi.fn(), + afterNavigate: vi.fn(), + invalidateAll: vi.fn() +})); const now = Date.parse('2026-10-07T12:00:00Z'); -const host: Host = { id: 'dallas', name: 'Dallas 1', hostname: 'dallas.example', mode: 'active', portMin: 28960, portMax: 28980, - bindAddress: '0.0.0.0', publicAddress: '', minimumFreeBytes: 100, createdAt: new Date(now).toISOString(), - lastHeartbeat: new Date(now).toISOString(), connected: false, connectionState: 'offline', readiness: 'ready', readinessReasons: [], - inventory: {}, enrolled: true, credentialActive: true, decommission: null }; -const match: PlayerMatch = { id: 17, game: 't5', server: 't5_zombie_theater_1', mapName: 'Kino Der Toten', startedAt: new Date(now).toISOString(), lastSeen: new Date(now).toISOString(), status: 'completed', completed: 1, highestRound: 30, roundsSurvived: 20, kills: 1200, headshots: 200, revives: 2, downs: 1, bestScore: 45000 }; +const host: Host = { + id: 'dallas', + name: 'Dallas 1', + hostname: 'dallas.example', + mode: 'active', + portMin: 28960, + portMax: 28980, + bindAddress: '0.0.0.0', + publicAddress: '', + minimumFreeBytes: 100, + createdAt: new Date(now).toISOString(), + lastHeartbeat: new Date(now).toISOString(), + connected: false, + connectionState: 'offline', + readiness: 'ready', + readinessReasons: [], + inventory: {}, + enrolled: true, + credentialActive: true, + decommission: null +}; +const match: PlayerMatch = { + id: 17, + game: 't5', + server: 't5_zombie_theater_1', + mapName: 'Kino Der Toten', + startedAt: new Date(now).toISOString(), + lastSeen: new Date(now).toISOString(), + status: 'completed', + completed: 1, + highestRound: 30, + roundsSurvived: 20, + kills: 1200, + headshots: 200, + revives: 2, + downs: 1, + bestScore: 45000 +}; it('pluralizes zero, one, multiple, fractional and irregular counts consistently', () => { expect(countLabel(0, 'server')).toBe('0 servers'); @@ -34,8 +77,19 @@ it('pluralizes zero, one, multiple, fractional and irregular counts consistently }); it('recommends a sampling upgrade only for a known older version', () => { - for (const version of ['1.0.0', '1.4.5', 'v1.4.0']) expect(hostCalibrationHint(version)).toContain('Update this host to agent 1.4.6'); - for (const version of ['1.4.6', '1.7.0', '1.10.0', '2.0.0', 'v2.0.0+build', null, undefined, 'unknown', '1.4']) { + for (const version of ['1.0.0', '1.4.5', 'v1.4.0']) + expect(hostCalibrationHint(version)).toContain('Update this host to agent 1.4.6'); + for (const version of [ + '1.4.6', + '1.7.0', + '1.10.0', + '2.0.0', + 'v2.0.0+build', + null, + undefined, + 'unknown', + '1.4' + ]) { expect(hostCalibrationHint(version)).toContain('still calibrating'); expect(hostCalibrationHint(version)).not.toContain('Update'); } @@ -44,29 +98,69 @@ it('recommends a sampling upgrade only for a known older version', () => { it('describes per-player completion as final-report evidence without inventing unfinished results', () => { expect(playerCompletionLabel(match)).toBe('Present in final report'); expect(playerCompletionLabel({ ...match, completed: 0 })).toBe('Not present in final report'); - expect(playerCompletionLabel({ ...match, status: 'active', completed: 0 })).toBe('Match in progress'); - expect(playerCompletionLabel({ ...match, status: 'interrupted', completed: 0 })).toBe('Final report unavailable'); - const body = render(PlayerMatchCards, { props: { matches: [match, { ...match, id: 18, completed: 0 }] } }).body; - expect(body).toContain('Present in final report'); expect(body).toContain('Not present in final report'); - expect(body).not.toContain('Completion credited'); expect(body).not.toContain('No completion credit'); + expect(playerCompletionLabel({ ...match, status: 'active', completed: 0 })).toBe( + 'Match in progress' + ); + expect(playerCompletionLabel({ ...match, status: 'interrupted', completed: 0 })).toBe( + 'Final report unavailable' + ); + const body = render(PlayerMatchCards, { + props: { matches: [match, { ...match, id: 18, completed: 0 }] } + }).body; + expect(body).toContain('Present in final report'); + expect(body).toContain('Not present in final report'); + expect(body).not.toContain('Completion credited'); + expect(body).not.toContain('No completion credit'); }); it('uses product game names while preserving unsupported game identifiers', () => { - expect(gameTitle('t5')).toBe('Black Ops I'); expect(gameTitle('T6')).toBe('Black Ops II'); - expect(gameTitle('t5', true)).toBe('BO1'); expect(gameTitle('t6', true)).toBe('BO2'); - expect(gameTitle('custom')).toBe('CUSTOM'); expect(gameTitle(null)).toBe('Unknown game'); + expect(gameTitle('t5')).toBe('Black Ops I'); + expect(gameTitle('T6')).toBe('Black Ops II'); + expect(gameTitle('t5', true)).toBe('BO1'); + expect(gameTitle('t6', true)).toBe('BO2'); + expect(gameTitle('custom')).toBe('CUSTOM'); + expect(gameTitle(null)).toBe('Unknown game'); expect(scopeLabel({ scope_type: 'game', scope_value: 't5' })).toBe('Black Ops I'); }); it('does not offer an existing-account sign-in path on first administrator setup', () => { const body = render(Setup).body; - expect(body).toContain('Create the first administrator'); expect(body).not.toContain('Already have an account'); + expect(body).toContain('Create the first administrator'); + expect(body).not.toContain('Already have an account'); expect(body).not.toContain('href="/login"'); }); it('renders singular host attention and explicit whitespace before an offline suffix', () => { - const body = render(Hosts, { props: { data: { hosts: [{ ...host, serverCount: 0, playerCount: 0, unreportedInstances: 0 }], serverNow: now, user: null, setupRequired: false } } }).body; - expect(body).toContain('1 host'); expect(body).toContain('1 needs attention'); expect(body).not.toContain('1 need attention'); - const files = render(GameFileHosts, { props: { hosts: [host], caches: [{ hostId: host.id, libraryVersionId: 'files', status: 'missing', bytesDownloaded: 0, bytesTotal: 0, manifestSha256: null, error: null, updatedAt: new Date(now).toISOString(), operationId: null }] } }).body; + const body = render(Hosts, { + props: { + data: { + hosts: [{ ...host, serverCount: 0, playerCount: 0, unreportedInstances: 0 }], + serverNow: now, + user: null, + setupRequired: false + } + } + }).body; + expect(body).toContain('1 host'); + expect(body).toContain('1 needs attention'); + expect(body).not.toContain('1 need attention'); + const files = render(GameFileHosts, { + props: { + hosts: [host], + caches: [ + { + hostId: host.id, + libraryVersionId: 'files', + status: 'missing', + bytesDownloaded: 0, + bytesTotal: 0, + manifestSha256: null, + error: null, + updatedAt: new Date(now).toISOString(), + operationId: null + } + ] + } + }).body; expect(files).toMatch(/Dallas 1<\/a>.*?> · Offline<\/span>/); }); diff --git a/tests/unit/ui-leaderboard.test.ts b/tests/unit/ui-leaderboard.test.ts index 1d6de13..c0df264 100644 --- a/tests/unit/ui-leaderboard.test.ts +++ b/tests/unit/ui-leaderboard.test.ts @@ -3,40 +3,118 @@ import { render } from 'svelte/server'; import { writable, type Writable } from 'svelte/store'; import { page } from '$app/stores'; import Leaderboard from '../../src/routes/leaderboard/+page.svelte'; -import { createLeaderboardUpdates, leaderboardPageSize } from '../../src/lib/leaderboard-updates.js'; -import type { LeaderboardEntry, LeaderboardResponse, RecordEvidence } from '../../src/lib/leaderboard.js'; +import { + createLeaderboardUpdates, + leaderboardPageSize +} from '../../src/lib/leaderboard-updates.js'; +import type { + LeaderboardEntry, + LeaderboardResponse, + RecordEvidence +} from '../../src/lib/leaderboard.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/leaderboard') }), navigating: writable(null) })); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/leaderboard') }), + navigating: writable(null) +})); vi.mock('$app/navigation', () => ({ goto: vi.fn() })); -const evidence: RecordEvidence = { matchId: 71, game: 't5', mapKey: 't5:theater', mapName: 'Kino Der Toten', server: 't5_zombie_theater_1', startedAt: '2026-10-06T12:00:00Z', endedAt: '2026-10-06T13:00:00Z', startObserved: true, presentAtStart: false, firstObservedRound: 8, value: 49 }; -const entry: LeaderboardEntry = { rank: 27, guid: '12345', name: 'Cherry', highestRound: 49, kills: 1000, roundsSurvived: 60, headshots: 100, revives: 3, downs: 5, gamesPlayed: 6, gamesCompleted: 5, bestKills: 500, bestScore: 4000, lastSeen: evidence.endedAt!, records: { highestRound: evidence, bestKills: evidence, bestScore: evidence }, headshotRate: 0.1, killsPerDown: 200, averageKills: 1000 / 6, averageRounds: 10, completionRate: 5 / 6 }; +const evidence: RecordEvidence = { + matchId: 71, + game: 't5', + mapKey: 't5:theater', + mapName: 'Kino Der Toten', + server: 't5_zombie_theater_1', + startedAt: '2026-10-06T12:00:00Z', + endedAt: '2026-10-06T13:00:00Z', + startObserved: true, + presentAtStart: false, + firstObservedRound: 8, + value: 49 +}; +const entry: LeaderboardEntry = { + rank: 27, + guid: '12345', + name: 'Cherry', + highestRound: 49, + kills: 1000, + roundsSurvived: 60, + headshots: 100, + revives: 3, + downs: 5, + gamesPlayed: 6, + gamesCompleted: 5, + bestKills: 500, + bestScore: 4000, + lastSeen: evidence.endedAt!, + records: { highestRound: evidence, bestKills: evidence, bestScore: evidence }, + headshotRate: 0.1, + killsPerDown: 200, + averageKills: 1000 / 6, + averageRounds: 10, + completionRate: 5 / 6 +}; function report(): LeaderboardResponse { - return { entries: [structuredClone(entry)], maps: [{ key: 't5:theater', game: 't5', map: 'theater', name: 'Kino Der Toten' }], total: 60, limit: 25, offset: 25, metric: 'highestRound', summary: { players: 473, matches: 411, kills: 112899, roundsSurvived: 7062, highestRound: 49, firstTrackedAt: evidence.startedAt, updatedAt: evidence.endedAt } }; + return { + entries: [structuredClone(entry)], + maps: [{ key: 't5:theater', game: 't5', map: 'theater', name: 'Kino Der Toten' }], + total: 60, + limit: 25, + offset: 25, + metric: 'highestRound', + summary: { + players: 473, + matches: 411, + kills: 112899, + roundsSurvived: 7062, + highestRound: 49, + firstTrackedAt: evidence.startedAt, + updatedAt: evidence.endedAt + } + }; } function scope(search = '') { - (page as unknown as Writable<{ url: URL }>).set({ url: new URL(`http://localhost/leaderboard${search}`) }); + (page as unknown as Writable<{ url: URL }>).set({ + url: new URL(`http://localhost/leaderboard${search}`) + }); +} +function html(data = report()) { + return render(Leaderboard, { + props: { data: { leaderboard: data, user: null, setupRequired: false } } + }).body; } -function html(data = report()) { return render(Leaderboard, { props: { data: { leaderboard: data, user: null, setupRequired: false } } }).body; } beforeEach(() => scope('?map=')); it('shows compact totals and core stat columns, preserving ranks without repeated row disclosures', () => { const body = html(); const headers = body.slice(body.indexOf('')); expect(headers.match(/scope="col"/g)).toHaveLength(6); - expect(headers).toContain('Highest round'); expect(headers).toContain('Total kills'); expect(headers).toContain('Games played'); + expect(headers).toContain('Highest round'); + expect(headers).toContain('Total kills'); + expect(headers).toContain('Games played'); expect(headers).toContain('Total rounds'); - for (const label of ['Headshots', 'Revives', 'Downs', 'Games completed', 'Best game kills', 'Highest reported score']) expect(body).toContain(label); + for (const label of [ + 'Headshots', + 'Revives', + 'Downs', + 'Games completed', + 'Best game kills', + 'Highest reported score' + ]) + expect(body).toContain(label); expect(body).toContain('Show all stats'); expect(body).toMatch(/]*>27<\/span>/); - expect(body).toContain('26–26 of 60 players'); + expect(body.replace(/\s+/g, ' ')).toContain('26–26 of 60 players'); const overview = body.slice(body.indexOf('class="overview'), body.indexOf('class="rankings')); expect(overview).toContain('All-time stats for All games & maps'); - for (const value of ['49', '112,899', '7,062', '411', '473']) expect(overview).toMatch(new RegExp(`]*>${value}`)); + for (const value of ['49', '112,899', '7,062', '411', '473']) + expect(overview).toMatch(new RegExp(`]*>${value}`)); expect(overview).toContain('Fleet record'); expect(overview).not.toContain(']*open/); - expect(body).not.toContain('Copy GUID'); expect(body).not.toContain('Every round counts'); - expect(body).not.toContain('Record details'); expect(body).not.toContain('Hide details'); + expect(body).not.toContain('Copy GUID'); + expect(body).not.toContain('Every round counts'); + expect(body).not.toContain('Record details'); + expect(body).not.toContain('Hide details'); const rows = body.slice(body.indexOf('')); expect(rows.match(/ { scope('?game=t5&map=t5%3Atheater&metric=kills&q=Cherry&offset=25'); - const data = report(); data.metric = 'kills'; + const data = report(); + data.metric = 'kills'; const body = html(data); const headers = body.slice(body.indexOf('')); expect(headers.match(/scope="col"/g)).toHaveLength(5); - expect(body).toContain('offset%3D25'); expect(body).toContain('q%3DCherry'); + expect(body).toContain('offset%3D25'); + expect(body).toContain('q%3DCherry'); scope('?game=t5&map=t5%3Atheater&metric=highestRound&q=Cherry&offset=25'); const recordBody = html(); expect(recordBody).toContain('/matches/71?returnTo=%2Fleaderboard'); expect(recordBody).toContain('map%3Dt5%253Atheater'); - expect(recordBody).toContain('Selection record'); expect(recordBody).not.toContain('Fleet record'); + expect(recordBody).toContain('Selection record'); + expect(recordBody).not.toContain('Fleet record'); }); it('provides separate mobile filter controls and preserves explicit all-map and squad context', () => { @@ -66,39 +147,50 @@ it('provides separate mobile filter controls and preserves explicit all-map and expect(body).toContain('id="leaderboard-players-mobile"'); expect(body).toContain('value="2" selected=""'); expect(body).toContain('All squad sizes'); - expect(body).toContain('Solo'); expect(body).toContain('Duo'); expect(body).toContain('Trio'); expect(body).toContain('Four players'); - expect(body).toContain('unknown starting counts appear only under All squad sizes'); + expect(body).toContain('Solo'); + expect(body).toContain('Duo'); + expect(body).toContain('Trio'); + expect(body).toContain('Four players'); + expect(body.replace(/\s+/g, ' ')).toContain( + 'unknown starting counts appear only under All squad sizes' + ); expect(body).toContain('players%3D2'); expect(body).toContain('map%3D'); - const ids = [...body.matchAll(/\bid="([^"]+)"/g)].map(match => match[1]); + const ids = [...body.matchAll(/\bid="([^"]+)"/g)].map((match) => match[1]); expect(new Set(ids).size).toBe(ids.length); expect(body).not.toMatch(/>Refresh<\/span>/); }); it('keeps qualification and denominator visible for efficiency rankings and missing values distinct', () => { scope('?metric=headshotRate'); - const data = report(); data.metric = 'headshotRate'; + const data = report(); + data.metric = 'headshotRate'; let body = html(data); expect(body).toContain('At least 5 ended games and 100 kills within the selected game and map'); expect(body).toContain('100 headshots / 1,000 kills · 6 games'); expect(body).toContain('10%'); data.entries[0].headshotRate = null; - body = html(data); expect(body).toContain('>—'); + body = html(data); + expect(body).toContain('>—'); }); it('holds changing ranks until explicitly applied, coalescing later polls even after focus leaves', () => { const first = report(); const updates = createLeaderboardUpdates('all', first); - const second = structuredClone(first); second.entries[0].rank = 28; + const second = structuredClone(first); + second.entries[0].rank = 28; expect(updates.receive('all', second, true)).toEqual({ current: first, pending: second }); - const third = structuredClone(second); third.entries[0].rank = 30; + const third = structuredClone(second); + third.entries[0].rank = 30; expect(updates.receive('all', third, false)).toEqual({ current: first, pending: third }); expect(updates.apply()).toEqual(third); expect(updates.receive('all', third, true)?.pending).toBeNull(); }); it('ignores old scopes, clears a reverted pending update, and accepts unattended refreshes', () => { - const first = report(); const second = structuredClone(first); second.total = 100; + const first = report(); + const second = structuredClone(first); + second.total = 100; const updates = createLeaderboardUpdates('all', first); updates.receive('all', second, true); expect(updates.receive('all', structuredClone(first), true)?.pending).toBeNull(); diff --git a/tests/unit/ui-library.test.ts b/tests/unit/ui-library.test.ts index a34fa82..01cbcd5 100644 --- a/tests/unit/ui-library.test.ts +++ b/tests/unit/ui-library.test.ts @@ -10,7 +10,9 @@ it('shows the promoted version once and keeps a newer failed version distinct', const promoted = render(LibraryVersions, { props: { current, latest: current } }).body; expect(promoted.match(/v1/g)).toHaveLength(1); expect(promoted.match(/Promoted/g)).toHaveLength(1); - const newer = render(LibraryVersions, { props: { current, latest: { id: 2, version: 2, state: 'test-failed' } } }).body; + const newer = render(LibraryVersions, { + props: { current, latest: { id: 2, version: 2, state: 'test-failed' } } + }).body; expect(newer).toContain('Test failed'); expect(newer).toContain('v2'); expect(newer).toContain('v1'); @@ -19,32 +21,68 @@ it('shows the promoted version once and keeps a newer failed version distinct', it('disambiguates two builds with the same release tag only when needed', () => { const current = { id: 1, version: 1, release_version: 'v1.1.0' }; - expect(render(LibraryVersions, { props: { current, latest: current } }).body).not.toContain('upload'); - const body = render(LibraryVersions, { props: { current, latest: { ...current, id: 2, version: 2, state: 'untested' } } }).body; + expect(render(LibraryVersions, { props: { current, latest: current } }).body).not.toContain( + 'upload' + ); + const body = render(LibraryVersions, { + props: { current, latest: { ...current, id: 2, version: 2, state: 'untested' } } + }).body; expect(body).toContain('v1.1.0 (upload 1)'); expect(body).toContain('v1.1.0 (upload 2)'); expect(body).toContain('Needs testing'); }); it('does not present an active beta as passed or an empty artifact as promoted', () => { - expect(render(LibraryVersions, { props: { current: null, latest: null } }).body).toContain('No versions'); - const body = render(LibraryVersions, { props: { current: null, latest: { id: 1, version: 1, state: 'tested' }, betaId: 12 } }).body; + expect(render(LibraryVersions, { props: { current: null, latest: null } }).body).toContain( + 'No versions' + ); + const body = render(LibraryVersions, { + props: { current: null, latest: { id: 1, version: 1, state: 'tested' }, betaId: 12 } + }).body; expect(body).toContain('Beta active'); expect(body).not.toContain('Test passed'); expect(body).toContain('Never promoted'); }); -const host = (id: string, connected = true) => ({ id, name: id, connected, enrolled: true, credentialActive: true, mode: 'active', readiness: 'ready' } as Host); -const cache = (id: string, status: HostLibraryCache['status'], error: string | null = null): HostLibraryCache => ({ - hostId: id, libraryVersionId: 'upload', status, error, bytesDownloaded: 100, bytesTotal: 100, - manifestSha256: null, updatedAt: '2026-10-06T12:00:00Z', operationId: null +const host = (id: string, connected = true) => + ({ + id, + name: id, + connected, + enrolled: true, + credentialActive: true, + mode: 'active', + readiness: 'ready' + }) as Host; +const cache = ( + id: string, + status: HostLibraryCache['status'], + error: string | null = null +): HostLibraryCache => ({ + hostId: id, + libraryVersionId: 'upload', + status, + error, + bytesDownloaded: 100, + bytesTotal: 100, + manifestSha256: null, + updatedAt: '2026-10-06T12:00:00Z', + operationId: null }); it('surfaces failures, transfers and offline hosts before collapsed healthy copies', () => { - const body = render(GameFileHosts, { props: { latest: true, - hosts: [host('healthy'), host('failed'), host('transfer'), host('offline', false)], - caches: [cache('healthy', 'ready'), cache('failed', 'failed', 'Checksum mismatch'), cache('transfer', 'verifying'), cache('offline', 'ready')] - } }).body; + const body = render(GameFileHosts, { + props: { + latest: true, + hosts: [host('healthy'), host('failed'), host('transfer'), host('offline', false)], + caches: [ + cache('healthy', 'ready'), + cache('failed', 'failed', 'Checksum mismatch'), + cache('transfer', 'verifying'), + cache('offline', 'ready') + ] + } + }).body; const ready = body.indexOf('Ready hosts (1)'); expect(body.indexOf('Checksum mismatch')).toBeLessThan(ready); expect(body.indexOf('Host verification progress')).toBeLessThan(ready); @@ -56,7 +94,16 @@ it('surfaces failures, transfers and offline hosts before collapsed healthy copi }); it('explains pending automatic downloads and keeps copy errors visible even for ready status', () => { - const body = render(GameFileHosts, { props: { latest: true, hosts: [host('waiting', false), host('inconsistent')], caches: [cache('waiting', 'missing'), cache('inconsistent', 'ready', 'Host report needs review')] } }).body; + const body = render(GameFileHosts, { + props: { + latest: true, + hosts: [host('waiting', false), host('inconsistent')], + caches: [ + cache('waiting', 'missing'), + cache('inconsistent', 'ready', 'Host report needs review') + ] + } + }).body; expect(body).toContain('Automatic setup resumes when the host reconnects.'); expect(body).toContain('Host report needs review'); expect(body).not.toContain('Ready hosts'); diff --git a/tests/unit/ui-lifecycle-actions.test.ts b/tests/unit/ui-lifecycle-actions.test.ts index 595cb65..71b76bf 100644 --- a/tests/unit/ui-lifecycle-actions.test.ts +++ b/tests/unit/ui-lifecycle-actions.test.ts @@ -12,12 +12,32 @@ vi.mock('$app/navigation', () => ({ goto: vi.fn() })); const now = get(uiNow); const iso = (offset = 0) => new Date(now + offset).toISOString(); const server: FleetServer = { - name: 't5_zombie_theater_1', game: 't5', map: 'zombie_theater', last_map: 'zombie_theater', - last_gametype: 'zclassic', last_players: 4, last_seen_at: iso(-10_000), last_error: null, - container_state: 'running', container_checked_at: iso(-5_000), last_checked_at: iso(-5_000), - roster: [], health: 'online', held: [], active_since: iso(-120_000), inactive_since: null, - game_started_at: iso(-120_000), game_ended_at: null, host_id: 'host-1', - host: { id: 'host-1', name: 'Dallas 1', connected: true, connectionState: 'online', lastHeartbeat: iso(-1_000) } + name: 't5_zombie_theater_1', + game: 't5', + map: 'zombie_theater', + last_map: 'zombie_theater', + last_gametype: 'zclassic', + last_players: 4, + last_seen_at: iso(-10_000), + last_error: null, + container_state: 'running', + container_checked_at: iso(-5_000), + last_checked_at: iso(-5_000), + roster: [], + health: 'online', + held: [], + active_since: iso(-120_000), + inactive_since: null, + game_started_at: iso(-120_000), + game_ended_at: null, + host_id: 'host-1', + host: { + id: 'host-1', + name: 'Dallas 1', + connected: true, + connectionState: 'online', + lastHeartbeat: iso(-1_000) + } }; describe('lifecycle action applicability', () => { @@ -29,50 +49,83 @@ describe('lifecycle action applicability', () => { const stopped = { ...server, health: 'offline' as const, container_state: 'stopped' }; expect(applicableRuntimeActions(stopped, now)).toEqual(['start', 'rebuild']); for (const change of [ - { container_checked_at: iso(-91_000) }, { container_checked_at: null }, - { telemetry_after: iso(-2_000) }, { last_seen_at: iso(-2_000) }, - { host: null }, { host: { ...server.host!, connected: false } } - ]) expect(applicableRuntimeActions({ ...stopped, ...change }, now)).not.toContain('start'); + { container_checked_at: iso(-91_000) }, + { container_checked_at: null }, + { telemetry_after: iso(-2_000) }, + { last_seen_at: iso(-2_000) }, + { host: null }, + { host: { ...server.host!, connected: false } } + ]) + expect(applicableRuntimeActions({ ...stopped, ...change }, now)).not.toContain('start'); }); it('permits recovery of an unresponsive game only with a current running-container observation', () => { - const unresponsive = { ...server, health: 'stale' as const, last_seen_at: iso(-300_000), last_error: 'timeout:no reply' }; + const unresponsive = { + ...server, + health: 'stale' as const, + last_seen_at: iso(-300_000), + last_error: 'timeout:no reply' + }; expect(applicableRuntimeActions(unresponsive, now)).toEqual(['restart', 'stop', 'rebuild']); - expect(applicableRuntimeActions({ ...unresponsive, container_checked_at: iso(-91_000) }, now)).toEqual([]); - expect(applicableRuntimeActions({ ...unresponsive, telemetry_after: iso(-2_000) }, now)).toEqual([]); + expect( + applicableRuntimeActions({ ...unresponsive, container_checked_at: iso(-91_000) }, now) + ).toEqual([]); + expect( + applicableRuntimeActions({ ...unresponsive, telemetry_after: iso(-2_000) }, now) + ).toEqual([]); }); it('does not offer runtime mutations during startup or without evidence', () => { for (const change of [ - { health: 'starting' as const }, { health: 'provisioning' as const }, + { health: 'starting' as const }, + { health: 'provisioning' as const }, { host: { ...server.host!, connected: false } }, { health: 'offline' as const, container_state: 'unknown', last_seen_at: null } - ]) expect(applicableRuntimeActions({ ...server, ...change }, now)).toEqual([]); + ]) + expect(applicableRuntimeActions({ ...server, ...change }, now)).toEqual([]); }); }); describe('lifecycle impact copy', () => { it('names current players and round with correct singular grammar', () => { - expect(lifecycleImpact('restart', 4, 30)).toBe('4 players are in round 30. Restart waits until the lobby is empty.'); - expect(lifecycleImpact('stop', 1, null)).toBe('1 player is currently online. Stop waits until the lobby is empty.'); + expect(lifecycleImpact('restart', 4, 30)).toBe( + '4 players are in round 30. Restart waits until the lobby is empty.' + ); + expect(lifecycleImpact('stop', 1, null)).toBe( + '1 player is currently online. Stop waits until the lobby is empty.' + ); }); it('does not confuse unknown or invalid player counts with an empty lobby', () => { - for (const players of [null, -1, NaN, 1.5]) expect(lifecycleImpact('destroy', players, 30)) - .toBe('Player count is unknown. Destruction waits for a live check confirming the lobby is empty.'); - expect(lifecycleImpact('rebuild', 0, 30)).toBe('The lobby is currently empty. A live check will confirm this before rebuild begins.'); + for (const players of [null, -1, NaN, 1.5]) + expect(lifecycleImpact('destroy', players, 30)).toBe( + 'Player count is unknown. Destruction waits for a live check confirming the lobby is empty.' + ); + expect(lifecycleImpact('rebuild', 0, 30)).toBe( + 'The lobby is currently empty. A live check will confirm this before rebuild begins.' + ); }); it('does not claim a round or occupancy wait when starting a stopped container', () => { - expect(lifecycleImpact('start', null, null)).toBe('Start the existing container, then verify that the game is responding.'); + expect(lifecycleImpact('start', null, null)).toBe( + 'Start the existing container, then verify that the game is responding.' + ); expect(lifecycleImpact('restart', 4, NaN)).toContain('4 players are currently online.'); }); }); describe('contextual lifecycle confirmation', () => { it('names the server and impact without re-asking the instance or action', () => { - const body = render(LifecycleDialog, { props: { open: true, initialServer: server.name, - initialAction: 'restart', serverContext: server, players: 4, round: 30 } }).body; + const body = render(LifecycleDialog, { + props: { + open: true, + initialServer: server.name, + initialAction: 'restart', + serverContext: server, + players: 4, + round: 30 + } + }).body; expect(body).toContain('Restart Kino Der Toten · Lobby 1?'); expect(body).toContain('4 players are in round 30.'); expect(body).toContain('Restart when empty'); @@ -81,16 +134,26 @@ describe('contextual lifecycle confirmation', () => { }); it('explains an unavailable action when the evidence no longer supports it', () => { - const body = render(LifecycleDialog, { props: { open: true, initialServer: server.name, - initialAction: 'start', serverContext: server } }).body; + const body = render(LifecycleDialog, { + props: { + open: true, + initialServer: server.name, + initialAction: 'start', + serverContext: server + } + }).body; expect(body).toContain('Start is unavailable. The game is responding.'); - const startButton = [...body.matchAll(/]*>[\s\S]*?<\/button>/g)].find(match => match[0].includes('Start server'))?.[0]; + const startButton = [...body.matchAll(/]*>[\s\S]*?<\/button>/g)].find((match) => + match[0].includes('Start server') + )?.[0]; expect(startButton).toContain('disabled'); expect(body).not.toContain('Start the existing container'); }); it('uses Destroy server consistently and preserves recorded history in the explanation', () => { - const body = render(DestroyServerModal, { props: { open: true, name: server.name, players: 4, round: 30 } }).body; + const body = render(DestroyServerModal, { + props: { open: true, name: server.name, players: 4, round: 30 } + }).body; expect(body).toContain('Destroy Kino Der Toten · Lobby 1?'); expect(body).toContain('4 players are in round 30.'); expect(body).toContain('Destroy server when empty'); diff --git a/tests/unit/ui-match-chat.test.ts b/tests/unit/ui-match-chat.test.ts index 7ae1250..d6902d4 100644 --- a/tests/unit/ui-match-chat.test.ts +++ b/tests/unit/ui-match-chat.test.ts @@ -5,16 +5,40 @@ import MatchPage from '../../src/routes/matches/[id]/+page.svelte'; import ChatPanel from '../../src/lib/ui/components/ChatPanel.svelte'; import type { MatchDetail } from '../../src/lib/match-history.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/matches/1'), data: { user: null } }) })); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/matches/1'), data: { user: null } }) +})); const match: MatchDetail = { - id: 1, server: 't5_zombie_theater_1', game: 't5', map: 'Kino Der Toten', - observedRound: 4, finalRound: 4, status: 'completed', startObserved: true, - startedAt: '2026-10-07T00:00:00Z', endedAt: '2026-10-07T00:01:00Z', durationMs: 60_000, - startPlayers: 1, uniquePlayers: 1, endPlayers: 1, players: [], - chat: [{ type: 'message', matchId: 1, - message: { time: '60:30', guid: 'abc', slot: 0, player: 'Player', message: '' }, - context: { realTime: '2026-10-07T00:00:30Z', matchElapsedMs: 30_000, round: 4 } }] + id: 1, + server: 't5_zombie_theater_1', + game: 't5', + map: 'Kino Der Toten', + observedRound: 4, + finalRound: 4, + status: 'completed', + startObserved: true, + startedAt: '2026-10-07T00:00:00Z', + endedAt: '2026-10-07T00:01:00Z', + durationMs: 60_000, + startPlayers: 1, + uniquePlayers: 1, + endPlayers: 1, + players: [], + chat: [ + { + type: 'message', + matchId: 1, + message: { + time: '60:30', + guid: 'abc', + slot: 0, + player: 'Player', + message: '' + }, + context: { realTime: '2026-10-07T00:00:30Z', matchElapsedMs: 30_000, round: 4 } + } + ] }; describe('historical match chat', () => { @@ -24,14 +48,16 @@ describe('historical match chat', () => { expect(body).toContain('<script>hello</script>'); expect(body).toContain('Message context'); expect(body).toContain('30s'); - expect(body.replace(//gs, '')).toMatch(/
    Round<\/dt>]*>4<\/dd>/); + expect(body.replace(//gs, '')).toMatch(/
    Round<\/dt>\s*]*>4<\/dd>/); expect(body).not.toContain('Refresh chat'); expect(body).not.toContain('Live · 5s'); expect(body).not.toContain('Loading chat'); }); it('explains an offline host without claiming live updates or offering a manual refresh', () => { - const body = render(ChatPanel, { props: { server: 't5_zombie_theater_1', hostOffline: true, hostName: 'Dallas 1' } }).body.replaceAll(//g, ''); + const body = render(ChatPanel, { + props: { server: 't5_zombie_theater_1', hostOffline: true, hostName: 'Dallas 1' } + }).body.replaceAll(//g, ''); expect(body).toContain('Chat resumes when Dallas 1 reconnects.'); expect(body).not.toContain('Live ·'); expect(body).not.toContain('Loading chat'); diff --git a/tests/unit/ui-match-history.test.ts b/tests/unit/ui-match-history.test.ts index 650214d..47456f4 100644 --- a/tests/unit/ui-match-history.test.ts +++ b/tests/unit/ui-match-history.test.ts @@ -8,27 +8,65 @@ import { matchStartedLabel, matchInstanceLabel } from '../../src/lib/ui/match-hi import type { MatchHistoryResponse, ServerMatch } from '../../src/lib/match-history.js'; vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/matches') }) })); vi.mock('$app/navigation', () => ({ goto: vi.fn() })); -const match: ServerMatch = { id: 1, server: 't6_zombie_town_1', game: 't6', map: 'Town Survival', observedRound: 20, finalRound: 20, status: 'completed', startedAt: '2026-10-06T17:26:00Z', endedAt: '2026-10-06T18:04:00Z', durationMs: 2280000, startPlayers: 2, uniquePlayers: 4, endPlayers: 2 }; -const report = (): MatchHistoryResponse => ({ matches: [match], total: 40, page: 1, limit: 25, maps: [{ key: 't6:town', game: 't6', name: 'Town Survival' }] }); -function scope(search = '') { (page as unknown as Writable<{ url: URL }>).set({ url: new URL(`http://localhost/matches${search}`) }); } -beforeEach(() => { scope(); vi.useFakeTimers(); vi.setSystemTime(new Date('2026-10-06T20:00:00Z')); }); +const match: ServerMatch = { + id: 1, + server: 't6_zombie_town_1', + game: 't6', + map: 'Town Survival', + observedRound: 20, + finalRound: 20, + status: 'completed', + startedAt: '2026-10-06T17:26:00Z', + endedAt: '2026-10-06T18:04:00Z', + durationMs: 2280000, + startPlayers: 2, + uniquePlayers: 4, + endPlayers: 2 +}; +const report = (): MatchHistoryResponse => ({ + matches: [match], + total: 40, + page: 1, + limit: 25, + maps: [{ key: 't6:town', game: 't6', name: 'Town Survival' }] +}); +function scope(search = '') { + (page as unknown as Writable<{ url: URL }>).set({ + url: new URL(`http://localhost/matches${search}`) + }); +} +beforeEach(() => { + scope(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-10-06T20:00:00Z')); +}); afterEach(() => vi.useRealTimers()); it('renders one page heading, six columns, a combined identity, and compact dates', () => { - const body = render(MatchHistoryPage, { props: { data: { history: report(), user: null, setupRequired: false } } }).body; + const body = render(MatchHistoryPage, { + props: { data: { history: report(), user: null, setupRequired: false } } + }).body; expect(body.match(/]/g)).toHaveLength(1); expect(body.match(/scope="col"/g)).toHaveLength(6); - expect(body).toContain('2 h ago'); expect(body).toContain('Lobby 1 · Black Ops II'); - expect(body).not.toContain('>Server'); expect(body).not.toContain('Show 25 more'); + expect(body).toContain('2 h ago'); + expect(body).toContain('Lobby 1 · Black Ops II'); + expect(body).not.toContain('>Server'); + expect(body).not.toContain('Show 25 more'); expect(body).toContain('matchPage=2'); }); it('preserves filters and page in match links and avoids linking deleted instances', () => { scope('?matchGame=t6&matchPage=2&matchSort=round&matchStatus=completed'); - const data = report(); data.page = 2; data.matches = [{ ...match, server_deleted_at: '2026-10-06' }]; + const data = report(); + data.page = 2; + data.matches = [{ ...match, server_deleted_at: '2026-10-06' }]; const body = render(MatchHistory, { props: { initial: data } }).body; - expect(body).toContain('/matches/1?returnTo=%2Fmatches'); expect(body).toContain('matchPage%3D2'); - expect(body).toContain('matchSort%3Dround'); expect(body).toContain('Historical instance'); - expect(body).not.toContain('href="/servers/'); expect(body).not.toContain('>Deleted<'); - expect(body).toContain('26–26 of 40 matches'); expect(body).toContain('Filters · 2'); + expect(body).toContain('/matches/1?returnTo=%2Fmatches'); + expect(body).toContain('matchPage%3D2'); + expect(body).toContain('matchSort%3Dround'); + expect(body).toContain('Historical instance'); + expect(body).not.toContain('href="/servers/'); + expect(body).not.toContain('>Deleted<'); + expect(body).toContain('26–26 of 40 matches'); + expect(body).toContain('Filters · 2'); }); it('keeps mobile filters and sorting in one labelled sheet with distinct controls and one named match link', () => { const body = render(MatchHistory, { props: { initial: report() } }).body; @@ -40,27 +78,59 @@ it('keeps mobile filters and sorting in one labelled sheet with distinct control const rows = body.slice(body.indexOf('')); expect(rows.match(/ match[1]); + const ids = [...body.matchAll(/\bid="([^"]+)"/g)].map((match) => match[1]); expect(new Set(ids).size).toBe(ids.length); expect(body).toContain('Updates every 10 seconds'); }); it('labels unfinished rounds and missing durations without inventing final results or detail links', () => { - const data = report(); data.matches = [{ ...match, status: 'interrupted', finalRound: null, durationMs: null, uniquePlayers: 0 }]; + const data = report(); + data.matches = [ + { ...match, status: 'interrupted', finalRound: null, durationMs: null, uniquePlayers: 0 } + ]; const body = render(MatchHistory, { props: { initial: data } }).body; - expect(body).toContain('Interrupted'); expect(body).toContain('Highest observed'); - expect(body).not.toContain('href="/matches/1'); expect(body).toContain('>—'); + expect(body).toContain('Interrupted'); + expect(body).toContain('Highest observed'); + expect(body).not.toContain('href="/matches/1'); + expect(body).toContain('>—'); expect(body).toContain('>0'); }); it('offers filter reset for an empty selection and retains the scoped instance view', () => { - scope('?matchQ=missing'); const data = report(); data.matches = []; data.total = 0; + scope('?matchQ=missing'); + const data = report(); + data.matches = []; + data.total = 0; const body = render(MatchHistory, { props: { initial: data } }).body; - expect(body).toContain('No matches found'); expect(body).toContain('Clear filters'); - const scoped = render(MatchHistory, { props: { initial: report(), server: match.server } }).body; - expect(scoped).not.toContain('id="match-search"'); expect(scoped).not.toContain('Historical instance'); + expect(body).toContain('No matches found'); + expect(body).toContain('Clear filters'); + const scoped = render(MatchHistory, { + props: { initial: report(), server: match.server } + }).body; + expect(scoped).not.toContain('id="match-search"'); + expect(scoped).not.toContain('Historical instance'); }); it('keeps custom instance names and calculates yesterday across DST and year boundaries', () => { - expect(matchInstanceLabel({ ...match, server: 'Weekend lobby' })).toBe('Weekend lobby · Black Ops II'); - expect(matchStartedLabel('2026-03-08T06:30:00Z', 'America/New_York', new Date('2026-03-09T04:15:00Z'))).toBe('21 h ago'); - expect(matchStartedLabel('2025-10-06T17:26:00Z', 'America/New_York', new Date('2026-10-06T20:00:00Z'))).toBe('Oct 6, 2025, 1:26 PM EDT'); - expect(matchStartedLabel('2026-10-06 17:26:00', 'America/New_York', new Date('2026-10-06T20:00:00Z'))).toBe('2 h ago'); + expect(matchInstanceLabel({ ...match, server: 'Weekend lobby' })).toBe( + 'Weekend lobby · Black Ops II' + ); + expect( + matchStartedLabel( + '2026-03-08T06:30:00Z', + 'America/New_York', + new Date('2026-03-09T04:15:00Z') + ) + ).toBe('21 h ago'); + expect( + matchStartedLabel( + '2025-10-06T17:26:00Z', + 'America/New_York', + new Date('2026-10-06T20:00:00Z') + ) + ).toBe('Oct 6, 2025, 1:26 PM EDT'); + expect( + matchStartedLabel( + '2026-10-06 17:26:00', + 'America/New_York', + new Date('2026-10-06T20:00:00Z') + ) + ).toBe('2 h ago'); }); diff --git a/tests/unit/ui-metrics-page.test.ts b/tests/unit/ui-metrics-page.test.ts index 52ead9c..305a11c 100644 --- a/tests/unit/ui-metrics-page.test.ts +++ b/tests/unit/ui-metrics-page.test.ts @@ -10,55 +10,121 @@ import { metricPage } from '../../src/lib/ui/metrics.js'; import { page } from '$app/state'; vi.mock('$app/state', () => ({ page: { url: new URL('http://localhost/metrics'), state: {} } })); -vi.mock('$app/navigation', () => ({ beforeNavigate: vi.fn(), afterNavigate: vi.fn(), goto: vi.fn(), replaceState: vi.fn() })); -const map = (i: number, changes: Partial = {}): MapMetrics => ({ game: 't5', map: `Map ${String(i).padStart(2, '0')}`, observed: 10, active: 5, idle: 5, unknown: 0, playerHours: i, full: 1, capacityObserved: 4, ...changes }); -const server = (i: number): ServerMetrics => ({ server: `t5_zombie_theater_${i}`, currentRegistered: i !== 1, observed: 10, active: i / 10, idle: 10 - i / 10, unknown: 0, playerHours: i }); +vi.mock('$app/navigation', () => ({ + beforeNavigate: vi.fn(), + afterNavigate: vi.fn(), + goto: vi.fn(), + replaceState: vi.fn() +})); +const map = (i: number, changes: Partial = {}): MapMetrics => ({ + game: 't5', + map: `Map ${String(i).padStart(2, '0')}`, + observed: 10, + active: 5, + idle: 5, + unknown: 0, + playerHours: i, + full: 1, + capacityObserved: 4, + ...changes +}); +const server = (i: number): ServerMetrics => ({ + server: `t5_zombie_theater_${i}`, + currentRegistered: i !== 1, + observed: 10, + active: i / 10, + idle: 10 - i / 10, + unknown: 0, + playerHours: i +}); function report(): MetricsData { const now = Date.parse('2026-10-06T12:00:00Z'); const summary = { observed: 10, active: 5, idle: 5, unknown: 2, playerHours: 20 }; - return { days: 7, since: now - 7 * 86400000, now, firstRetainedSampleAt: now - 14 * 86400000, latestSampleAt: now, - registeredServerCount: 24, scope: { game: null, map: null, timeZone: 'America/New_York' }, scopeOptions: { games: ['t5'], maps: [] }, - scopeFirstSampleAt: now - 14 * 86400000, scopeLatestSampleAt: now, summary, - maps: Array.from({ length: 24 }, (_, i) => map(i + 1)), servers: Array.from({ length: 24 }, (_, i) => server(i + 1)), - points: [{ ...summary, at: now - 86400000, startAt: now - 86400000, endAt: now, partial: true }], - previous: { since: now - 14 * 86400000, until: now - 7 * 86400000, summary: { ...summary, active: 4, idle: 6, playerHours: 10 }, firstSampleAt: now - 14 * 86400000, lastSampleAt: now - 7 * 86400000, partialHistory: true }, - pooledPressure: { value: 25, fullMapHours: 1, knownMapHours: 4, coveredMaps: 24, totalMaps: 24 }, heatmap: { timeZone: 'America/New_York', cells: [] } + return { + days: 7, + since: now - 7 * 86400000, + now, + firstRetainedSampleAt: now - 14 * 86400000, + latestSampleAt: now, + registeredServerCount: 24, + scope: { game: null, map: null, timeZone: 'America/New_York' }, + scopeOptions: { games: ['t5'], maps: [] }, + scopeFirstSampleAt: now - 14 * 86400000, + scopeLatestSampleAt: now, + summary, + maps: Array.from({ length: 24 }, (_, i) => map(i + 1)), + servers: Array.from({ length: 24 }, (_, i) => server(i + 1)), + points: [ + { ...summary, at: now - 86400000, startAt: now - 86400000, endAt: now, partial: true } + ], + previous: { + since: now - 14 * 86400000, + until: now - 7 * 86400000, + summary: { ...summary, active: 4, idle: 6, playerHours: 10 }, + firstSampleAt: now - 14 * 86400000, + lastSampleAt: now - 7 * 86400000, + partialHistory: true + }, + pooledPressure: { + value: 25, + fullMapHours: 1, + knownMapHours: 4, + coveredMaps: 24, + totalMaps: 24 + }, + heatmap: { timeZone: 'America/New_York', cells: [] } }; } function scope(search: string, stateSearch?: string) { - Object.assign(page, { url: new URL(`http://localhost/metrics${search}`), state: stateSearch ? { metricsTableSearch: stateSearch } : {} }); + Object.assign(page, { + url: new URL(`http://localhost/metrics${search}`), + state: stateSearch ? { metricsTableSearch: stateSearch } : {} + }); } beforeEach(() => scope('')); it('shows a summary and one paginated breakdown, preserving scope on range links', () => { scope('?game=t5&days=7&view=maps&mapPage=2'); - const body = render(MetricsPage, { props: { data: { ...report(), user: null, setupRequired: false } } }).body; + const body = render(MetricsPage, { + props: { data: { ...report(), user: null, setupRequired: false } } + }).body; expect(body).toContain('aria-label="Metrics breakdown"'); expect(body).toContain('aria-label="Map metrics table"'); expect(body).not.toContain('aria-label="Instance metrics table"'); - expect(body).toContain('16–24 of 24 maps'); + expect(body.replace(/\s+/g, ' ')).toContain('16–24 of 24 maps'); expect(body).toContain('game=t5&days=30&view=maps'); expect(body).toContain('Partial previous history'); expect(body).toContain('About this report'); expect(body).not.toContain('Explore this period'); expect(body).toContain('Updates every minute'); expect(body).not.toContain('>Refresh<'); - const heatmap = body.slice(body.indexOf('class="heatmap'), body.indexOf('aria-label="Metrics breakdown"')); - expect(heatmap).toContain(']*open/); data.maps = [map(1, { capacityObserved: 0 })]; - expect(render(MetricsExplore, { props: { report: data } }).body).toContain('No hosting trials meet the evidence threshold'); + expect(render(MetricsExplore, { props: { report: data } }).body).toContain( + 'No hosting trials meet the evidence threshold' + ); }); it('keeps accessible chart selection, partial periods and gaps while collapsing inspection controls', () => { const data = report(); - data.points.push({ at: data.now, startAt: data.now, endAt: data.now + 1000, partial: false, observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 }); - const body = render(MetricsChart, { props: { points: data.points, days: data.days, timeZone: data.scope.timeZone } }).body; + data.points.push({ + at: data.now, + startAt: data.now, + endAt: data.now + 1000, + partial: false, + observed: 0, + active: 0, + idle: 0, + unknown: 0, + playerHours: 0 + }); + const body = render(MetricsChart, { + props: { points: data.points, days: data.days, timeZone: data.scope.timeZone } + }).body; expect(body).toContain('aria-label="Selected period summary"'); expect(body).toContain('No captured time · not evidence of idle servers'); expect(body).toContain('tabindex="0"'); @@ -128,12 +214,16 @@ it('keeps accessible chart selection, partial periods and gaps while collapsing it('uses shallow URL state for the selected breakdown while retaining the common summary', () => { scope('?view=maps', '?view=instances&serverQ=theater&serverPage=2'); - let body = render(MetricsPage, { props: { data: { ...report(), user: null, setupRequired: false } } }).body; + let body = render(MetricsPage, { + props: { data: { ...report(), user: null, setupRequired: false } } + }).body; expect(body).toContain('aria-label="Instance metrics table"'); expect(body).not.toContain('aria-label="Map metrics table"'); - expect(body).toContain('16–24 of 24 instances'); + expect(body.replace(/\s+/g, ' ')).toContain('16–24 of 24 instances'); scope('?view=maps', '?view=insights'); - body = render(MetricsPage, { props: { data: { ...report(), user: null, setupRequired: false } } }).body; + body = render(MetricsPage, { + props: { data: { ...report(), user: null, setupRequired: false } } + }).body; expect(body).toContain('id="metrics-explore-title"'); expect(body).toContain('Time all lobbies were full'); expect(body).not.toContain('aria-label="Map metrics table"'); diff --git a/tests/unit/ui-metrics.test.ts b/tests/unit/ui-metrics.test.ts index 7560120..9db9122 100644 --- a/tests/unit/ui-metrics.test.ts +++ b/tests/unit/ui-metrics.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { compareMetricValues, metricDuration, metricNumber, metricPercent, metricPeriodLabels } from '../../src/lib/ui/metrics.js'; +import { + compareMetricValues, + metricDuration, + metricNumber, + metricPercent, + metricPeriodLabels +} from '../../src/lib/ui/metrics.js'; describe('metric presentation', () => { it('distinguishes unavailable evidence, zero, and tiny nonzero values', () => { @@ -15,23 +21,50 @@ describe('metric presentation', () => { it('sorts numerically while putting unavailable observations last in both directions', () => { const values = [2, null, 100, 0, 10]; - expect([...values].sort((a, b) => compareMetricValues(a, b, 'desc'))).toEqual([100, 10, 2, 0, null]); - expect([...values].sort((a, b) => compareMetricValues(a, b, 'asc'))).toEqual([0, 2, 10, 100, null]); + expect([...values].sort((a, b) => compareMetricValues(a, b, 'desc'))).toEqual([ + 100, + 10, + 2, + 0, + null + ]); + expect([...values].sort((a, b) => compareMetricValues(a, b, 'asc'))).toEqual([ + 0, + 2, + 10, + 100, + null + ]); }); it('labels daily buckets by their actual starting weekday in the report timezone', () => { - const points = Array.from({ length: 8 }, (_, day) => ({ startAt: Date.parse('2026-10-01T00:00:00Z') + day * 86_400_000 })); - expect(metricPeriodLabels(points, 7, 'America/New_York').map(item => item.label)).toEqual(['Wed', 'Thu', 'Fri', 'Sat', 'Sun', 'Mon', 'Tue', 'Wed']); + const points = Array.from({ length: 8 }, (_, day) => ({ + startAt: Date.parse('2026-10-01T00:00:00Z') + day * 86_400_000 + })); + expect(metricPeriodLabels(points, 7, 'America/New_York').map((item) => item.label)).toEqual( + ['Wed', 'Thu', 'Fri', 'Sat', 'Sun', 'Mon', 'Tue', 'Wed'] + ); expect(metricPeriodLabels(points, 7, 'UTC')[0].label).toBe('Thu'); expect(metricPeriodLabels(points, 7, 'UTC')[0].position).toBe(6.25); expect(metricPeriodLabels([], 7, 'UTC')).toEqual([]); }); it('keeps hourly and long-range labels sparse without relabelling UTC buckets as local days', () => { - const points = Array.from({ length: 31 }, (_, day) => ({ startAt: Date.parse('2026-10-01T00:00:00Z') + day * 86_400_000 })); + const points = Array.from({ length: 31 }, (_, day) => ({ + startAt: Date.parse('2026-10-01T00:00:00Z') + day * 86_400_000 + })); const labels = metricPeriodLabels(points, 30, 'UTC'); - expect(labels.length).toBeLessThanOrEqual(6); expect(labels[0]).toMatchObject({ label: 'Thu', date: '1' }); - const hourly = metricPeriodLabels(Array.from({ length: 25 }, (_, hour) => ({ startAt: Date.parse('2026-11-01T04:00:00Z') + hour * 3_600_000 })), 1, 'America/New_York'); - expect(hourly[0].label).toBe('12 AM'); expect(hourly[1].label).toBe('4 AM'); expect(hourly).toHaveLength(5); + expect(labels.length).toBeLessThanOrEqual(6); + expect(labels[0]).toMatchObject({ label: 'Thu', date: '1' }); + const hourly = metricPeriodLabels( + Array.from({ length: 25 }, (_, hour) => ({ + startAt: Date.parse('2026-11-01T04:00:00Z') + hour * 3_600_000 + })), + 1, + 'America/New_York' + ); + expect(hourly[0].label).toBe('12 AM'); + expect(hourly[1].label).toBe('4 AM'); + expect(hourly).toHaveLength(5); }); }); diff --git a/tests/unit/ui-navigation.test.ts b/tests/unit/ui-navigation.test.ts index 21e2850..2ac0232 100644 --- a/tests/unit/ui-navigation.test.ts +++ b/tests/unit/ui-navigation.test.ts @@ -8,9 +8,21 @@ import { fleet } from '../../src/lib/ui/stores/fleet.js'; import type { AuthUser } from '../../src/lib/auth.js'; const currentPage = writable({ url: new URL('http://localhost/campaigns/42'), data: {} }); -vi.mock('$app/stores', () => ({ page: { subscribe: (run: Parameters[0]) => currentPage.subscribe(run) } })); -vi.mock('$app/navigation', () => ({ goto: vi.fn(), beforeNavigate: vi.fn(), afterNavigate: vi.fn(), onNavigate: vi.fn() })); -afterEach(() => { fleet.set({ servers: [], operations: [], receivedAt: 0 }); currentPage.set({ url: new URL('http://localhost/campaigns/42'), data: {} }); }); +vi.mock('$app/stores', () => ({ + page: { + subscribe: (run: Parameters[0]) => currentPage.subscribe(run) + } +})); +vi.mock('$app/navigation', () => ({ + goto: vi.fn(), + beforeNavigate: vi.fn(), + afterNavigate: vi.fn(), + onNavigate: vi.fn() +})); +afterEach(() => { + fleet.set({ servers: [], operations: [], receivedAt: 0 }); + currentPage.set({ url: new URL('http://localhost/campaigns/42'), data: {} }); +}); describe('sidebar navigation', () => { it('exposes every allowed page in one navigation and marks the current page', () => { @@ -26,27 +38,65 @@ describe('sidebar navigation', () => { }); it('does not show invented attention counts before the fleet snapshot loads', () => { - expect(render(Navigation, { props: { role: 'admin' } }).body).not.toMatch(/needs? attention/); - fleet.set({ receivedAt: Date.now(), operations: [], servers: [{ - name: 't5_zombie_theater_1', game: 't5', map: 'zombie_theater', last_players: null, - last_map: null, last_gametype: null, last_seen_at: null, last_error: null, roster: [], - health: 'offline', held: [], active_since: null, inactive_since: null, game_started_at: null, game_ended_at: null - }] }); - expect(render(Navigation, { props: { role: 'admin' } }).body).toContain('aria-label="Servers, 1 needs attention"'); + expect(render(Navigation, { props: { role: 'admin' } }).body).not.toMatch( + /needs? attention/ + ); + fleet.set({ + receivedAt: Date.now(), + operations: [], + servers: [ + { + name: 't5_zombie_theater_1', + game: 't5', + map: 'zombie_theater', + last_players: null, + last_map: null, + last_gametype: null, + last_seen_at: null, + last_error: null, + roster: [], + health: 'offline', + held: [], + active_since: null, + inactive_since: null, + game_started_at: null, + game_ended_at: null + } + ] + }); + expect(render(Navigation, { props: { role: 'admin' } }).body).toContain( + 'aria-label="Servers, 1 needs attention"' + ); }); it('counts hosts without servers and clears the badge when the next host summary is healthy', () => { fleet.set({ receivedAt: Date.now(), operations: [], servers: [], hostAttention: 2 }); - expect(render(Navigation, { props: { role: 'admin' } }).body).toContain('aria-label="Hosts, 2 need attention"'); - fleet.update(value => ({ ...value, hostAttention: 0 })); - expect(render(Navigation, { props: { role: 'admin' } }).body).not.toMatch(/aria-label="Hosts, \d/); + expect(render(Navigation, { props: { role: 'admin' } }).body).toContain( + 'aria-label="Hosts, 2 need attention"' + ); + fleet.update((value) => ({ ...value, hostAttention: 0 })); + expect(render(Navigation, { props: { role: 'admin' } }).body).not.toMatch( + /aria-label="Hosts, \d/ + ); }); }); -const user: AuthUser = { id: '1', username: 'operator', displayName: 'Operator', role: 'operator', disabled: false, - mustChangePassword: false, createdAt: '', updatedAt: '', lastLoginAt: null }; +const user: AuthUser = { + id: '1', + username: 'operator', + displayName: 'Operator', + role: 'operator', + disabled: false, + mustChangePassword: false, + createdAt: '', + updatedAt: '', + lastLoginAt: null +}; const children = createRawSnippet(() => ({ render: () => '

    Page content

    ' })); -const shell = (overrides: Partial = {}) => render(Layout, { props: { children, data: { user: { ...user, ...overrides }, setupRequired: false } } }).body; +const shell = (overrides: Partial = {}) => + render(Layout, { + props: { children, data: { user: { ...user, ...overrides }, setupRequired: false } } + }).body; describe('application shell', () => { it('offers collapsible desktop navigation, a labelled mobile drawer, search and help', () => { diff --git a/tests/unit/ui-player-map-records.test.ts b/tests/unit/ui-player-map-records.test.ts index 133ae6d..457fe98 100644 --- a/tests/unit/ui-player-map-records.test.ts +++ b/tests/unit/ui-player-map-records.test.ts @@ -5,23 +5,59 @@ import PlayerProfilePage from '../../src/routes/leaderboard/players/[guid]/+page import PlayerMapCards from '../../src/lib/ui/components/PlayerMapCards.svelte'; import type { PlayerMapRecord, PlayerProfile, RecordEvidence } from '../../src/lib/leaderboard.js'; -vi.mock('$app/stores', () => ({ page: writable({ - url: new URL('http://localhost/leaderboard/players/101?players=2'), data: { user: null } -}) })); +vi.mock('$app/stores', () => ({ + page: writable({ + url: new URL('http://localhost/leaderboard/players/101?players=2'), + data: { user: null } + }) +})); vi.mock('$app/navigation', () => ({ goto: vi.fn(), invalidateAll: vi.fn() })); -const evidence: RecordEvidence = { matchId: 71, game: 't5', mapKey: 't5:theater', mapName: 'Kino Der Toten', - server: 't5_zombie_theater_1', startedAt: '2026-10-06T12:00:00Z', endedAt: '2026-10-06T13:00:00Z', - startObserved: true, presentAtStart: true, firstObservedRound: 0, value: 22 }; -const stats = { highestRound: 22, bestKills: 507, bestScore: 5290, kills: 722, roundsSurvived: 41, - headshots: 10, revives: 2, downs: 3, gamesPlayed: 9, gamesCompleted: 8 }; -const map: PlayerMapRecord = { ...stats, game: 't5', key: 't5:theater', name: 'Kino Der Toten', - lastSeen: evidence.endedAt!, ranks: { highestRound: 7, bestKills: 12, bestScore: 3 }, - records: { highestRound: evidence, bestKills: { ...evidence, matchId: 72, value: 507 }, - bestScore: { ...evidence, matchId: 73, value: 5290 } } }; +const evidence: RecordEvidence = { + matchId: 71, + game: 't5', + mapKey: 't5:theater', + mapName: 'Kino Der Toten', + server: 't5_zombie_theater_1', + startedAt: '2026-10-06T12:00:00Z', + endedAt: '2026-10-06T13:00:00Z', + startObserved: true, + presentAtStart: true, + firstObservedRound: 0, + value: 22 +}; +const stats = { + highestRound: 22, + bestKills: 507, + bestScore: 5290, + kills: 722, + roundsSurvived: 41, + headshots: 10, + revives: 2, + downs: 3, + gamesPlayed: 9, + gamesCompleted: 8 +}; +const map: PlayerMapRecord = { + ...stats, + game: 't5', + key: 't5:theater', + name: 'Kino Der Toten', + lastSeen: evidence.endedAt!, + ranks: { highestRound: 7, bestKills: 12, bestScore: 3 }, + records: { + highestRound: evidence, + bestKills: { ...evidence, matchId: 72, value: 507 }, + bestScore: { ...evidence, matchId: 73, value: 5290 } + } +}; function expectRankedRecords(body: string) { - for (const [label, rank] of [['Best round', 7], ['Best kills', 12], ['Best score', 3]]) { + for (const [label, rank] of [ + ['Best round', 7], + ['Best kills', 12], + ['Best score', 3] + ]) { expect(body).toContain(`aria-label="Kino Der Toten ${label} rank ${rank}"`); expect(body).toContain(`Rank #${rank}`); } @@ -30,16 +66,40 @@ function expectRankedRecords(body: string) { } it('shows each record rank and preserves match links in mobile map cards', () => { - expectRankedRecords(render(PlayerMapCards, { props: { maps: [map], returnTo: '/leaderboard/players/101?players=2' } }).body); + expectRankedRecords( + render(PlayerMapCards, { + props: { maps: [map], returnTo: '/leaderboard/players/101?players=2' } + }).body + ); }); it('shows the same ranks in the desktop profile with lifetime map scope explained', () => { - const profile: PlayerProfile = { guid: '101', name: 'Cherry', firstSeen: evidence.startedAt, lastSeen: evidence.endedAt!, - stats, records: map.records, maps: [map], matches: [], matchesTotal: 0, limit: 25, offset: 0 }; - const body = render(PlayerProfilePage, { props: { data: { profile, returnTo: '/leaderboard', - scope: { game: '', map: '', players: '2', metric: 'kills', entry: null } } } }).body; + const profile: PlayerProfile = { + guid: '101', + name: 'Cherry', + firstSeen: evidence.startedAt, + lastSeen: evidence.endedAt!, + stats, + records: map.records, + maps: [map], + matches: [], + matchesTotal: 0, + limit: 25, + offset: 0 + }; + const body = render(PlayerProfilePage, { + props: { + data: { + profile, + returnTo: '/leaderboard', + scope: { game: '', map: '', players: '2', metric: 'kills', entry: null } + } + } + }).body; const table = body.slice(body.indexOf('')); expectRankedRecords(table); - expect(body).toContain('Lifetime personal bests across all squad sizes, ranked against other players on each map.'); + expect(body.replace(/\s+/g, ' ')).toContain( + 'Lifetime personal bests across all squad sizes, ranked against other players on each map.' + ); expect(body).toContain('Tied records share a rank.'); }); diff --git a/tests/unit/ui-poll.test.ts b/tests/unit/ui-poll.test.ts index c1fe5c5..45b4d2f 100644 --- a/tests/unit/ui-poll.test.ts +++ b/tests/unit/ui-poll.test.ts @@ -8,38 +8,64 @@ beforeEach(() => { documentStub = Object.assign(new EventTarget(), { visibilityState: 'visible' }); vi.stubGlobal('document', documentStub); }); -afterEach(() => { stop?.(); stop = undefined; vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); }); -const visibility = (state: string) => { documentStub.visibilityState = state; documentStub.dispatchEvent(new Event('visibilitychange')); }; +afterEach(() => { + stop?.(); + stop = undefined; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); +const visibility = (state: string) => { + documentStub.visibilityState = state; + documentStub.dispatchEvent(new Event('visibilitychange')); +}; describe('visible page polling', () => { it('waits for a request to finish before scheduling the next request', async () => { let finish!: () => void; - const task = vi.fn(() => new Promise(resolve => { finish = resolve; })); + const task = vi.fn( + () => + new Promise((resolve) => { + finish = resolve; + }) + ); stop = pollVisible(task, 1000); await vi.advanceTimersByTimeAsync(5000); visibility('visible'); expect(task).toHaveBeenCalledTimes(1); - finish(); await vi.advanceTimersByTimeAsync(999); + finish(); + await vi.advanceTimersByTimeAsync(999); expect(task).toHaveBeenCalledTimes(1); await vi.advanceTimersByTimeAsync(1); expect(task).toHaveBeenCalledTimes(2); }); it('pauses when hidden and refreshes on return, including hiding during a request', async () => { let finish!: () => void; - const task = vi.fn(() => new Promise(resolve => { finish = resolve; })); + const task = vi.fn( + () => + new Promise((resolve) => { + finish = resolve; + }) + ); stop = pollVisible(task, 1000); - visibility('hidden'); finish(); + visibility('hidden'); + finish(); await vi.advanceTimersByTimeAsync(5000); expect(task).toHaveBeenCalledTimes(1); visibility('visible'); expect(task).toHaveBeenCalledTimes(2); - finish(); stop(); - visibility('visible'); await vi.advanceTimersByTimeAsync(5000); + finish(); + stop(); + visibility('visible'); + await vi.advanceTimersByTimeAsync(5000); expect(task).toHaveBeenCalledTimes(2); }); it('recovers after a failed request and supports a delayed initial refresh', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}); - const task = vi.fn().mockRejectedValueOnce(new Error('offline')).mockResolvedValue(undefined); + const task = vi + .fn() + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValue(undefined); stop = pollVisible(task, 1000, false); expect(task).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(2000); diff --git a/tests/unit/ui-provisioning-detail.test.ts b/tests/unit/ui-provisioning-detail.test.ts index a79aeed..f17bf8e 100644 --- a/tests/unit/ui-provisioning-detail.test.ts +++ b/tests/unit/ui-provisioning-detail.test.ts @@ -5,34 +5,87 @@ import type { ProvisioningDetail, RecoveryObservation } from '../../src/lib/prov import { provisioningNeedsRefresh } from '../../src/lib/ui/provisioning-presentation.js'; import ProvisioningPage from '../../src/routes/provisioning/[id]/+page.svelte'; -vi.mock('$app/stores', () => ({ page: writable({ - url: new URL('http://localhost/provisioning/operation-a'), data: { user: null } -}) })); +vi.mock('$app/stores', () => ({ + page: writable({ + url: new URL('http://localhost/provisioning/operation-a'), + data: { user: null } + }) +})); vi.mock('$app/navigation', () => ({ goto: vi.fn() })); const start = '2026-10-09T12:00:00Z'; const finish = '2026-10-09T12:01:00Z'; function fixture(state = 'done', recovery: RecoveryObservation | null = null): ProvisioningDetail { return { - deployment: { id: 'operation-a', server: 't5_zombie_theater_1', action: 'restart', state, - stage: 'Completed', server_deleted_at: null, server_exists: 1, started_at: start, finished_at: finish, - triggered_by: 'watchdog:incident-a', requested_by: 'Maxis', error: null, warnings: '[]' }, - generatedAt: finish, events: [], activity: { total: 0, latestId: null, failureExcerpt: null }, - sources: [{ kind: 'watchdog', source_id: 'incident-a', label: 'Deleted watchdog incident', href: null }], - recipe: null, blueprintName: null, canResumeCleanup: false, canResumeRebuild: false, - canResumeVerification: false, canCancelRemoval: false, + deployment: { + id: 'operation-a', + server: 't5_zombie_theater_1', + action: 'restart', + state, + stage: 'Completed', + server_deleted_at: null, + server_exists: 1, + started_at: start, + finished_at: finish, + triggered_by: 'watchdog:incident-a', + requested_by: 'Maxis', + error: null, + warnings: '[]' + }, + generatedAt: finish, + events: [], + activity: { total: 0, latestId: null, failureExcerpt: null }, + sources: [ + { + kind: 'watchdog', + source_id: 'incident-a', + label: 'Deleted watchdog incident', + href: null + } + ], + recipe: null, + blueprintName: null, + canResumeCleanup: false, + canResumeRebuild: false, + canResumeVerification: false, + canCancelRemoval: false, collection: { id: 'collection-a', name: 'Evening lobbies' }, - progress: { version: 1, phase: 'verify-game', phaseStartedAt: '2026-10-09T12:00:50Z', observedAt: finish, - blocker: null, checks: [], checkpoints: [], command: null, recovery, recipeStatus: null }, - agentOperation: { id: 'command-a', hostId: 'host-a', state: 'completed', createdAt: start, updatedAt: finish, error: null }, - capacity: { evaluatedAt: finish, pools: [] }, warningMessages: [] + progress: { + version: 1, + phase: 'verify-game', + phaseStartedAt: '2026-10-09T12:00:50Z', + observedAt: finish, + blocker: null, + checks: [], + checkpoints: [], + command: null, + recovery, + recipeStatus: null + }, + agentOperation: { + id: 'command-a', + hostId: 'host-a', + state: 'completed', + createdAt: start, + updatedAt: finish, + error: null + }, + capacity: { evaluatedAt: finish, pools: [] }, + warningMessages: [] }; } function pageBody(detail: ProvisioningDetail) { - return render(ProvisioningPage, { props: { data: { detail, user: null, setupRequired: false } } }).body; + return render(ProvisioningPage, { + props: { data: { detail, user: null, setupRequired: false } } + }).body; } function recovery(state: RecoveryObservation['state']): RecoveryObservation { - return { state, at: finish, message: 'Recovery status updated', ...(state === 'transferred' ? { operationId: 'rebuild-a' } : {}) }; + return { + state, + at: finish, + message: 'Recovery status updated', + ...(state === 'transferred' ? { operationId: 'rebuild-a' } : {}) + }; } describe('provisioning detail summary and technical details', () => { @@ -53,14 +106,18 @@ describe('provisioning detail summary and technical details', () => { it('keeps operation, command and source IDs and copy controls in closed Technical details', () => { const body = pageBody(fixture()); - const technical = body.match(/]*class="technical-details[^>]*>[\s\S]*?<\/details>/)?.[0]; + const technical = body.match( + /]*class="technical-details[^>]*>[\s\S]*?<\/details>/ + )?.[0]; expect(technical).toBeDefined(); expect(technical).toContain('Technical details'); expect(technical!.split('>')[0]).not.toMatch(/\bopen\b/); const outside = body.replace(technical!, ''); for (const [value, label] of [ - ['operation-a', 'operation ID'], ['command-a', 'agent command ID'], - ['collection-a', 'collection launch ID'], ['incident-a', 'Deleted watchdog incident source ID'] + ['operation-a', 'operation ID'], + ['command-a', 'agent command ID'], + ['collection-a', 'collection launch ID'], + ['incident-a', 'Deleted watchdog incident source ID'] ]) { const code = new RegExp(`]*>${value}`); expect(technical).toMatch(code); @@ -100,38 +157,49 @@ describe('provisioning detail summary and technical details', () => { }); describe('provisioning refresh policy', () => { - it.each(['queued', 'waiting', 'running', 'awaiting-health'])('refreshes %s operations', state => { - expect(provisioningNeedsRefresh(fixture(state))).toBe(true); - }); + it.each(['queued', 'waiting', 'running', 'awaiting-health'])( + 'refreshes %s operations', + (state) => { + expect(provisioningNeedsRefresh(fixture(state))).toBe(true); + } + ); - it.each(['done', 'failed', 'interrupted', 'warning', 'cancelled'])('stops refreshing %s operations without held recovery', state => { - const detail = fixture(state); - expect(provisioningNeedsRefresh(detail)).toBe(false); - for (const value of ['released', 'transferred', 'unknown'] as const) { - detail.progress!.recovery = recovery(value); + it.each(['done', 'failed', 'interrupted', 'warning', 'cancelled'])( + 'stops refreshing %s operations without held recovery', + (state) => { + const detail = fixture(state); + expect(provisioningNeedsRefresh(detail)).toBe(false); + for (const value of ['released', 'transferred', 'unknown'] as const) { + detail.progress!.recovery = recovery(value); + expect(provisioningNeedsRefresh(detail)).toBe(false); + } + detail.progress = null; expect(provisioningNeedsRefresh(detail)).toBe(false); } - detail.progress = null; - expect(provisioningNeedsRefresh(detail)).toBe(false); - }); + ); - it.each(['released', 'transferred'] as const)('keeps failed restart recovery refreshing until %s without marking it live', state => { - const detail = fixture('failed', recovery('held')); - expect(provisioningNeedsRefresh(detail)).toBe(true); - const held = pageBody(detail); - expect(held).toContain('Recovery capacity held'); - expect(held).not.toContain('Updates every 5 seconds'); - expect(held).not.toContain('View recovery operation'); + it.each(['released', 'transferred'] as const)( + 'keeps failed restart recovery refreshing until %s without marking it live', + (state) => { + const detail = fixture('failed', recovery('held')); + expect(provisioningNeedsRefresh(detail)).toBe(true); + const held = pageBody(detail); + expect(held).toContain('Recovery capacity held'); + expect(held).not.toContain('Updates every 5 seconds'); + expect(held).not.toContain('View recovery operation'); - detail.progress!.recovery = recovery(state); - expect(provisioningNeedsRefresh(detail)).toBe(false); - const settled = pageBody(detail); - expect(settled).not.toContain('Recovery capacity held'); - expect(settled).not.toContain('Updates every 5 seconds'); - expect(settled).toContain(state === 'released' ? 'Recovery capacity released' : 'Recovery transferred'); - if (state === 'transferred') { - expect(settled).toContain('href="/provisioning/rebuild-a"'); - expect(settled).toContain('View recovery operation'); + detail.progress!.recovery = recovery(state); + expect(provisioningNeedsRefresh(detail)).toBe(false); + const settled = pageBody(detail); + expect(settled).not.toContain('Recovery capacity held'); + expect(settled).not.toContain('Updates every 5 seconds'); + expect(settled).toContain( + state === 'released' ? 'Recovery capacity released' : 'Recovery transferred' + ); + if (state === 'transferred') { + expect(settled).toContain('href="/provisioning/rebuild-a"'); + expect(settled).toContain('View recovery operation'); + } } - }); + ); }); diff --git a/tests/unit/ui-rebalancing.test.ts b/tests/unit/ui-rebalancing.test.ts index 3c9c935..412756e 100644 --- a/tests/unit/ui-rebalancing.test.ts +++ b/tests/unit/ui-rebalancing.test.ts @@ -1,22 +1,65 @@ import { expect, it } from 'vitest'; import { render } from 'svelte/server'; -import type { RebalanceCandidate, RebalancePreview, RebalanceRun } from '../../src/lib/rebalancing.js'; +import type { + RebalanceCandidate, + RebalancePreview, + RebalanceRun +} from '../../src/lib/rebalancing.js'; import RebalancePreviewView from '../../src/lib/ui/components/capacity/RebalancePreviewView.svelte'; import RebalanceRunView from '../../src/lib/ui/components/capacity/RebalanceRunView.svelte'; const at = '2026-10-08T12:00:00Z'; -const move: RebalanceCandidate = { server: 't5_zombie_theater_1', instanceId: 'instance-a', generation: 1, - sourceHostId: 'source-a', sourceHostName: 'Source A', targetHostId: 'target-b', targetHostName: 'Target B', improvement: 0.1 }; -const preview: RebalancePreview = { id: 'preview-a', createdAt: at, expiresAt: '2026-10-08T12:05:00Z', - settings: { hostIds: ['source-a', 'target-b'], game: 'all', maxMoves: 5 }, moves: [move], - hosts: [{ id: 'source-a', name: 'Source A', before: { cpuPercent: 75, memoryPercent: 50, instances: 6 }, after: { cpuPercent: 62.5, memoryPercent: 42, instances: 5 } }], - exclusions: [{ server: 't6_zm_transit_1', reason: 'Lobby has players.' }], warnings: ['Occupied lobbies stay in place.'] }; -const run: RebalanceRun = { id: 'run-a', createdAt: at, updatedAt: at, settings: preview.settings, state: 'running', message: 'Moving one lobby.', - moves: [{ ...move, id: 'move-a', state: 'moving', operationId: 'operation-a', message: 'Waiting for destination health.', updatedAt: at }] }; +const move: RebalanceCandidate = { + server: 't5_zombie_theater_1', + instanceId: 'instance-a', + generation: 1, + sourceHostId: 'source-a', + sourceHostName: 'Source A', + targetHostId: 'target-b', + targetHostName: 'Target B', + improvement: 0.1 +}; +const preview: RebalancePreview = { + id: 'preview-a', + createdAt: at, + expiresAt: '2026-10-08T12:05:00Z', + settings: { hostIds: ['source-a', 'target-b'], game: 'all', maxMoves: 5 }, + moves: [move], + hosts: [ + { + id: 'source-a', + name: 'Source A', + before: { cpuPercent: 75, memoryPercent: 50, instances: 6 }, + after: { cpuPercent: 62.5, memoryPercent: 42, instances: 5 } + } + ], + exclusions: [{ server: 't6_zm_transit_1', reason: 'Lobby has players.' }], + warnings: ['Occupied lobbies stay in place.'] +}; +const run: RebalanceRun = { + id: 'run-a', + createdAt: at, + updatedAt: at, + settings: preview.settings, + state: 'running', + message: 'Moving one lobby.', + moves: [ + { + ...move, + id: 'move-a', + state: 'moving', + operationId: 'operation-a', + message: 'Waiting for destination health.', + updatedAt: at + } + ] +}; const noop = () => {}; it('shows current and projected resources, the move destination, and exclusion reasons before starting', () => { - const body = render(RebalancePreviewView, { props: { preview, expired: false, onstart: noop } }).body; + const body = render(RebalancePreviewView, { + props: { preview, expired: false, onstart: noop } + }).body; expect(body).toContain('6 → 5'); expect(body).toContain('75% → 62.5%'); expect(body).toContain('50% → 42%'); @@ -27,11 +70,15 @@ it('shows current and projected resources, the move destination, and exclusion r expect(body).not.toMatch(/]*disabled/); }); -it.each(['expired', 'empty', 'blocked'] as const)('prevents starting a %s preview', condition => { - const body = render(RebalancePreviewView, { props: { - preview: condition === 'empty' ? { ...preview, moves: [] } : preview, - expired: condition === 'expired', disabled: condition === 'blocked', onstart: noop - } }).body; +it.each(['expired', 'empty', 'blocked'] as const)('prevents starting a %s preview', (condition) => { + const body = render(RebalancePreviewView, { + props: { + preview: condition === 'empty' ? { ...preview, moves: [] } : preview, + expired: condition === 'expired', + disabled: condition === 'blocked', + onstart: noop + } + }).body; expect(body).toMatch(/]*disabled/); if (condition === 'expired') expect(body).toContain('This preview has expired.'); if (condition === 'empty') expect(body).toContain('No eligible moves would improve'); @@ -41,15 +88,23 @@ it('keeps run progress tied to its provisioning record and explains stopping sem const body = render(RebalanceRunView, { props: { run, canManage: true, onstop: noop } }).body; expect(body).toContain('0 of 1 moves completed'); expect(body).toContain('Waiting for destination health.'); - expect(body).toContain('href="/provisioning/operation-a?returnTo=%2Fcapacity%3Frebalance%3D1%26rebalanceRun%3Drun-a"'); + expect(body).toContain( + 'href="/provisioning/operation-a?returnTo=%2Fcapacity%3Frebalance%3D1%26rebalanceRun%3Drun-a"' + ); expect(body).toContain('Stop remaining moves'); - expect(body).toContain('A move already in progress continues through health verification.'); + expect(body.replace(/\s+/g, ' ')).toContain( + 'A move already in progress continues through health verification.' + ); }); it('hides stop from viewers and terminal or already-stopping runs', () => { - expect(render(RebalanceRunView, { props: { run, canManage: false, onstop: noop } }).body).not.toContain('Stop remaining moves'); + expect( + render(RebalanceRunView, { props: { run, canManage: false, onstop: noop } }).body + ).not.toContain('Stop remaining moves'); for (const state of ['stopping', 'done', 'stopped', 'failed'] as const) { - const body = render(RebalanceRunView, { props: { run: { ...run, state }, canManage: true, onstop: noop } }).body; + const body = render(RebalanceRunView, { + props: { run: { ...run, state }, canManage: true, onstop: noop } + }).body; expect(body).not.toContain('Stop remaining moves'); expect(body).toContain('View provisioning'); } diff --git a/tests/unit/ui-server-card.test.ts b/tests/unit/ui-server-card.test.ts index 97d2a18..8dc3983 100644 --- a/tests/unit/ui-server-card.test.ts +++ b/tests/unit/ui-server-card.test.ts @@ -6,25 +6,44 @@ import ServerCard from '../../src/lib/ui/components/ServerCard.svelte'; import { telemetryByServer, uiNow } from '../../src/lib/ui/stores/servers.js'; import { gameEvent } from '../helpers/telemetry.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/?filter=occupied') }) })); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/?filter=occupied') }) +})); const name = 't5_zombie_theater_1'; const at = (age: number) => new Date(get(uiNow) - age).toISOString(); -const telemetry = (round = 31, age = 1000) => ({ ...gameEvent('heartbeat', round, 4, name), receivedAt: at(age) }); +const telemetry = (round = 31, age = 1000) => ({ + ...gameEvent('heartbeat', round, 4, name), + receivedAt: at(age) +}); function html(props: Partial> = {}) { - return render(ServerCard, { props: { name, health: 'online', players: 4, capacity: 4, - telemetry: telemetry(), gameStartedAt: at(6_000_000), ...props } }).body.replaceAll(//g, ''); + return render(ServerCard, { + props: { + name, + health: 'online', + players: 4, + capacity: 4, + telemetry: telemetry(), + gameStartedAt: at(6_000_000), + ...props + } + }).body.replaceAll(//g, ''); } afterEach(() => telemetryByServer.set(new Map())); describe('compact server cards', () => { it('leads with identity and three labelled metrics, without healthy freshness or duplicate activity clocks', () => { - const body = html({ gametype: 'zom', activeSince: at(6_120_000), lastSeenAt: at(1000), longestMatch: true }); + const body = html({ + gametype: 'zom', + activeSince: at(6_120_000), + lastSeenAt: at(1000), + longestMatch: true + }); expect(body).toContain('Kino Der Toten'); expect(body).toContain('Lobby 1'); expect(body).toContain('BO1'); - expect(body).toMatch(/]*>Players<\/dt>]*>4]*>\/4<\/span><\/dd>/); - expect(body).toMatch(/]*>Round<\/dt>]*>31<\/dd>/); - expect(body).toMatch(/]*>Match time<\/dt>]*>1h 40m<\/dd>/); + expect(body).toMatch(/]*>Players<\/dt>\s*]*>4]*>\/4<\/span><\/dd>/); + expect(body).toMatch(/]*>Round<\/dt>\s*]*>31<\/dd>/); + expect(body).toMatch(/]*>Match time<\/dt>\s*]*>1h 40m<\/dd>/); expect(body).toContain('Longest match'); expect(body).not.toMatch(/Last seen|Last reported|Zombies Classic|>Active { it('uses the newest fresh report while respecting explicit round data and match boundaries', () => { telemetryByServer.set(new Map([[name, telemetry(35, 1000)]])); - expect(html({ telemetry: telemetry(31, 10_000) })).toMatch(/]*>Round<\/dt>]*>35<\/dd>/); - expect(html({ round: 40 })).toMatch(/]*>Round<\/dt>]*>40<\/dd>/); - expect(html({ round: null })).toMatch(/]*>Round<\/dt>]*>—<\/dd>/); - expect(html({ gameStartedAt: at(0) })).toMatch(/]*>Round<\/dt>]*>—<\/dd>/); + expect(html({ telemetry: telemetry(31, 10_000) })).toMatch( + /]*>Round<\/dt>\s*]*>35<\/dd>/ + ); + expect(html({ round: 40 })).toMatch(/]*>Round<\/dt>\s*]*>40<\/dd>/); + expect(html({ round: null })).toMatch(/]*>Round<\/dt>\s*]*>—<\/dd>/); + expect(html({ gameStartedAt: at(0) })).toMatch(/]*>Round<\/dt>\s*]*>—<\/dd>/); telemetryByServer.set(new Map()); - expect(html({ telemetry: telemetry(31, 100_000) })).toMatch(/]*>Round<\/dt>]*>—<\/dd>/); + expect(html({ telemetry: telemetry(31, 100_000) })).toMatch( + /]*>Round<\/dt>\s*]*>—<\/dd>/ + ); expect(html({ gameEndedAt: at(1000), round: 40 })).toContain('Match ended'); expect(html({ gameEndedAt: at(1000), round: 40 })).not.toMatch(/]*>40<\/dd>/); }); it('gives the roster real column headers and formatted scores without repeating ping units', () => { - const body = html({ roster: [ - { num: 0, guid: 'one', name: 'Vexx', score: 99170, ping: 36 }, - { num: 1, guid: 'two', name: 'Connecting player', score: null, ping: 999 } - ] }); + const body = html({ + roster: [ + { num: 0, guid: 'one', name: 'Vexx', score: 99170, ping: 36 }, + { num: 1, guid: 'two', name: 'Connecting player', score: null, ping: 999 } + ] + }); expect(body).toContain('aria-label="Current players"'); expect(body).toMatch(/]*scope="col"[^>]*>Player<\/th>/); expect(body).toContain('99,170'); @@ -73,7 +98,12 @@ describe('compact server cards', () => { }); it('collapses empty healthy lobbies to readiness and empty duration', () => { - const body = html({ players: 0, inactiveSince: at(9 * 60_000), roster: [], compactEmpty: true }); + const body = html({ + players: 0, + inactiveSince: at(9 * 60_000), + roster: [], + compactEmpty: true + }); expect(body).toContain('Ready'); expect(body).toContain('Empty for 9m 00s'); expect(body).not.toContain(' = {}): FleetServer { return { - name: 't5_zombie_theater_1', game: 't5', map: 'zombie_theater', - last_players: 0, last_map: 'zombie_theater', last_gametype: 'zom', - last_seen_at: new Date(timestamp).toISOString(), last_error: null, - roster: [], health: 'online', held: [], active_since: null, - inactive_since: new Date(epoch).toISOString(), game_started_at: null, - game_ended_at: null, ...changes + name: 't5_zombie_theater_1', + game: 't5', + map: 'zombie_theater', + last_players: 0, + last_map: 'zombie_theater', + last_gametype: 'zom', + last_seen_at: new Date(timestamp).toISOString(), + last_error: null, + roster: [], + health: 'online', + held: [], + active_since: null, + inactive_since: new Date(epoch).toISOString(), + game_started_at: null, + game_ended_at: null, + ...changes }; } function status(timestamp: number, changes: Partial = {}): ServerStatus { return { - server: 't5_zombie_theater_1', players: 0, map: 'zombie_theater', gametype: 'zom', - roster: [], health: 'online', lastSeenAt: new Date(timestamp).toISOString(), - lastError: null, endpoint: null, reason: null, - activeSince: null, inactiveSince: new Date(epoch).toISOString(), - gameStartedAt: null, gameEndedAt: null, ...changes + server: 't5_zombie_theater_1', + players: 0, + map: 'zombie_theater', + gametype: 'zom', + roster: [], + health: 'online', + lastSeenAt: new Date(timestamp).toISOString(), + lastError: null, + endpoint: null, + reason: null, + activeSince: null, + inactiveSince: new Date(epoch).toISOString(), + gameStartedAt: null, + gameEndedAt: null, + ...changes }; } function emit(serverNow: number, entries: ServerStatus[]) { - source.dispatchEvent(new MessageEvent('server-status', { data: JSON.stringify({ serverNow, entries }) })); + source.dispatchEvent( + new MessageEvent('server-status', { data: JSON.stringify({ serverNow, entries }) }) + ); flushFrame(); } beforeEach(() => { vi.resetModules(); - vi.useFakeTimers({ toFake: ['Date', 'performance', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval'] }); + vi.useFakeTimers({ + toFake: [ + 'Date', + 'performance', + 'setTimeout', + 'clearTimeout', + 'setInterval', + 'clearInterval' + ] + }); vi.setSystemTime(epoch); frames = new Map(); let frameId = 0; - vi.stubGlobal('requestAnimationFrame', vi.fn((callback: FrameRequestCallback) => { - frames.set(++frameId, callback); - return frameId; - })); - vi.stubGlobal('cancelAnimationFrame', vi.fn((id: number) => { frames.delete(id); })); + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + frames.set(++frameId, callback); + return frameId; + }) + ); + vi.stubGlobal( + 'cancelAnimationFrame', + vi.fn((id: number) => { + frames.delete(id); + }) + ); snapshot = { serverNow: epoch, servers: [server()], integrations: {} }; - vi.stubGlobal('fetch', vi.fn(async (url: string) => new Response(JSON.stringify( - url === '/api/servers' ? snapshot : { operations: [] } - )))); - vi.stubGlobal('EventSource', class extends EventTarget { - constructor() { super(); source = this; } - close() {} - }); + vi.stubGlobal( + 'fetch', + vi.fn( + async (url: string) => + new Response(JSON.stringify(url === '/api/servers' ? snapshot : { operations: [] })) + ) + ); + vi.stubGlobal( + 'EventSource', + class extends EventTarget { + constructor() { + super(); + // eslint-disable-next-line @typescript-eslint/no-this-alias -- Expose the constructed EventSource mock so tests can dispatch stream events. + source = this; + } + close() {} + } + ); }); afterEach(() => { - stop?.(); stop = undefined; + stop?.(); + stop = undefined; vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); @@ -92,7 +143,12 @@ describe('animation-frame stream batching', () => { const { fleetServers } = await setup(); expect(get(fleetServers)[0]).toMatchObject({ capacity: 4, capacity_observed_at: at }); await vi.advanceTimersByTimeAsync(1000); - emit(epoch + 1000, [status(epoch + 1000, { capacity: 8, capacityObservedAt: new Date(epoch + 1000).toISOString() })]); + emit(epoch + 1000, [ + status(epoch + 1000, { + capacity: 8, + capacityObservedAt: new Date(epoch + 1000).toISOString() + }) + ]); expect(get(fleetServers)[0].capacity).toBe(8); await vi.advanceTimersByTimeAsync(1000); emit(epoch + 2000, [status(epoch + 2000, { capacity: null, capacityObservedAt: null })]); @@ -101,15 +157,31 @@ describe('animation-frame stream batching', () => { it('expires capacity independently of fresh rosters and rejects the previous lifecycle limit', async () => { const at = (offset = 0) => new Date(epoch + offset).toISOString(); - snapshot.servers = [server(epoch, { capacity: 4, capacity_observed_at: at(-90_000), - container_state: 'running', container_checked_at: at() })]; + snapshot.servers = [ + server(epoch, { + capacity: 4, + capacity_observed_at: at(-90_000), + container_state: 'running', + container_checked_at: at() + }) + ]; const { fleetServers } = await setup(); expect(get(fleetServers)[0].capacity).toBe(4); await vi.advanceTimersByTimeAsync(1000); - queueEvent('server-telemetry', { serverNow: epoch + 1000, entries: [{ ...gameEvent('heartbeat', 15, 3), receivedAt: at(1000) }] }); + queueEvent('server-telemetry', { + serverNow: epoch + 1000, + entries: [{ ...gameEvent('heartbeat', 15, 3), receivedAt: at(1000) }] + }); flushFrame(); - expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, capacity: null, capacity_observed_at: null }); - emit(epoch + 1000, [status(epoch + 1000, { capacity: 4, capacityObservedAt: at(), telemetryAfter: at(1) })]); + expect(get(fleetServers)[0]).toMatchObject({ + health: 'online', + last_players: 3, + capacity: null, + capacity_observed_at: null + }); + emit(epoch + 1000, [ + status(epoch + 1000, { capacity: 4, capacityObservedAt: at(), telemetryAfter: at(1) }) + ]); expect(get(fleetServers)[0]).toMatchObject({ capacity: null, capacity_observed_at: null }); }); @@ -118,49 +190,88 @@ describe('animation-frame stream batching', () => { emit(epoch, [status(epoch), status(epoch, { server: 'two' })]); const before = get(stores.statusByServer); await vi.advanceTimersByTimeAsync(1000); - const watched = [stores.statusByServer, stores.statusReceivedAt, stores.telemetryByServer, stores.matchEventsByServer, stores.uiNow]; + const watched = [ + stores.statusByServer, + stores.statusReceivedAt, + stores.telemetryByServer, + stores.matchEventsByServer, + stores.uiNow + ]; const changes = watched.map(() => vi.fn()); const unsubscribe = watched.map((store, index) => store.subscribe(changes[index])); - changes.forEach(change => change.mockClear()); + changes.forEach((change) => change.mockClear()); vi.mocked(requestAnimationFrame).mockClear(); try { for (let i = 0; i < 100; i++) { - queueEvent('server-status', { delta: true, serverNow: epoch + 1000 + i, - entries: [status(epoch + 1000, { players: i })] }); - queueEvent('server-telemetry', { serverNow: epoch + 1000 + i, - entries: [{ ...gameEvent(i === 0 ? 'game_start' : 'heartbeat', i, 1), receivedAt: new Date(epoch + 1000 + i).toISOString() }] }); + queueEvent('server-status', { + delta: true, + serverNow: epoch + 1000 + i, + entries: [status(epoch + 1000, { players: i })] + }); + queueEvent('server-telemetry', { + serverNow: epoch + 1000 + i, + entries: [ + { + ...gameEvent(i === 0 ? 'game_start' : 'heartbeat', i, 1), + receivedAt: new Date(epoch + 1000 + i).toISOString() + } + ] + }); } expect(requestAnimationFrame).toHaveBeenCalledTimes(1); - changes.forEach(change => expect(change).not.toHaveBeenCalled()); + changes.forEach((change) => expect(change).not.toHaveBeenCalled()); expect(get(stores.statusByServer)).toBe(before); expect(before.get('t5_zombie_theater_1')?.players).toBe(0); flushFrame(); - changes.forEach(change => expect(change).toHaveBeenCalledTimes(1)); + changes.forEach((change) => expect(change).toHaveBeenCalledTimes(1)); expect(get(stores.statusByServer).size).toBe(2); expect(get(stores.statusByServer).get('t5_zombie_theater_1')?.players).toBe(99); - expect(get(stores.telemetryByServer).get('t5_zombie_theater_1')).toMatchObject({ event: 'heartbeat', round: 99 }); - expect(get(stores.matchEventsByServer).get('t5_zombie_theater_1')?.event).toBe('game_start'); + expect(get(stores.telemetryByServer).get('t5_zombie_theater_1')).toMatchObject({ + event: 'heartbeat', + round: 99 + }); + expect(get(stores.matchEventsByServer).get('t5_zombie_theater_1')?.event).toBe( + 'game_start' + ); - queueEvent('server-telemetry', { entries: [{ ...gameEvent('heartbeat', 100, 1), receivedAt: new Date(epoch + 1100).toISOString() }] }); + queueEvent('server-telemetry', { + entries: [ + { + ...gameEvent('heartbeat', 100, 1), + receivedAt: new Date(epoch + 1100).toISOString() + } + ] + }); flushFrame(); expect(requestAnimationFrame).toHaveBeenCalledTimes(2); expect(changes[2]).toHaveBeenCalledTimes(2); expect(changes[3]).toHaveBeenCalledTimes(1); - } finally { unsubscribe.forEach(unsubscribe => unsubscribe()); } + } finally { + unsubscribe.forEach((unsubscribe) => unsubscribe()); + } }); it('applies snapshots and subsequent deltas in order, including empty snapshots', async () => { const { statusByServer, telemetryByServer, matchEventsByServer } = await setup(); - const telemetry = (event: GameEvent['event'], name: string) => ({ ...gameEvent(event, 1, 1, name), receivedAt: new Date(epoch).toISOString() }); + const telemetry = (event: GameEvent['event'], name: string) => ({ + ...gameEvent(event, 1, 1, name), + receivedAt: new Date(epoch).toISOString() + }); emit(epoch, [status(epoch, { server: 'old' })]); queueEvent('server-telemetry', { entries: [telemetry('game_start', 'old')] }); flushFrame(); - queueEvent('server-status', { delta: true, entries: [status(epoch, { server: 'discarded' })] }); + queueEvent('server-status', { + delta: true, + entries: [status(epoch, { server: 'discarded' })] + }); queueEvent('server-status', { entries: [status(epoch, { server: 'two' })] }); queueEvent('server-status', { delta: true, entries: [status(epoch, { server: 'three' })] }); queueEvent('server-telemetry', { entries: [telemetry('game_end', 'discarded')] }); - queueEvent('server-telemetry', { snapshot: true, entries: [telemetry('heartbeat', 'two')] }); + queueEvent('server-telemetry', { + snapshot: true, + entries: [telemetry('heartbeat', 'two')] + }); queueEvent('server-telemetry', { entries: [telemetry('game_end', 'three')] }); queueEvent('server-telemetry', { entries: [telemetry('heartbeat', 'three')] }); flushFrame(); @@ -181,11 +292,23 @@ describe('animation-frame stream batching', () => { it('retains event arrival times so buffered status cannot overwrite a newer HTTP snapshot', async () => { const { fleetServers, refreshFleet, statusByServer, statusReceivedAt } = await setup(); await vi.advanceTimersByTimeAsync(1000); - queueEvent('server-status', { delta: true, serverNow: epoch + 1000, entries: [status(epoch + 1000, { players: 1 })] }); + queueEvent('server-status', { + delta: true, + serverNow: epoch + 1000, + entries: [status(epoch + 1000, { players: 1 })] + }); await vi.advanceTimersByTimeAsync(1000); - snapshot = { ...snapshot, serverNow: epoch + 2000, servers: [server(epoch + 2000, { last_players: 4 })] }; + snapshot = { + ...snapshot, + serverNow: epoch + 2000, + servers: [server(epoch + 2000, { last_players: 4 })] + }; await refreshFleet(); - queueEvent('server-status', { delta: true, serverNow: epoch + 2000, entries: [status(epoch + 2000, { server: 'two' })] }); + queueEvent('server-status', { + delta: true, + serverNow: epoch + 2000, + entries: [status(epoch + 2000, { server: 'two' })] + }); flushFrame(); expect(get(statusByServer).get('t5_zombie_theater_1')?.receivedAt).toBe(epoch + 1000); expect(get(statusReceivedAt)).toBe(epoch + 2000); @@ -212,7 +335,9 @@ describe('animation-frame stream batching', () => { const oldSource = source; const close = vi.spyOn(oldSource, 'close'); queueEvent('server-status', { entries: [status(epoch)] }); - queueEvent('server-telemetry', { entries: [{ ...gameEvent('game_start'), receivedAt: new Date(epoch).toISOString() }] }); + queueEvent('server-telemetry', { + entries: [{ ...gameEvent('game_start'), receivedAt: new Date(epoch).toISOString() }] + }); const callback = [...frames.values()][0]; const oldStop = stop!; oldStop(); @@ -224,7 +349,11 @@ describe('animation-frame stream batching', () => { expect(get(telemetryByServer).size).toBe(0); stop = startServerStream(); oldStop(); - oldSource.dispatchEvent(new MessageEvent('server-status', { data: JSON.stringify({ entries: [status(epoch)] }) })); + oldSource.dispatchEvent( + new MessageEvent('server-status', { + data: JSON.stringify({ entries: [status(epoch)] }) + }) + ); queueEvent('server-status', { delta: true, entries: [status(epoch, { server: 'new' })] }); flushFrame(); expect([...get(statusByServer).keys()]).toEqual(['new']); @@ -238,7 +367,12 @@ describe('Fleet report clock', () => { await vi.advanceTimersByTimeAsync(2000); emit(epoch + 2000, [status(epoch + 2000, { players: 3 })]); const integrations = { rcon: true, rconTimeoutMs: 3000, defaultRconHost: '' }; - hydrateFleet({ serverNow: epoch + 1000, servers: [server(epoch + 1000), server(epoch, { name: 'two' })], operations: [], integrations }); + hydrateFleet({ + serverNow: epoch + 1000, + servers: [server(epoch + 1000), server(epoch, { name: 'two' })], + operations: [], + integrations + }); expect(get(fleetServers)).toHaveLength(2); expect(get(fleetServers)[0].last_players).toBe(3); hydrateFleet({ serverNow: epoch, servers: [], operations: [], integrations }); @@ -249,12 +383,20 @@ describe('Fleet report clock', () => { const { fleetServers, statusByServer } = await setup(); emit(epoch, [status(epoch), status(epoch, { server: 'two' })]); await vi.advanceTimersByTimeAsync(1000); - source.dispatchEvent(new MessageEvent('server-status', { data: JSON.stringify({ - delta: true, serverNow: epoch + 1000, entries: [status(epoch + 1000, { players: 2 })] - }) })); + source.dispatchEvent( + new MessageEvent('server-status', { + data: JSON.stringify({ + delta: true, + serverNow: epoch + 1000, + entries: [status(epoch + 1000, { players: 2 })] + }) + }) + ); flushFrame(); expect(get(statusByServer).size).toBe(2); - expect(get(fleetServers).find(server => server.name === 't5_zombie_theater_1')?.last_players).toBe(2); + expect( + get(fleetServers).find((server) => server.name === 't5_zombie_theater_1')?.last_players + ).toBe(2); const changes = vi.fn(); const unsubscribe = fleetServers.subscribe(changes); changes.mockClear(); @@ -264,19 +406,45 @@ describe('Fleet report clock', () => { }); it('keeps persisted occupancy and match anchors through live reports before the next status poll', async () => { const at = (seconds: number) => new Date(epoch + seconds * 1000).toISOString(); - snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: at(0) })]; + snapshot.servers = [ + server(epoch, { container_state: 'running', container_checked_at: at(0) }) + ]; const { fleetServers } = await setup(); - const report = (event: GameEvent['event'], seconds: number, players: number, timing: ServerTiming) => { - source.dispatchEvent(new MessageEvent('server-telemetry', { data: JSON.stringify({ serverNow: epoch + seconds * 1000, - entries: [{ ...gameEvent(event, 2, players), receivedAt: at(seconds), - timing: { matchId: 1, players: {}, server: timing } }] }) })); + const report = ( + event: GameEvent['event'], + seconds: number, + players: number, + timing: ServerTiming + ) => { + source.dispatchEvent( + new MessageEvent('server-telemetry', { + data: JSON.stringify({ + serverNow: epoch + seconds * 1000, + entries: [ + { + ...gameEvent(event, 2, players), + receivedAt: at(seconds), + timing: { matchId: 1, players: {}, server: timing } + } + ] + }) + }) + ); flushFrame(); return get(fleetServers)[0]; }; - const active = { activeSince: at(1), inactiveSince: null, gameStartedAt: at(1), gameEndedAt: null }; + const active = { + activeSince: at(1), + inactiveSince: null, + gameStartedAt: at(1), + gameEndedAt: null + }; report('game_start', 1, 1, active); expect(report('heartbeat', 20, 1, active)).toMatchObject({ - active_since: at(1), inactive_since: null, game_started_at: at(1), game_ended_at: null + active_since: at(1), + inactive_since: null, + game_started_at: at(1), + game_ended_at: null }); const ended = { ...active, gameEndedAt: at(30) }; report('game_end', 30, 1, ended); @@ -284,48 +452,136 @@ describe('Fleet report clock', () => { const nextMatch = { ...active, gameStartedAt: at(32) }; report('game_start', 32, 1, nextMatch); expect(report('heartbeat', 33, 1, nextMatch)).toMatchObject({ - active_since: at(1), game_started_at: at(32), game_ended_at: null + active_since: at(1), + game_started_at: at(32), + game_ended_at: null + }); + expect( + report('heartbeat', 34, 0, { + activeSince: null, + inactiveSince: at(34), + gameStartedAt: null, + gameEndedAt: null + }) + ).toMatchObject({ + active_since: null, + inactive_since: at(34), + game_started_at: null, + game_ended_at: null + }); + expect( + report('heartbeat', 35, 1, { + activeSince: at(35), + inactiveSince: null, + gameStartedAt: null, + gameEndedAt: null + }) + ).toMatchObject({ + active_since: at(35), + inactive_since: null, + game_started_at: null, + game_ended_at: null }); - expect(report('heartbeat', 34, 0, { activeSince: null, inactiveSince: at(34), gameStartedAt: null, gameEndedAt: null })) - .toMatchObject({ active_since: null, inactive_since: at(34), game_started_at: null, game_ended_at: null }); - expect(report('heartbeat', 35, 1, { activeSince: at(35), inactiveSince: null, gameStartedAt: null, gameEndedAt: null })) - .toMatchObject({ active_since: at(35), inactive_since: null, game_started_at: null, game_ended_at: null }); }); it('restores timers from a telemetry snapshot even when the browser missed game_start', async () => { const startedAt = new Date(epoch - 19_000).toISOString(); - snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: new Date(epoch).toISOString() })]; + snapshot.servers = [ + server(epoch, { + container_state: 'running', + container_checked_at: new Date(epoch).toISOString() + }) + ]; const { fleetServers } = await setup(); - source.dispatchEvent(new MessageEvent('server-telemetry', { data: JSON.stringify({ serverNow: epoch, snapshot: true, - entries: [{ ...gameEvent('heartbeat', 2, 1), receivedAt: new Date(epoch).toISOString(), - timing: { matchId: 1, players: {}, server: { activeSince: startedAt, inactiveSince: null, - gameStartedAt: startedAt, gameEndedAt: null } } }] }) })); + source.dispatchEvent( + new MessageEvent('server-telemetry', { + data: JSON.stringify({ + serverNow: epoch, + snapshot: true, + entries: [ + { + ...gameEvent('heartbeat', 2, 1), + receivedAt: new Date(epoch).toISOString(), + timing: { + matchId: 1, + players: {}, + server: { + activeSince: startedAt, + inactiveSince: null, + gameStartedAt: startedAt, + gameEndedAt: null + } + } + } + ] + }) + }) + ); flushFrame(); - expect(get(fleetServers)[0]).toMatchObject({ active_since: startedAt, game_started_at: startedAt }); + expect(get(fleetServers)[0]).toMatchObject({ + active_since: startedAt, + game_started_at: startedAt + }); }); it('updates count and roster on the next frame and expires them together', async () => { - snapshot.servers = [server(epoch, { container_state: 'running', container_checked_at: new Date(epoch).toISOString(), last_players: 4 })]; + snapshot.servers = [ + server(epoch, { + container_state: 'running', + container_checked_at: new Date(epoch).toISOString(), + last_players: 4 + }) + ]; const { fleetServers, uiNow } = await setup(); await vi.advanceTimersByTimeAsync(1000); - source.dispatchEvent(new MessageEvent('server-telemetry', { data: JSON.stringify({ serverNow: epoch + 1000, - entries: [{ ...gameEvent('heartbeat', 15, 3), receivedAt: new Date(epoch + 1000).toISOString() }] }) })); + source.dispatchEvent( + new MessageEvent('server-telemetry', { + data: JSON.stringify({ + serverNow: epoch + 1000, + entries: [ + { + ...gameEvent('heartbeat', 15, 3), + receivedAt: new Date(epoch + 1000).toISOString() + } + ] + }) + }) + ); flushFrame(); - expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' }); + expect(get(fleetServers)[0]).toMatchObject({ + health: 'online', + last_players: 3, + status_source: 'telemetry' + }); expect(get(fleetServers)[0].roster).toHaveLength(3); - expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true); + expect( + matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow)) + ).toBe(true); await vi.advanceTimersByTimeAsync(91_000); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false); - expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false); + expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe( + false + ); }); it('applies game lifecycle events on the next frame and clears them after an empty report', async () => { const { fleetServers } = await setup(); const report = (event: string, time: number) => { - source.dispatchEvent(new MessageEvent('server-telemetry', { - data: JSON.stringify({ serverNow: time, entries: [{ server: 't5_zombie_theater_1', event, - receivedAt: new Date(time).toISOString(), players: [] }] }) - })); + source.dispatchEvent( + new MessageEvent('server-telemetry', { + data: JSON.stringify({ + serverNow: time, + entries: [ + { + server: 't5_zombie_theater_1', + event, + receivedAt: new Date(time).toISOString(), + players: [] + } + ] + }) + }) + ); flushFrame(); }; emit(epoch, [status(epoch, { players: 1 })]); @@ -343,30 +599,33 @@ describe('Fleet report clock', () => { expect(get(fleetServers)[0].game_ended_at).toBeNull(); }); - it.each([-2500, 60_000])('keeps fresh HTTP and streamed reports healthy with a browser clock offset of %i ms', async (offset) => { - vi.setSystemTime(epoch + offset); - const { fleetServers, uiNow, refreshFleet } = await setup(); - const expectHealthy = () => { - const current = get(fleetServers)[0]; - expect(freshServer(current, get(uiNow))).toBe(true); - expect(matchesFleet(current, '', 'empty', '', [], [], get(uiNow))).toBe(true); - expect(matchesFleet(current, '', 'attention', '', [], [], get(uiNow))).toBe(false); - }; - expect(get(uiNow)).toBe(epoch); - expectHealthy(); - for (let cycle = 1; cycle <= 3; cycle++) { - for (let second = 0; second < 30; second++) { - await vi.advanceTimersByTimeAsync(1000); + it.each([-2500, 60_000])( + 'keeps fresh HTTP and streamed reports healthy with a browser clock offset of %i ms', + async (offset) => { + vi.setSystemTime(epoch + offset); + const { fleetServers, uiNow, refreshFleet } = await setup(); + const expectHealthy = () => { + const current = get(fleetServers)[0]; + expect(freshServer(current, get(uiNow))).toBe(true); + expect(matchesFleet(current, '', 'empty', '', [], [], get(uiNow))).toBe(true); + expect(matchesFleet(current, '', 'attention', '', [], [], get(uiNow))).toBe(false); + }; + expect(get(uiNow)).toBe(epoch); + expectHealthy(); + for (let cycle = 1; cycle <= 3; cycle++) { + for (let second = 0; second < 30; second++) { + await vi.advanceTimersByTimeAsync(1000); + expectHealthy(); + } + const now = epoch + cycle * 30_000; + emit(now, [status(now)]); + expectHealthy(); + snapshot = { ...snapshot, serverNow: now, servers: [server(now)] }; + await refreshFleet(); expectHealthy(); } - const now = epoch + cycle * 30_000; - emit(now, [status(now)]); - expectHealthy(); - snapshot = { ...snapshot, serverNow: now, servers: [server(now)] }; - await refreshFleet(); - expectHealthy(); } - }); + ); it('updates the clock before exposing a new status after a suspended UI ticker', async () => { const { fleetServers, uiNow } = await setup(); @@ -381,7 +640,12 @@ describe('Fleet report clock', () => { it('does not rewind the clock when an older HTTP snapshot follows a newer stream update', async () => { const { fleetServers, uiNow, refreshFleet } = await setup(); let resolveSnapshot!: (response: Response) => void; - vi.mocked(fetch).mockImplementationOnce(() => new Promise(resolve => { resolveSnapshot = resolve; })); + vi.mocked(fetch).mockImplementationOnce( + () => + new Promise((resolve) => { + resolveSnapshot = resolve; + }) + ); const refreshing = refreshFleet(); await vi.advanceTimersByTimeAsync(4000); emit(epoch + 4000, [status(epoch + 4000)]); @@ -401,7 +665,9 @@ describe('Fleet report clock', () => { await vi.advanceTimersByTimeAsync(1000); expect(get(uiNow)).toBe(epoch + 91_000); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false); - expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe(true); + expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe( + true + ); }); it('applies the same RCON failure grace to HTTP snapshots and live updates', async () => { @@ -411,23 +677,39 @@ describe('Fleet report clock', () => { expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true); expect(get(fleetServers)[0].rcon_status?.last_error).toBe('timeout:no reply'); await vi.advanceTimersByTimeAsync(30_000); - emit(epoch + 30_000, [status(epoch, { health: 'stale', lastError: 'timeout:no reply', - rconStatus: { ...raw, rcon_failures: 2 } })]); + emit(epoch + 30_000, [ + status(epoch, { + health: 'stale', + lastError: 'timeout:no reply', + rconStatus: { ...raw, rcon_failures: 2 } + }) + ]); expect(get(fleetServers)[0].health).toBe('stale'); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false); await vi.advanceTimersByTimeAsync(30_000); - emit(epoch + 60_000, [status(epoch + 60_000, { - rconStatus: { ...raw, last_seen_at: new Date(epoch + 60_000).toISOString(), last_error: null, rcon_failures: 0 } - })]); + emit(epoch + 60_000, [ + status(epoch + 60_000, { + rconStatus: { + ...raw, + last_seen_at: new Date(epoch + 60_000).toISOString(), + last_error: null, + rcon_failures: 0 + } + }) + ]); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true); }); it('preserves actual RCON failures and stopped-container health', async () => { const { fleetServers, uiNow } = await setup(); await vi.advanceTimersByTimeAsync(1000); - emit(epoch + 1000, [status(epoch, { health: 'stale', lastError: 'timeout:no reply', reason: 'timeout' })]); + emit(epoch + 1000, [ + status(epoch, { health: 'stale', lastError: 'timeout:no reply', reason: 'timeout' }) + ]); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false); - expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe(true); + expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe( + true + ); emit(epoch + 1000, [status(epoch, { health: 'offline' })]); expect(get(fleetServers)[0].health).toBe('offline'); expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false); @@ -437,9 +719,11 @@ describe('Fleet report clock', () => { vi.setSystemTime(epoch - 60_000); const { startServerStream, uiNow } = await import('../../src/lib/ui/stores/servers.js'); stop = startServerStream(); - source.dispatchEvent(new MessageEvent('server-telemetry', { - data: JSON.stringify({ serverNow: epoch, entries: [], snapshot: true }) - })); + source.dispatchEvent( + new MessageEvent('server-telemetry', { + data: JSON.stringify({ serverNow: epoch, entries: [], snapshot: true }) + }) + ); flushFrame(); expect(get(uiNow)).toBe(epoch); await vi.advanceTimersByTimeAsync(1000); diff --git a/tests/unit/ui-server-detail.test.ts b/tests/unit/ui-server-detail.test.ts index f4bc7cc..1e4280d 100644 --- a/tests/unit/ui-server-detail.test.ts +++ b/tests/unit/ui-server-detail.test.ts @@ -11,20 +11,44 @@ import { uiNow } from '../../src/lib/ui/stores/servers.js'; import { gameEvent } from '../helpers/telemetry.js'; import { page } from '$app/stores'; -vi.mock('$app/state', () => ({ page: { url: new URL('http://localhost/servers/t5_zombie_theater_1') } })); -vi.mock('$app/stores', () => ({ page: writable({ params: { name: 't5_zombie_theater_1' }, url: new URL('http://localhost/servers/t5_zombie_theater_1') }) })); -vi.mock('$app/navigation', () => ({ beforeNavigate: vi.fn(), afterNavigate: vi.fn(), goto: vi.fn() })); +vi.mock('$app/state', () => ({ + page: { url: new URL('http://localhost/servers/t5_zombie_theater_1') } +})); +vi.mock('$app/stores', () => ({ + page: writable({ + params: { name: 't5_zombie_theater_1' }, + url: new URL('http://localhost/servers/t5_zombie_theater_1') + }) +})); +vi.mock('$app/navigation', () => ({ + beforeNavigate: vi.fn(), + afterNavigate: vi.fn(), + goto: vi.fn() +})); const report = (current = true): GameTelemetry => { const receivedAt = new Date(get(uiNow) - (current ? 10_000 : 300_000)).toISOString(); const joined = new Date(Date.parse(receivedAt) - 60_000).toISOString(); - return { ...gameEvent('heartbeat', 4, 1), receivedAt, timing: { matchId: 1, players: { - '12345678901234560': { serverJoinedAt: joined, matchJoinedAt: joined, roundStarted: 1 } - } } }; + return { + ...gameEvent('heartbeat', 4, 1), + receivedAt, + timing: { + matchId: 1, + players: { + '12345678901234560': { + serverJoinedAt: joined, + matchJoinedAt: joined, + roundStarted: 1 + } + } + } + }; }; it('keeps scoreboard columns focused and collapses player identifiers and timings', () => { - const body = render(PlayerScoreboard, { props: { report: report(), matchElapsed: '4m 19s' } }).body; + const body = render(PlayerScoreboard, { + props: { report: report(), matchElapsed: '4m 19s' } + }).body; expect(body).toContain('Live scoreboard'); expect(body).toContain('Round 4'); expect(body).toContain('Match 4m 19s'); @@ -45,7 +69,9 @@ it('freezes historical timing and keeps unreported scores distinct from zero', ( expect(body).toContain('Time on instance: 1m 00s'); expect(body).not.toContain('Time on instance: 6m'); const start = { ...gameEvent('game_start', 0, 1), receivedAt: report().receivedAt }; - expect(render(PlayerScoreboard, { props: { report: start } }).body).toContain('aria-label="Not yet reported"'); + expect(render(PlayerScoreboard, { props: { report: start } }).body).toContain( + 'aria-label="Not yet reported"' + ); }); it('omits the fixed instance selector while keeping promoted/live distinction', () => { @@ -56,13 +82,45 @@ it('omits the fixed instance selector while keeping promoted/live distinction', }); function pageBody(tab: string, role = 'admin') { - (page as unknown as ReturnType).set({ data: { user: { role } }, params: { name: 't5_zombie_theater_1' }, url: new URL(`http://localhost/servers/t5_zombie_theater_1?tab=${tab}&returnTo=%2F%3Ffilter%3Doccupied`) }); + (page as unknown as ReturnType).set({ + data: { user: { role } }, + params: { name: 't5_zombie_theater_1' }, + url: new URL( + `http://localhost/servers/t5_zombie_theater_1?tab=${tab}&returnTo=%2F%3Ffilter%3Doccupied` + ) + }); const telemetry = report(); - const server = { name: 't5_zombie_theater_1', game: 't5', map: 'zombie_theater', last_map: 'zombie_theater', health: 'online', last_players: 1, - last_seen_at: telemetry.receivedAt, last_error: null, held: [], telemetry, status_source: 'telemetry', host_id: 'host-1', native_instance_id: 'instance-1', - host: { name: 'ec-1', connected: true, lastHeartbeat: telemetry.receivedAt }, replySnippet: 'legacy private reply 192.0.2.123:28960', - rcon_status: { last_seen_at: null, last_error: 'timeout: no reply' }, game_started_at: null, game_ended_at: null, active_since: null, inactive_since: null }; - const data = { serverSnapshot: { servers: [server], operations: [], serverNow: get(uiNow), integrations: {}, reservedNames: [] } } as unknown as ComponentProps['data']; + const server = { + name: 't5_zombie_theater_1', + game: 't5', + map: 'zombie_theater', + last_map: 'zombie_theater', + health: 'online', + last_players: 1, + last_seen_at: telemetry.receivedAt, + last_error: null, + held: [], + telemetry, + status_source: 'telemetry', + host_id: 'host-1', + native_instance_id: 'instance-1', + host: { name: 'ec-1', connected: true, lastHeartbeat: telemetry.receivedAt }, + replySnippet: 'legacy private reply 192.0.2.123:28960', + rcon_status: { last_seen_at: null, last_error: 'timeout: no reply' }, + game_started_at: null, + game_ended_at: null, + active_since: null, + inactive_since: null + }; + const data = { + serverSnapshot: { + servers: [server], + operations: [], + serverNow: get(uiNow), + integrations: {}, + reservedNames: [] + } + } as unknown as ComponentProps['data']; return render(ServerPage, { props: { data } }).body; } it('keeps match content out of the shared header and preserves the fleet return filter', () => { diff --git a/tests/unit/ui-server-health.test.ts b/tests/unit/ui-server-health.test.ts index 0dc287b..bfca849 100644 --- a/tests/unit/ui-server-health.test.ts +++ b/tests/unit/ui-server-health.test.ts @@ -5,38 +5,84 @@ import { serverHealth } from '../../src/lib/ui/server-health.js'; const now = Date.parse('2026-10-07T12:00:00Z'); const iso = (offset = 0) => new Date(now + offset).toISOString(); const server: FleetServer = { - name: 't5_zombie_theater_1', game: 't5', map: 'zombie_theater', last_map: 'zombie_theater', - last_gametype: 'zclassic', last_players: 4, last_seen_at: iso(-10_000), last_error: null, - container_state: 'running', container_checked_at: iso(-5_000), last_checked_at: iso(-5_000), - roster: [], health: 'online', held: [], active_since: iso(-120_000), inactive_since: null, - game_started_at: iso(-120_000), game_ended_at: null, host_id: 'host-1', - host: { id: 'host-1', name: 'Dallas 1', connected: true, connectionState: 'online', lastHeartbeat: iso(-1_000) } + name: 't5_zombie_theater_1', + game: 't5', + map: 'zombie_theater', + last_map: 'zombie_theater', + last_gametype: 'zclassic', + last_players: 4, + last_seen_at: iso(-10_000), + last_error: null, + container_state: 'running', + container_checked_at: iso(-5_000), + last_checked_at: iso(-5_000), + roster: [], + health: 'online', + held: [], + active_since: iso(-120_000), + inactive_since: null, + game_started_at: iso(-120_000), + game_ended_at: null, + host_id: 'host-1', + host: { + id: 'host-1', + name: 'Dallas 1', + connected: true, + connectionState: 'online', + lastHeartbeat: iso(-1_000) + } }; describe('server health presentation', () => { it('uses current game evidence for Online and expires that evidence with the UI clock', () => { - expect(serverHealth(server, now)).toMatchObject({ state: 'online', since: server.last_seen_at }); - expect(serverHealth({ ...server, host: undefined, host_id: undefined }, now + 91_000)).toMatchObject({ - state: 'not-responding', actionHref: `/servers/${server.name}?tab=diagnostics`, since: server.last_seen_at + expect(serverHealth(server, now)).toMatchObject({ + state: 'online', + since: server.last_seen_at + }); + expect( + serverHealth({ ...server, host: undefined, host_id: undefined }, now + 91_000) + ).toMatchObject({ + state: 'not-responding', + actionHref: `/servers/${server.name}?tab=diagnostics`, + since: server.last_seen_at }); }); it('identifies host loss without claiming the game stopped, including stale heartbeat snapshots', () => { - for (const host of [{ ...server.host!, connected: false, connectionState: 'offline' as const }, - { ...server.host!, lastHeartbeat: iso(-91_000) }]) { - expect(serverHealth({ ...server, host, health: 'offline', container_state: 'stopped' }, now)).toEqual({ - state: 'host-offline', label: 'Host offline', reason: "Dallas 1's host agent is unavailable. The game may still be running.", - shortReason: 'Dallas 1 unavailable; game may still run.', actionLabel: 'Check host', actionHref: '/hosts/host-1', since: host.lastHeartbeat + for (const host of [ + { ...server.host!, connected: false, connectionState: 'offline' as const }, + { ...server.host!, lastHeartbeat: iso(-91_000) } + ]) { + expect( + serverHealth( + { ...server, host, health: 'offline', container_state: 'stopped' }, + now + ) + ).toEqual({ + state: 'host-offline', + label: 'Host offline', + reason: "Dallas 1's host agent is unavailable. The game may still be running.", + shortReason: 'Dallas 1 unavailable; game may still run.', + actionLabel: 'Check host', + actionHref: '/hosts/host-1', + since: host.lastHeartbeat }); } }); - it.each(['stopped', 'exited', 'created', 'dead'])('only offers Start after a current %s container observation', container_state => { - expect(serverHealth({ ...server, health: 'offline', container_state }, now)).toMatchObject({ - state: 'stopped', since: server.container_checked_at, actionLabel: 'Start server', - actionHref: `/servers/${server.name}?action=start` - }); - }); + it.each(['stopped', 'exited', 'created', 'dead'])( + 'only offers Start after a current %s container observation', + (container_state) => { + expect( + serverHealth({ ...server, health: 'offline', container_state }, now) + ).toMatchObject({ + state: 'stopped', + since: server.container_checked_at, + actionLabel: 'Start server', + actionHref: `/servers/${server.name}?action=start` + }); + } + ); it.each([ { container_checked_at: iso(-91_000), last_checked_at: iso() }, @@ -48,44 +94,120 @@ describe('server health presentation', () => { { last_seen_at: iso(-1_000) }, { last_seen_at: iso(-5_000) }, { host: null } - ])('does not treat outdated or conflicting stop evidence as a safe Start action: %j', change => { - const result = serverHealth({ ...server, health: 'offline', container_state: 'stopped', ...change }, now); - expect(result.state).toBe('unknown'); - expect(result.actionHref).not.toContain('action=start'); - }); + ])( + 'does not treat outdated or conflicting stop evidence as a safe Start action: %j', + (change) => { + const result = serverHealth( + { ...server, health: 'offline', container_state: 'stopped', ...change }, + now + ); + expect(result.state).toBe('unknown'); + expect(result.actionHref).not.toContain('action=start'); + } + ); it('separates a running container with failed game checks from a stopped or missing container', () => { - expect(serverHealth({ ...server, health: 'offline', last_seen_at: iso(-300_000), last_error: 'timeout:Internal endpoint 10.0.0.1 timed out' }, now)).toEqual({ - state: 'not-responding', label: 'Not responding', reason: 'The game did not answer the latest status check.', - shortReason: 'No reply to the status check.', actionLabel: 'Test connection', actionHref: `/servers/${server.name}?tab=diagnostics`, since: iso(-300_000) + expect( + serverHealth( + { + ...server, + health: 'offline', + last_seen_at: iso(-300_000), + last_error: 'timeout:Internal endpoint 10.0.0.1 timed out' + }, + now + ) + ).toEqual({ + state: 'not-responding', + label: 'Not responding', + reason: 'The game did not answer the latest status check.', + shortReason: 'No reply to the status check.', + actionLabel: 'Test connection', + actionHref: `/servers/${server.name}?tab=diagnostics`, + since: iso(-300_000) }); - expect(serverHealth({ ...server, health: 'offline', container_state: 'missing', last_seen_at: null }, now).state).toBe('unknown'); + expect( + serverHealth( + { ...server, health: 'offline', container_state: 'missing', last_seen_at: null }, + now + ).state + ).toBe('unknown'); }); it('uses generic copy for unrecognized failures without exposing internal paths or prefixes', () => { - const result = serverHealth({ ...server, health: 'offline', last_error: 'legacy-failure:/private/server/secret', container_checked_at: null, last_seen_at: null }, now); - expect(result).toMatchObject({ state: 'unknown', since: null, reason: 'There is no current evidence of the game or container state.' }); + const result = serverHealth( + { + ...server, + health: 'offline', + last_error: 'legacy-failure:/private/server/secret', + container_checked_at: null, + last_seen_at: null + }, + now + ); + expect(result).toMatchObject({ + state: 'unknown', + since: null, + reason: 'There is no current evidence of the game or container state.' + }); expect(result.reason).not.toContain('secret'); }); it('retains explicit startup phases without mistaking old stopped observations for current state', () => { for (const health of ['starting', 'provisioning'] as const) { - expect(serverHealth({ ...server, health, container_state: 'stopped', telemetry_after: iso(-2_000) }, now)).toMatchObject({ - state: health, actionLabel: 'View progress', since: iso(-2_000) + expect( + serverHealth( + { ...server, health, container_state: 'stopped', telemetry_after: iso(-2_000) }, + now + ) + ).toMatchObject({ + state: health, + actionLabel: 'View progress', + since: iso(-2_000) }); } }); it('rejects old-process and stale-container evidence for Online', () => { - expect(serverHealth({ ...server, telemetry_after: iso(-2_000) }, now).state).toBe('unknown'); - expect(serverHealth({ ...server, container_checked_at: iso(-91_000) }, now).state).not.toBe('online'); + expect(serverHealth({ ...server, telemetry_after: iso(-2_000) }, now).state).toBe( + 'unknown' + ); + expect(serverHealth({ ...server, container_checked_at: iso(-91_000) }, now).state).not.toBe( + 'online' + ); }); it('uses only valid evidence times and encodes navigation identities', () => { - expect(serverHealth({ ...server, name: 'server / 1', health: 'offline', container_checked_at: null, last_seen_at: iso(30_000), host: null, host_id: 'host / 1' }, now)).toMatchObject({ - state: 'unknown', actionHref: '/hosts/host%20%2F%201', since: null + expect( + serverHealth( + { + ...server, + name: 'server / 1', + health: 'offline', + container_checked_at: null, + last_seen_at: iso(30_000), + host: null, + host_id: 'host / 1' + }, + now + ) + ).toMatchObject({ + state: 'unknown', + actionHref: '/hosts/host%20%2F%201', + since: null }); - expect(serverHealth({ ...server, name: 'server / 1', health: 'offline', container_state: 'unknown', host: undefined, host_id: undefined }, now).actionHref) - .toBe('/servers/server%20%2F%201?tab=diagnostics'); + expect( + serverHealth( + { + ...server, + name: 'server / 1', + health: 'offline', + container_state: 'unknown', + host: undefined, + host_id: undefined + }, + now + ).actionHref + ).toBe('/servers/server%20%2F%201?tab=diagnostics'); }); }); diff --git a/tests/unit/ui-server-launch.test.ts b/tests/unit/ui-server-launch.test.ts index f6eaa44..f8f6153 100644 --- a/tests/unit/ui-server-launch.test.ts +++ b/tests/unit/ui-server-launch.test.ts @@ -1,21 +1,48 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { LaunchPreview } from '../../src/lib/deployment-plan.js'; -import { createServerLaunchController, type ServerLaunchInput, type ServerLaunchState } from '../../src/lib/ui/server-launch.js'; +import { + createServerLaunchController, + type ServerLaunchInput, + type ServerLaunchState +} from '../../src/lib/ui/server-launch.js'; -const selection: ServerLaunchInput = { game: 't5', mapSlug: 'kino', libraryId: 'library-1', image: 'runtime:latest' }; +const selection: ServerLaunchInput = { + game: 't5', + mapSlug: 'kino', + libraryId: 'library-1', + image: 'runtime:latest' +}; const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status }); const preview = (id = 'preview-1', expiresIn = 60_000): LaunchPreview => ({ - id, expiresAt: new Date(Date.now() + expiresIn).toISOString(), name: 't5_zombie_theater_1', - port: null, hasSavedKey: true, blueprintName: null, checks: [], ready: true, - recipe: { game: 't5', mapSlug: 'kino', mapName: 'Kino Der Toten', expectedMap: 'zombie_theater', - capturedAt: new Date().toISOString(), image: 'runtime:latest', configFile: 'server.cfg', - configSource: 'T5 Zombies', files: [], fingerprint: 'recipe-1' }, + id, + expiresAt: new Date(Date.now() + expiresIn).toISOString(), + name: 't5_zombie_theater_1', + port: null, + hasSavedKey: true, + blueprintName: null, + checks: [], + ready: true, + recipe: { + game: 't5', + mapSlug: 'kino', + mapName: 'Kino Der Toten', + expectedMap: 'zombie_theater', + capturedAt: new Date().toISOString(), + image: 'runtime:latest', + configFile: 'server.cfg', + configSource: 'T5 Zombies', + files: [], + fingerprint: 'recipe-1' + } }); function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; - const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); return { promise, resolve, reject }; } @@ -29,10 +56,16 @@ beforeEach(() => { vi.setSystemTime(new Date('2026-10-07T18:00:00Z')); state = { preview: null, checking: false, error: '' }; fetchPreview = vi.fn(); - changes = vi.fn((next: ServerLaunchState) => { state = next; }); + changes = vi.fn((next: ServerLaunchState) => { + state = next; + }); controller = createServerLaunchController(changes, { fetch: fetchPreview }); }); -afterEach(() => { controller.dispose(); vi.useRealTimers(); vi.restoreAllMocks(); }); +afterEach(() => { + controller.dispose(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); describe('automatic server launch checks', () => { it('debounces rapid selections and sends only the final game, map, and native overrides', async () => { @@ -46,7 +79,11 @@ describe('automatic server launch checks', () => { await vi.advanceTimersByTimeAsync(1); expect(fetchPreview).toHaveBeenCalledTimes(1); expect(fetchPreview.mock.calls[0][0]).toBe('/api/provisioning/preview'); - expect(JSON.parse(fetchPreview.mock.calls[0][1]!.body as string)).toEqual({ ...selection, mapSlug: 'five', publicPort: 30100 }); + expect(JSON.parse(fetchPreview.mock.calls[0][1]!.body as string)).toEqual({ + ...selection, + mapSlug: 'five', + publicPort: 30100 + }); expect(state.preview?.id).toBe('preview-1'); expect(state.checking).toBe(false); }); @@ -78,26 +115,33 @@ describe('automatic server launch checks', () => { expect(state.preview?.id).toBe('preview-1'); }); - it.each(['success', 'error'] as const)('aborts changed requests and ignores stale %s responses', async (result) => { - const first = deferred(); - fetchPreview.mockReturnValueOnce(first.promise).mockResolvedValueOnce(response(preview('new-preview'))); - controller.select(selection); - await vi.advanceTimersByTimeAsync(200); - const signal = fetchPreview.mock.calls[0][1]!.signal!; - controller.select({ ...selection, mapSlug: 'five' }); - expect(signal.aborted).toBe(true); - await vi.advanceTimersByTimeAsync(200); - if (result === 'success') first.resolve(response(preview('old-preview'))); - else first.reject(new Error('Old network error')); - await vi.advanceTimersByTimeAsync(0); - expect(state.preview?.id).toBe('new-preview'); - expect(state.error).toBe(''); - expect(state.checking).toBe(false); - }); + it.each(['success', 'error'] as const)( + 'aborts changed requests and ignores stale %s responses', + async (result) => { + const first = deferred(); + fetchPreview + .mockReturnValueOnce(first.promise) + .mockResolvedValueOnce(response(preview('new-preview'))); + controller.select(selection); + await vi.advanceTimersByTimeAsync(200); + const signal = fetchPreview.mock.calls[0][1]!.signal!; + controller.select({ ...selection, mapSlug: 'five' }); + expect(signal.aborted).toBe(true); + await vi.advanceTimersByTimeAsync(200); + if (result === 'success') first.resolve(response(preview('old-preview'))); + else first.reject(new Error('Old network error')); + await vi.advanceTimersByTimeAsync(0); + expect(state.preview?.id).toBe('new-preview'); + expect(state.error).toBe(''); + expect(state.checking).toBe(false); + } + ); it('ignores a response from a dismissed session after reopening the same selection', async () => { const first = deferred(); - fetchPreview.mockReturnValueOnce(first.promise).mockResolvedValueOnce(response(preview('reopened-preview'))); + fetchPreview + .mockReturnValueOnce(first.promise) + .mockResolvedValueOnce(response(preview('reopened-preview'))); controller.select(selection); await vi.advanceTimersByTimeAsync(200); controller.clear(); @@ -120,7 +164,9 @@ describe('automatic server launch checks', () => { it('refreshes immediately and never retains an old preview if the refresh fails', async () => { const refresh = deferred(); - fetchPreview.mockResolvedValueOnce(response(preview())).mockReturnValueOnce(refresh.promise); + fetchPreview + .mockResolvedValueOnce(response(preview())) + .mockReturnValueOnce(refresh.promise); controller.select(selection); await vi.advanceTimersByTimeAsync(200); controller.refresh(); @@ -128,7 +174,11 @@ describe('automatic server launch checks', () => { expect(state).toEqual({ preview: null, checking: true, error: '' }); refresh.resolve(response({ error: 'No runtime is available.' }, 409)); await vi.advanceTimersByTimeAsync(0); - expect(state).toEqual({ preview: null, checking: false, error: 'No runtime is available.' }); + expect(state).toEqual({ + preview: null, + checking: false, + error: 'No runtime is available.' + }); fetchPreview.mockResolvedValueOnce(response(preview('retry-preview'))); controller.refresh(); expect(state.error).toBe(''); @@ -138,7 +188,9 @@ describe('automatic server launch checks', () => { it('automatically renews an expired preview and removes it while checking', async () => { const renewal = deferred(); - fetchPreview.mockResolvedValueOnce(response(preview('initial', 1000))).mockReturnValueOnce(renewal.promise); + fetchPreview + .mockResolvedValueOnce(response(preview('initial', 1000))) + .mockReturnValueOnce(renewal.promise); controller.select(selection); await vi.advanceTimersByTimeAsync(999); expect(state.preview?.id).toBe('initial'); @@ -155,7 +207,11 @@ describe('automatic server launch checks', () => { controller.select(selection); await vi.advanceTimersByTimeAsync(2000); expect(fetchPreview).toHaveBeenCalledTimes(1); - expect(state).toEqual({ preview: null, checking: false, error: 'The launch check expired. Try again.' }); + expect(state).toEqual({ + preview: null, + checking: false, + error: 'The launch check expired. Try again.' + }); }); it('freezes a preview during submission and resumes checks after a failed launch', async () => { diff --git a/tests/unit/ui-settings.test.ts b/tests/unit/ui-settings.test.ts index 71374b1..0812d30 100644 --- a/tests/unit/ui-settings.test.ts +++ b/tests/unit/ui-settings.test.ts @@ -4,7 +4,8 @@ import SettingsEditor from '../../src/lib/ui/components/SettingsEditor.svelte'; import { defaultFleetSettings } from '../../src/lib/fleet-settings.js'; vi.mock('$app/navigation', () => ({ beforeNavigate: vi.fn(), goto: vi.fn() })); -const html = (initial = defaultFleetSettings) => render(SettingsEditor, { props: { initial } }).body.replaceAll(//g, ''); +const html = (initial = defaultFleetSettings) => + render(SettingsEditor, { props: { initial } }).body.replaceAll(//g, ''); describe('grouped Settings editor', () => { it('shows one section with meaningful defaults, and keeps every category reachable from the rail and mobile selector', () => { @@ -27,7 +28,9 @@ describe('grouped Settings editor', () => { expect(body).not.toContain('class="changed-marker"'); expect(body).toMatch(/]*id="setting-preparationConcurrency"[^>]*value="7"/); expect(body).toMatch(/]*disabled[^>]*>[\s\S]*?]*>Discard<\/span>/); - expect(body).toMatch(/]*type="submit"[^>]*form="[^"]+"[^>]*disabled[^>]*>Save changes<\/button>/); + expect(body).toMatch( + /]*type="submit"[^>]*form="[^"]+"[^>]*disabled[^>]*>Save changes<\/button>/ + ); }); it('keeps zero-mode controls labelled and cleanup tools outside the one settings form', () => { @@ -35,7 +38,9 @@ describe('grouped Settings editor', () => { expect(body).toMatch(/]*for="setting-restartConcurrencyPerHost-zero"/); expect(body).toMatch(/]*id="setting-restartConcurrencyPerHost-zero"[^>]*checked/); expect(body.match(/')).toBeLessThan(body.indexOf('Cleanup preview and protected records')); + expect(body.indexOf('')).toBeLessThan( + body.indexOf('Cleanup preview and protected records') + ); expect(body).toContain('Expired data cannot be restored by increasing retention.'); expect(body).toContain('Player safety checks still apply.'); }); diff --git a/tests/unit/ui-status-accessibility.test.ts b/tests/unit/ui-status-accessibility.test.ts index b5c8204..3562c15 100644 --- a/tests/unit/ui-status-accessibility.test.ts +++ b/tests/unit/ui-status-accessibility.test.ts @@ -9,7 +9,9 @@ import { statusPresentation } from '../../src/lib/ui/status.js'; import { uiNow } from '../../src/lib/ui/stores/servers.js'; import { gameEvent } from '../helpers/telemetry.js'; -vi.mock('$app/stores', () => ({ page: writable({ url: new URL('http://localhost/?filter=occupied') }) })); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/?filter=occupied') }) +})); describe('status accessibility', () => { it.each([ @@ -19,14 +21,17 @@ describe('status accessibility', () => { ['stale', 'Not responding', 'warning', 'triangle'], ['offline', 'Offline', 'danger', 'square'], ['queued', 'Queued', 'neutral', 'hollow'] - ])('pairs %s with a consistent word and non-color cue without creating a live region', (state, label, tone, shape) => { - expect(statusPresentation(state)).toEqual({ label, tone, shape }); - const body = render(StatusBadge, { props: { state } }).body; - expect(body).toContain(label); - expect(body).toContain(`shape-${shape}`); - expect(body).toContain(`v-${tone}`); - expect(body).not.toMatch(/role="status"|aria-live=/); - }); + ])( + 'pairs %s with a consistent word and non-color cue without creating a live region', + (state, label, tone, shape) => { + expect(statusPresentation(state)).toEqual({ label, tone, shape }); + const body = render(StatusBadge, { props: { state } }).body; + expect(body).toContain(label); + expect(body).toContain(`shape-${shape}`); + expect(body).toContain(`v-${tone}`); + expect(body).not.toMatch(/role="status"|aria-live=/); + } + ); it('provides a static accessible name only when an isolated pip needs one', () => { const decorative = render(StatusPip, { props: { health: 'online' } }).body; @@ -40,35 +45,47 @@ describe('status accessibility', () => { it('names the card link from its title and keeps its clock and roster out of live regions', () => { const receivedAt = new Date(get(uiNow) - 10_000).toISOString(); - const body = render(ServerCard, { props: { - name: 't5_zombie_theater_1', health: 'online', players: 4, - telemetry: { ...gameEvent('heartbeat', 30, 4), receivedAt }, - activeSince: receivedAt, gameStartedAt: receivedAt, - roster: [{ num: 0, name: 'Player One', score: 30000, ping: 40, guid: '1' }] - } }).body; + const body = render(ServerCard, { + props: { + name: 't5_zombie_theater_1', + health: 'online', + players: 4, + telemetry: { ...gameEvent('heartbeat', 30, 4), receivedAt }, + activeSince: receivedAt, + gameStartedAt: receivedAt, + roster: [{ num: 0, name: 'Player One', score: 30000, ping: 40, guid: '1' }] + } + }).body; const card = body.match(/]*>/)?.[0] ?? ''; const titleId = card.match(/aria-labelledby="([^"]+)"/)?.[1]?.split(' ')[0]; expect(titleId).toBeTruthy(); expect(body).toContain(`id="${titleId}"`); expect(card).not.toContain('aria-label='); expect(body).toContain('Online'); - expect(body).toMatch(/]*>Round<\/dt>]*>30<\/dd>/); + expect(body).toMatch(/]*>Round<\/dt>\s*]*>30<\/dd>/); expect(body).toContain('Player One'); expect(body).not.toMatch(/aria-live=|role="status"/); }); it('decodes persisted errors and keeps unknown technical errors in diagnostics', () => { const props = { name: 't5_zombie_theater_1', health: 'offline' as const }; - const known = render(ServerCard, { props: { ...props, lastError: 'timeout:Agent received no reply.' } }).body; + const known = render(ServerCard, { + props: { ...props, lastError: 'timeout:Agent received no reply.' } + }).body; expect(known).toContain('Agent received no reply.'); expect(known).not.toContain('timeout:'); - const unknown = render(ServerCard, { props: { ...props, lastError: 'unknown-error:raw diagnostic data' } }).body; + const unknown = render(ServerCard, { + props: { ...props, lastError: 'unknown-error:raw diagnostic data' } + }).body; expect(unknown).toContain('Open diagnostics for details.'); expect(unknown).not.toContain('raw diagnostic data'); }); it('uses a page-level heading by default and allows nested empty states without live announcements', () => { - const props = { title: 'No campaigns', description: 'Campaigns apply file changes to your servers.' }; + const props = { + title: 'No campaigns', + description: 'Campaigns apply file changes to your servers.' + }; const body = render(EmptyState, { props }).body; expect(body.replaceAll(//g, '')).toMatch(/]*>No campaigns<\/h2>/); expect(body).not.toMatch(/role="status"|aria-live=/); diff --git a/tests/unit/ui-view-transition.test.ts b/tests/unit/ui-view-transition.test.ts index da884b0..4d62dab 100644 --- a/tests/unit/ui-view-transition.test.ts +++ b/tests/unit/ui-view-transition.test.ts @@ -5,18 +5,29 @@ afterEach(() => vi.unstubAllGlobals()); it('allows navigation to continue when a transition is skipped or superseded', async () => { let finish: () => void = () => {}; - const finished = new Promise(resolve => { finish = resolve; }); + const finished = new Promise((resolve) => { + finish = resolve; + }); const complete = Promise.reject(new Error('navigation aborted')); - vi.stubGlobal('document', { startViewTransition: (update: () => Promise) => { - void update().then(finish); - return { ready: Promise.reject(new DOMException('Skipped transition', 'AbortError')), finished }; - } }); + vi.stubGlobal('document', { + startViewTransition: (update: () => Promise) => { + void update().then(finish); + return { + ready: Promise.reject(new DOMException('Skipped transition', 'AbortError')), + finished + }; + } + }); await enableViewTransitions({ complete }); await finished; }); it('continues without animation when the browser cannot start a transition', async () => { - vi.stubGlobal('document', { startViewTransition: () => { throw new Error('Unavailable'); } }); + vi.stubGlobal('document', { + startViewTransition: () => { + throw new Error('Unavailable'); + } + }); await enableViewTransitions({ complete: Promise.resolve() }); vi.stubGlobal('document', {}); expect(enableViewTransitions({ complete: Promise.resolve() })).toBeUndefined(); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index 885052d..59fc197 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -1,17 +1,29 @@ import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent } from '../helpers/telemetry.js'; -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(), + rconStatusResult: vi.fn(), + rconPlutoniumVersion: vi.fn() +})); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); const server = 't5_zombie_theater_1'; -const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }; -const missingRcon = { ok: false as const, endpoint: { host: 'test', port: 28960 }, reason: 'timeout' as const, detail: 'No reply' }; +const healthyRcon = { + ok: true as const, + endpoint: { host: 'test', port: 28960 }, + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } +}; +const missingRcon = { + ok: false as const, + endpoint: { host: 'test', port: 28960 }, + reason: 'timeout' as const, + detail: 'No reply' +}; let fixture: Fixture; type Engine = typeof import('../../src/lib/server/watchdog/index.js'); let watchdog: Engine; @@ -21,12 +33,23 @@ let refreshHost: () => unknown; let rcon: typeof import('../../src/lib/server/integrations/rcon.js'); beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); + vi.resetModules(); + vi.resetAllMocks(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-27T12:00:00Z')); fixture = await newFixture([server]); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - for (const [key, value] of Object.entries({ RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1' })) vi.stubEnv(key, value); + vi.stubGlobal( + 'fetch', + vi.fn(async () => + Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }) + ) + ); + for (const [key, value] of Object.entries({ + RCON_PASSWORD: 'test', + DEPLOYMENT_CREATE_CONCURRENCY: '1', + DEPLOYMENT_RESTART_CONCURRENCY: '1' + })) + vi.stubEnv(key, value); database = await import('../../src/lib/server/db/index.js'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); @@ -44,24 +67,57 @@ beforeEach(async () => { vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); watchdog = await import('../../src/lib/server/watchdog/index.js'); }); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); +afterEach(async () => { + await fixture.cleanup(); + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); const incidents = () => watchdog.watchdogOverview().incidents; -const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; }; -const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[]; -function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); } -function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); } -function missing() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'missing', reason: 'Container missing' }); } +const onlyIncident = () => { + expect(incidents()).toHaveLength(1); + return incidents()[0]; +}; +const operations = () => + database + .db() + .prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid') + .all() as { id: string; action: string; state: string }[]; +function advance(ms: number) { + vi.setSystemTime(Date.now() + ms); + refreshHost(); +} +function down() { + vi.mocked(backend.nativeHealth).mockResolvedValue({ + ok: false, + state: 'exited', + reason: 'Container exited' + }); +} +function missing() { + vi.mocked(backend.nativeHealth).mockResolvedValue({ + ok: false, + state: 'missing', + reason: 'Container missing' + }); +} async function pollStatus() { - const worker = await vi.importActual('../../src/lib/server/pipeline/worker.js'); + const worker = await vi.importActual( + '../../src/lib/server/pipeline/worker.js' + ); await worker.fanoutStatus(); } async function sustainedFailure() { down(); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([ + expect.objectContaining({ action: 'restart', state: 'waiting' }) + ]); return onlyIncident(); } @@ -78,66 +134,90 @@ describe('Watchdog detection and ownership', () => { expect(operations()).toEqual([]); }); - it.each(['container', 'game'])('requests recovery from sustained %s failures collected by status polling', async kind => { - if (kind === 'container') down(); - else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); - for (let minute = 0; minute < 3; minute++) { - await pollStatus(); - await watchdog.watchdogTick(); - if (minute < 2) { - expect(onlyIncident().failures).toBe(minute + 1); - expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); + it.each(['container', 'game'])( + 'requests recovery from sustained %s failures collected by status polling', + async (kind) => { + if (kind === 'container') down(); + else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); + for (let minute = 0; minute < 3; minute++) { + await pollStatus(); + await watchdog.watchdogTick(); + if (minute < 2) { + expect(onlyIncident().failures).toBe(minute + 1); + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); + } } + expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' }); + expect(operations()).toEqual([ + expect.objectContaining({ action: 'restart', state: 'waiting' }) + ]); + expect(backend.nativeHealth).toHaveBeenCalledTimes(3); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0); } - expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' }); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]); - expect(backend.nativeHealth).toHaveBeenCalledTimes(3); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0); - }); + ); - it.each([false, true])('rebuilds a confirmed missing container once after three samples over two minutes (manual=%s)', async manual => { - missing(); - const drift = await import('../../src/lib/server/fleet/drift.js'); - const scan = vi.spyOn(drift, 'prepareDriftCampaign'); - for (let sample = 0; sample < 3; sample++) { - if (!manual) await pollStatus(); - await watchdog.watchdogTick(manual); - expect(onlyIncident().failures).toBe(sample + 1); - if (sample < 2) { - expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); + it.each([false, true])( + 'rebuilds a confirmed missing container once after three samples over two minutes (manual=%s)', + async (manual) => { + missing(); + const drift = await import('../../src/lib/server/fleet/drift.js'); + const scan = vi.spyOn(drift, 'prepareDriftCampaign'); + for (let sample = 0; sample < 3; sample++) { + if (!manual) await pollStatus(); + await watchdog.watchdogTick(manual); + expect(onlyIncident().failures).toBe(sample + 1); + if (sample < 2) { + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); + } } + const incident = onlyIncident(); + const queued = operations(); + expect(incident).toMatchObject({ kind: 'health', state: 'recovering', failures: 3 }); + expect(queued).toEqual([ + expect.objectContaining({ action: 'rebuild', state: 'waiting' }) + ]); + expect(watchdog.watchdogDetail(incident.id)?.actions).toEqual([ + expect.objectContaining({ + action: 'rebuild', + operation_id: queued[0].id, + campaign_id: null + }) + ]); + expect( + database + .db() + .prepare('SELECT kind,source_id FROM provisioning_sources WHERE operation_id=?') + .all(queued[0].id) + ).toEqual([{ kind: 'watchdog', source_id: incident.id }]); + expect(rcon.rconStatusResult).not.toHaveBeenCalled(); + expect(scan).not.toHaveBeenCalled(); + expect(backend.nativeRequest).not.toHaveBeenCalled(); + database.closeDb(); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + expect(operations()).toEqual(queued); + expect(watchdog.watchdogDetail(incident.id)?.actions).toHaveLength(1); } - const incident = onlyIncident(); - const queued = operations(); - expect(incident).toMatchObject({ kind: 'health', state: 'recovering', failures: 3 }); - expect(queued).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]); - expect(watchdog.watchdogDetail(incident.id)?.actions).toEqual([ - expect.objectContaining({ action: 'rebuild', operation_id: queued[0].id, campaign_id: null }) - ]); - expect(database.db().prepare('SELECT kind,source_id FROM provisioning_sources WHERE operation_id=?').all(queued[0].id)) - .toEqual([{ kind: 'watchdog', source_id: incident.id }]); - expect(rcon.rconStatusResult).not.toHaveBeenCalled(); - expect(scan).not.toHaveBeenCalled(); - expect(backend.nativeRequest).not.toHaveBeenCalled(); - database.closeDb(); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - expect(operations()).toEqual(queued); - expect(watchdog.watchdogDetail(incident.id)?.actions).toHaveLength(1); - }); + ); it('does not accelerate missing-container recovery through repeated manual checks', async () => { missing(); await watchdog.watchdogTick(true); for (let check = 0; check < 5; check++) await watchdog.watchdogTick(true); expect(onlyIncident().failures).toBe(1); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS - 1); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS - 1); + await watchdog.watchdogTick(true); expect(operations()).toEqual([]); - advance(1); await watchdog.watchdogTick(true); + advance(1); + await watchdog.watchdogTick(true); expect(onlyIncident().failures).toBe(3); - expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]); + expect(operations()).toEqual([ + expect.objectContaining({ action: 'rebuild', state: 'waiting' }) + ]); }); it('dispatches missing-container recovery as a rebuild without requesting an empty-player count', async () => { @@ -150,43 +230,69 @@ describe('Watchdog detection and ownership', () => { expect(operation).toMatchObject({ action: 'rebuild', state: 'waiting' }); const readFiles = vi.mocked(backend.nativeRequest).getMockImplementation()!; vi.mocked(backend.nativeRequest).mockImplementation(async (instance, kind, payload) => { - if (kind === 'instance.inspect') return { exists: false, healthy: false, state: 'missing' } as never; + if (kind === 'instance.inspect') + return { exists: false, healthy: false, state: 'missing' } as never; return readFiles(instance, kind, payload); }); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning(); - const binding = database.db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(operation.id) as { operation_id: string | null }; + const binding = database + .db() + .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') + .get(operation.id) as { operation_id: string | null }; const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); expect(binding.operation_id).toEqual(expect.any(String)); - expect(getHostOperation(binding.operation_id!)).toMatchObject({ kind: 'instance.rebuild', generation: 1, - payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } }); - expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]); + expect(getHostOperation(binding.operation_id!)).toMatchObject({ + kind: 'instance.rebuild', + generation: 1, + payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } + }); + expect( + database + .db() + .prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'") + .all() + ).toEqual([]); expect(rcon.rconGetPlayerCount).not.toHaveBeenCalled(); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); }); - it.each(['health disabled', 'maintenance paused'])('keeps missing containers observational with %s even when files have drifted', async policy => { - missing(); - const policies = await import('../../src/lib/server/maintenance/policies.js'); - policies.setMaintenancePolicy('drift', true); - if (policy === 'health disabled') policies.setMaintenancePolicy('health', false); - else policies.pauseMaintenance(true); - await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); - const drift = await import('../../src/lib/server/fleet/drift.js'); - const scan = vi.spyOn(drift, 'prepareDriftCampaign'); - for (let sample = 0; sample < 4; sample++) { + it.each(['health disabled', 'maintenance paused'])( + 'keeps missing containers observational with %s even when files have drifted', + async (policy) => { + missing(); + const policies = await import('../../src/lib/server/maintenance/policies.js'); + policies.setMaintenancePolicy('drift', true); + if (policy === 'health disabled') policies.setMaintenancePolicy('health', false); + else policies.pauseMaintenance(true); + await writeFile( + join( + fixture.t5, + server, + 'runtime/plutonium/storage/t5/raw/scripts/sp', + 'stray.gsc' + ), + 'unexpected' + ); + const drift = await import('../../src/lib/server/fleet/drift.js'); + const scan = vi.spyOn(drift, 'prepareDriftCampaign'); + for (let sample = 0; sample < 4; sample++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } + expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 4 }); + expect(operations()).toEqual([]); + expect(scan).not.toHaveBeenCalled(); + expect(backend.nativeRequest).not.toHaveBeenCalled(); + expect(rcon.rconStatusResult).not.toHaveBeenCalled(); + policies.setMaintenancePolicy('health', true); + policies.pauseMaintenance(false); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); + expect(operations()).toEqual([ + expect.objectContaining({ action: 'rebuild', state: 'waiting' }) + ]); } - expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 4 }); - expect(operations()).toEqual([]); - expect(scan).not.toHaveBeenCalled(); - expect(backend.nativeRequest).not.toHaveBeenCalled(); - expect(rcon.rconStatusResult).not.toHaveBeenCalled(); - policies.setMaintenancePolicy('health', true); policies.pauseMaintenance(false); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]); - }); + ); it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => { down(); @@ -203,7 +309,10 @@ describe('Watchdog detection and ownership', () => { advance(90_001); await pollStatus(); await watchdog.watchdogTick(); - expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() }); + expect(onlyIncident()).toMatchObject({ + failures: 1, + first_seen_at: new Date().toISOString() + }); expect(operations()).toEqual([]); }); @@ -237,36 +346,45 @@ describe('Watchdog detection and ownership', () => { advance(30_000); await pollStatus(); await watchdog.watchdogTick(); - expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() }); + expect(onlyIncident()).toMatchObject({ + failures: 1, + first_seen_at: new Date().toISOString() + }); expect(operations()).toEqual([]); expect(rcon.rconStatusResult).toHaveBeenCalledTimes(3); }); - it.each([0, 4])('uses fresh telemetry with %i players without extra Watchdog probes and waits for polled RCON when telemetry expires', async players => { - const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); - vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); - saveTelemetry({ ...gameEvent('heartbeat', 17, players, server), receivedAt: new Date().toISOString() }); - // Status polling supplements occupied rosters with pings; Watchdog adds no probes. - const supplementaryProbes = players > 0 ? 1 : 0; - await pollStatus(); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); - await watchdog.watchdogTick(); - expect(incidents()).toEqual([]); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); - advance(watchdog.WATCHDOG_SCAN_MS); - await watchdog.watchdogTick(); - expect(incidents()).toEqual([]); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); - advance(30_001); - await pollStatus(); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1); - advance(30_000); - database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run(); - await watchdog.watchdogTick(); - expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' }); - expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1); - expect(operations()).toEqual([]); - }); + it.each([0, 4])( + 'uses fresh telemetry with %i players without extra Watchdog probes and waits for polled RCON when telemetry expires', + async (players) => { + const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); + saveTelemetry({ + ...gameEvent('heartbeat', 17, players, server), + receivedAt: new Date().toISOString() + }); + // Status polling supplements occupied rosters with pings; Watchdog adds no probes. + const supplementaryProbes = players > 0 ? 1 : 0; + await pollStatus(); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); + await watchdog.watchdogTick(); + expect(incidents()).toEqual([]); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(); + expect(incidents()).toEqual([]); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes); + advance(30_001); + await pollStatus(); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1); + advance(30_000); + database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run(); + await watchdog.watchdogTick(); + expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' }); + expect(rcon.rconStatusResult).toHaveBeenCalledTimes(supplementaryProbes + 1); + expect(operations()).toEqual([]); + } + ); it('starts a new failure window for a replacement registration', async () => { down(); @@ -277,10 +395,16 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(); expect(onlyIncident().failures).toBe(2); advance(watchdog.WATCHDOG_SCAN_MS); - database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server); + database + .db() + .prepare('UPDATE servers SET created_at=? WHERE name=?') + .run(new Date().toISOString(), server); await pollStatus(); await watchdog.watchdogTick(); - expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() }); + expect(onlyIncident()).toMatchObject({ + failures: 1, + first_seen_at: new Date().toISOString() + }); expect(operations()).toEqual([]); }); @@ -304,7 +428,10 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(true); expect(scan).toHaveBeenCalledTimes(3); advance(watchdog.WATCHDOG_SCAN_MS); - database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server); + database + .db() + .prepare('UPDATE servers SET created_at=? WHERE name=?') + .run(new Date().toISOString(), server); await pollStatus(); await watchdog.watchdogTick(); expect(scan).toHaveBeenCalledTimes(4); @@ -313,7 +440,9 @@ describe('Watchdog detection and ownership', () => { it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => { const drift = await import('../../src/lib/server/fleet/drift.js'); - const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable')); + const scan = vi + .spyOn(drift, 'prepareDriftCampaign') + .mockRejectedValueOnce(new Error('Agent unavailable')); await pollStatus(); await watchdog.watchdogTick(); expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable'); @@ -344,35 +473,57 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(true); const row = onlyIncident(); expect(row).toMatchObject({ state: 'recovering', failures: 3 }); - expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]); + expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([ + expect.objectContaining({ action: 'restart', operation_state: 'waiting' }) + ]); await watchdog.watchdogTick(true); expect(operations()).toHaveLength(1); - expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]); + expect( + database + .db() + .prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'") + .all() + ).toEqual([]); }); it.each([ ['unavailable', undefined, 'host agent request timed out'], ['unknown', 'unknown', 'Agent could not inspect the container'], ['offline', 'unknown', 'Host agent is disconnected'] - ] as const)('does not recover an %s host agent state even after repeated sustained observations', async (_label, state, reason) => { - vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state, reason }); - for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' }); - expect(operations()).toEqual([]); - expect(rcon.rconStatusResult).not.toHaveBeenCalled(); - down(); - await watchdog.watchdogTick(true); - expect(onlyIncident().failures).toBe(1); - expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); - }); + ] as const)( + 'does not recover an %s host agent state even after repeated sustained observations', + async (_label, state, reason) => { + vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state, reason }); + for (let n = 0; n < 8; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } + expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' }); + expect(operations()).toEqual([]); + expect(rcon.rconStatusResult).not.toHaveBeenCalled(); + down(); + await watchdog.watchdogTick(true); + expect(onlyIncident().failures).toBe(1); + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); + } + ); it('allows a longer initial startup grace period before requesting recovery', async () => { - vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); - for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } + vi.mocked(backend.nativeHealth).mockResolvedValue({ + ok: false, + state: 'starting', + reason: 'Starting' + }); + for (let n = 0; n < 5; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } expect(operations()).toEqual([]); await watchdog.watchdogTick(true); expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); @@ -381,25 +532,37 @@ describe('Watchdog detection and ownership', () => { it('accepts fresh validated telemetry when RCON is unavailable and rechecks after it becomes stale', async () => { const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); - saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() }); + saveTelemetry({ + ...gameEvent('heartbeat', 17, 4, server), + receivedAt: new Date().toISOString() + }); await watchdog.watchdogTick(true); expect(incidents()).toEqual([]); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); advance(90_001); await watchdog.watchdogTick(true); expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 }); - saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() }); + saveTelemetry({ + ...gameEvent('heartbeat', 18, 4, server), + receivedAt: new Date().toISOString() + }); await watchdog.watchdogTick(true); expect(onlyIncident().state).toBe('resolved'); expect(operations()).toEqual([]); }); - it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => { - vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason }); - for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(onlyIncident()).toMatchObject({ state: 'open' }); - expect(operations()).toEqual([]); - }); + it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)( + 'keeps %s RCON failures observational', + async (reason) => { + vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason }); + for (let n = 0; n < 4; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } + expect(onlyIncident()).toMatchObject({ state: 'open' }); + expect(operations()).toEqual([]); + } + ); it('breaks the sustained failure window on a successful probe between watchdog samples', async () => { vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); @@ -412,66 +575,130 @@ describe('Watchdog detection and ownership', () => { await POST({ params: { name: server } } as Parameters[0]); advance(40_000); await watchdog.watchdogTick(true); - expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() }); + expect(onlyIncident()).toMatchObject({ + failures: 1, + state: 'open', + first_seen_at: new Date().toISOString() + }); expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); }); - it.each(['exited', 'missing'])('preserves intentional stop intent for %s containers after completion and database reopening', async state => { - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const stop = requestLifecycle(server, 'stop'); - expect(stop.ok).toBe(true); - if (!stop.ok) throw new Error(stop.error); - database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId); - if (state === 'missing') missing(); else down(); - database.closeDb(); - for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(incidents()).toEqual([]); - expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); - expect(backend.nativeHealth).not.toHaveBeenCalled(); - }); + it.each(['exited', 'missing'])( + 'preserves intentional stop intent for %s containers after completion and database reopening', + async (state) => { + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + const stop = requestLifecycle(server, 'stop'); + expect(stop.ok).toBe(true); + if (!stop.ok) throw new Error(stop.error); + database + .db() + .prepare( + "UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?" + ) + .run(new Date().toISOString(), stop.deploymentId); + if (state === 'missing') missing(); + else down(); + database.closeDb(); + for (let n = 0; n < 4; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } + expect(incidents()).toEqual([]); + expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); + expect(backend.nativeHealth).not.toHaveBeenCalled(); + } + ); it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => { if (owner === 'beta') { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { script, version } = createScript({ name: 'beta', content: Buffer.from('experimental'), targetRelPath: 'beta.gsc' }); - await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], waitForHealth: false, waitForMatch: false }); + const { script, version } = createScript({ + name: 'beta', + content: Buffer.from('experimental'), + targetRelPath: 'beta.gsc' + }); + await openBeta({ + scripts: [{ scriptId: script.id, scriptVersionId: version.id }], + servers: [server], + waitForHealth: false, + waitForMatch: false + }); } else { - database.db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)").run(new Date().toISOString()); - database.db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')").run(server); + database + .db() + .prepare( + "INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)" + ) + .run(new Date().toISOString()); + database + .db() + .prepare( + "INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')" + ) + .run(server); } down(); - for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } + for (let n = 0; n < 4; n++) { + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + } expect(incidents()).toEqual([]); expect(operations()).toEqual([]); expect(backend.nativeHealth).not.toHaveBeenCalled(); }); - it.each(['exited', 'missing'])('rechecks lifecycle ownership after a %s probe yields to an operator stop', async state => { - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => { - expect(requestLifecycle(server, 'stop').ok).toBe(true); - return { ok: false, state, reason: `Container ${state}` }; - }); - await watchdog.watchdogTick(true); - expect(incidents()).toEqual([]); - expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); - }); + it.each(['exited', 'missing'])( + 'rechecks lifecycle ownership after a %s probe yields to an operator stop', + async (state) => { + const { requestLifecycle } = + await import('../../src/lib/server/provisioning/requests.js'); + vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => { + expect(requestLifecycle(server, 'stop').ok).toBe(true); + return { ok: false, state, reason: `Container ${state}` }; + }); + await watchdog.watchdogTick(true); + expect(incidents()).toEqual([]); + expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); + } + ); it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])( - 'suppresses health and drift throughout %s provisioning, then resumes', async state => { + 'suppresses health and drift throughout %s provisioning, then resumes', + async (state) => { const retained = state === 'retained recovery'; const now = new Date().toISOString(); - database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`) + database + .db() + .prepare( + `INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) + VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')` + ) .run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now); - if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at) - VALUES (?, 'manual-operation', ?)`).run(server, now); - await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); + if (retained) + database + .db() + .prepare( + `INSERT INTO restart_rollout (server, operation_id, requested_at) + VALUES (?, 'manual-operation', ?)` + ) + .run(server, now); + await writeFile( + join( + fixture.t5, + server, + 'runtime/plutonium/storage/t5/raw/scripts/sp', + 'stray.gsc' + ), + 'unexpected' + ); down(); for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); @@ -482,13 +709,24 @@ describe('Watchdog detection and ownership', () => { expect(rcon.rconStatusResult).not.toHaveBeenCalled(); expect(operations()).toHaveLength(1); - database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run(); - if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server); + database + .db() + .prepare( + "UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'" + ) + .run(); + if (retained) + database + .db() + .prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?') + .run(new Date().toISOString(), server); await watchdog.watchdogTick(true); - expect(incidents()).toEqual(expect.arrayContaining([ - expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }), - expect.objectContaining({ kind: 'drift', state: 'recovering' }) - ])); + expect(incidents()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }), + expect.objectContaining({ kind: 'drift', state: 'recovering' }) + ]) + ); expect(operations()).toHaveLength(2); expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' }); } @@ -503,49 +741,69 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(true); expect(incidents()).toEqual([]); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]); + expect(operations()).toEqual([ + expect.objectContaining({ action: 'restart', state: 'waiting' }) + ]); }); }); describe('Watchdog recovery persistence', () => { - it('escalates a submitted restart that becomes missing after its five-minute recovery window', async () => { const incident = await sustainedFailure(); const restart = operations()[0]; const submittedAt = new Date().toISOString(); - database.db().prepare("UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?") + database + .db() + .prepare( + "UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?" + ) .run(submittedAt, restart.id); - database.db().prepare('INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES(?,?,?)') + database + .db() + .prepare('INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES(?,?,?)') .run(server, restart.id, submittedAt); missing(); vi.mocked(rcon.rconStatusResult).mockClear(); - advance(5 * 60_000 - 1); await watchdog.watchdogTick(true); + advance(5 * 60_000 - 1); + await watchdog.watchdogTick(true); expect(operations()).toEqual([{ ...restart, state: 'awaiting-health' }]); - advance(1); await watchdog.watchdogTick(true); + advance(1); + await watchdog.watchdogTick(true); const rebuilt = operations(); expect(rebuilt).toEqual([ - { ...restart, state: 'failed' }, expect.objectContaining({ action: 'rebuild', state: 'waiting' }) + { ...restart, state: 'failed' }, + expect.objectContaining({ action: 'rebuild', state: 'waiting' }) ]); - expect(watchdog.watchdogDetail(incident.id)?.actions.map(action => action.action)).toEqual(['restart', 'rebuild']); - expect(database.db().prepare('SELECT operation_id FROM restart_rollout WHERE server=?').get(server)) - .toEqual({ operation_id: rebuilt[1].id }); + expect( + watchdog.watchdogDetail(incident.id)?.actions.map((action) => action.action) + ).toEqual(['restart', 'rebuild']); + expect( + database + .db() + .prepare('SELECT operation_id FROM restart_rollout WHERE server=?') + .get(server) + ).toEqual({ operation_id: rebuilt[1].id }); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(operations()).toEqual(rebuilt); }); it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => { down(); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); const id = onlyIncident().id; database.closeDb(); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(onlyIncident().id).toBe(id); const queued = operations(); expect(queued).toHaveLength(1); database.closeDb(); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(operations()).toEqual(queued); expect(watchdog.watchdogDetail(id)?.actions).toHaveLength(1); }); @@ -555,73 +813,134 @@ describe('Watchdog recovery persistence', () => { await watchdog.watchdogTick(true); const id = onlyIncident().id; expect(watchdog.updateWatchdogIncident(id, 'dismiss')).toEqual({ ok: true }); - for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); } + for (let n = 0; n < 4; n++) { + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + } expect(operations()).toEqual([]); expect(onlyIncident().state).toBe('dismissed'); expect(watchdog.updateWatchdogIncident(id, 'retry')).toEqual({ ok: true }); await watchdog.watchdogTick(true); expect(operations()).toEqual([]); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(onlyIncident()).toMatchObject({ id, state: 'recovering', cycle: 2 }); - expect(watchdog.watchdogDetail(id)?.actions).toEqual([expect.objectContaining({ action: 'restart', cycle: 2 })]); + expect(watchdog.watchdogDetail(id)?.actions).toEqual([ + expect.objectContaining({ action: 'restart', cycle: 2 }) + ]); }); }); describe('Watchdog APIs', () => { - const request = (body: unknown, raw = false) => new Request('http://localhost/api/watchdog', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: raw ? String(body) : JSON.stringify(body) - }); + const request = (body: unknown, raw = false) => + new Request('http://localhost/api/watchdog', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: raw ? String(body) : JSON.stringify(body) + }); it('returns overview state and validates scan actions and pause booleans', async () => { const { GET, POST } = await import('../../src/routes/api/watchdog/+server.js'); - expect(await GET().json()).toEqual({ enabled: true, lastScanAt: null, scanError: null, incidents: [] }); + expect(await GET().json()).toEqual({ + enabled: true, + lastScanAt: null, + scanError: null, + incidents: [] + }); expect((await POST({ request: request('{', true) })).status).toBe(400); - for (const body of [null, [], {}, { action: 'restart' }, { enabled: 'false' }, { enabled: 0 }, { action: 'scan', enabled: 'false' }]) { + for (const body of [ + null, + [], + {}, + { action: 'restart' }, + { enabled: 'false' }, + { enabled: 0 }, + { action: 'scan', enabled: 'false' } + ]) { expect((await POST({ request: request(body) })).status).toBe(400); } - expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({ enabled: false }); + expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({ + enabled: false + }); database.closeDb(); expect(await GET().json()).toMatchObject({ enabled: false }); down(); const scanned = await POST({ request: request({ action: 'scan' }) }); expect(scanned.status).toBe(200); - expect(await scanned.json()).toMatchObject({ enabled: false, lastScanAt: expect.any(String), incidents: [expect.objectContaining({ kind: 'health', state: 'open' })] }); + expect(await scanned.json()).toMatchObject({ + enabled: false, + lastScanAt: expect.any(String), + incidents: [expect.objectContaining({ kind: 'health', state: 'open' })] + }); expect(operations()).toEqual([]); }); it('validates incident actions and refuses retries while provisioning is pending', async () => { const { GET, POST } = await import('../../src/routes/api/watchdog/[id]/+server.js'); expect(GET({ params: { id: 'missing' } }).status).toBe(404); - expect((await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) })).status).toBe(404); + expect( + (await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) })) + .status + ).toBe(404); for (const body of [null, {}, [], { action: 'restart' }]) { - expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe(400); + expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe( + 400 + ); } - expect((await POST({ params: { id: 'missing' }, request: request('{', true) })).status).toBe(400); + expect( + (await POST({ params: { id: 'missing' }, request: request('{', true) })).status + ).toBe(400); const row = await sustainedFailure(); expect(await GET({ params: { id: row.id } }).json()).toMatchObject({ - incident: { id: row.id, state: 'recovering' }, actions: [expect.objectContaining({ action: 'restart' })], events: expect.any(Array) + incident: { id: row.id, state: 'recovering' }, + actions: [expect.objectContaining({ action: 'restart' })], + events: expect.any(Array) }); for (const action of ['retry', 'dismiss']) { const response = await POST({ params: { id: row.id }, request: request({ action }) }); expect(response.status).toBe(409); - expect(await response.json()).toMatchObject({ error: expect.stringContaining('Provisioning is still active') }); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('Provisioning is still active') + }); } expect(operations()).toHaveLength(1); }); it('keeps legacy drift previews readable and directs mutations to Watchdog without changing files', async () => { const { GET, POST } = await import('../../src/routes/api/drift/+server.js'); - const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'); + const path = join( + fixture.t5, + server, + 'runtime/plutonium/storage/t5/raw/scripts/sp', + 'stray.gsc' + ); await writeFile(path, 'unexpected content'); - expect(await (await GET()).json()).toMatchObject({ totalDrift: 1, servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }] }); + expect(await (await GET()).json()).toMatchObject({ + totalDrift: 1, + servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }] + }); const preview = await POST({ request: request({ dryRun: true, servers: [server] }) }); expect(preview.status).toBe(200); const body = await preview.json(); - expect(body).toMatchObject({ ok: true, plan: { servers: [server], fingerprint: expect.any(String), files: [expect.objectContaining({ relPath: 'stray.gsc' })] } }); - const mutation = await POST({ request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint }) }); + expect(body).toMatchObject({ + ok: true, + plan: { + servers: [server], + fingerprint: expect.any(String), + files: [expect.objectContaining({ relPath: 'stray.gsc' })] + } + }); + const mutation = await POST({ + request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint }) + }); expect(mutation.status).toBe(410); - expect(await mutation.json()).toMatchObject({ ok: false, watchdogUrl: '/watchdog', error: expect.stringContaining('Watchdog') }); + expect(await mutation.json()).toMatchObject({ + ok: false, + watchdogUrl: '/watchdog', + error: expect.stringContaining('Watchdog') + }); expect(await readFile(path, 'utf8')).toBe('unexpected content'); expect(operations()).toEqual([]); expect(database.db().prepare('SELECT id FROM deployments').all()).toEqual([]); @@ -629,17 +948,31 @@ describe('Watchdog APIs', () => { }); describe('Watchdog drift campaigns', () => { - it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => { // Simulate legacy registry damage; administrators cannot remove a leased key. database.db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(server); - const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'); + const path = join( + fixture.t5, + server, + 'runtime/plutonium/storage/t5/raw/scripts/sp', + 'stray.gsc' + ); await writeFile(path, 'unexpected'); await watchdog.watchdogTick(true); - expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('server key are required') }); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + expect(onlyIncident()).toMatchObject({ + kind: 'drift', + state: 'attention', + summary: expect.stringContaining('server key are required') + }); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); expect(operations()).toEqual([]); expect(await readFile(path, 'utf8')).toBe('unexpected'); - expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]); + expect( + database + .db() + .prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'") + .all() + ).toEqual([]); }); }); diff --git a/tests/unit/worker-scheduling.test.ts b/tests/unit/worker-scheduling.test.ts index 371e334..a85fabe 100644 --- a/tests/unit/worker-scheduling.test.ts +++ b/tests/unit/worker-scheduling.test.ts @@ -5,19 +5,31 @@ import { STATUS_REFRESH_MS } from '../../src/lib/server-freshness.js'; vi.mock('../../src/lib/server/pipeline/sweep.js', () => ({ sweep: vi.fn() })); vi.mock('../../src/lib/server/pipeline/beta-runner.js', () => ({ betaTick: vi.fn() })); vi.mock('../../src/lib/server/maintenance/index.js', () => ({ runMaintenance: vi.fn() })); -vi.mock('../../src/lib/server/game-files/index.js', () => ({ cleanupControllerLibraries: vi.fn(async () => {}) })); -vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ processProvisioning: vi.fn(async () => {}) })); +vi.mock('../../src/lib/server/game-files/index.js', () => ({ + cleanupControllerLibraries: vi.fn(async () => {}) +})); +vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ + processProvisioning: vi.fn(async () => {}) +})); vi.mock('../../src/lib/server/retention.js', () => ({ drainRetention: vi.fn(async () => 0) })); vi.mock('../../src/lib/server/fleet/chat.js', () => ({ collectFleetChat: vi.fn(async () => {}) })); function deferred() { let resolve!: () => void; - const promise = new Promise(done => { resolve = done; }); + const promise = new Promise((done) => { + resolve = done; + }); return { promise, resolve }; } let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); }); -afterEach(async () => { await fixture.cleanup(); }); +beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + fixture = await newFixture([]); +}); +afterEach(async () => { + await fixture.cleanup(); +}); async function setup() { const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); @@ -30,40 +42,59 @@ async function setup() { const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const worker = await import('../../src/lib/server/pipeline/worker.js'); const { workerActivity } = await import('../../src/lib/server/pipeline/activity.js'); - return { ...worker, workerActivity, background, process: vi.mocked(processProvisioning), sweep, betaTick, runMaintenance }; + return { + ...worker, + workerActivity, + background, + process: vi.mocked(processProvisioning), + sweep, + betaTick, + runMaintenance + }; } describe('independent provisioning scheduling', () => { - it.each(['startup', 'scheduled'] as const)('logs a failed %s status snapshot and retries on the next interval', async phase => { - const s = await setup(); - const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js'); - const refresh = vi.spyOn(MonitoringSnapshot.prototype, 'refresh'); - const log = vi.spyOn(console, 'error').mockImplementation(() => {}); - const failure = new Error('database or disk is full'); - vi.useFakeTimers(); - try { - if (phase === 'scheduled') { - s.startWorker(); - await vi.advanceTimersByTimeAsync(0); - refresh.mockClear(); - } - refresh.mockImplementationOnce(() => { throw failure; }); - if (phase === 'startup') s.startWorker(); - await vi.advanceTimersByTimeAsync(phase === 'startup' ? 0 : STATUS_REFRESH_MS); - expect(refresh).toHaveBeenCalledTimes(1); - expect(log).toHaveBeenCalledExactlyOnceWith('[worker] status fanout error:', failure); + it.each(['startup', 'scheduled'] as const)( + 'logs a failed %s status snapshot and retries on the next interval', + async (phase) => { + const s = await setup(); + const { MonitoringSnapshot } = + await import('../../src/lib/server/fleet/monitoring-snapshot.js'); + const refresh = vi.spyOn(MonitoringSnapshot.prototype, 'refresh'); + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + const failure = new Error('database or disk is full'); + vi.useFakeTimers(); + try { + if (phase === 'scheduled') { + s.startWorker(); + await vi.advanceTimersByTimeAsync(0); + refresh.mockClear(); + } + refresh.mockImplementationOnce(() => { + throw failure; + }); + if (phase === 'startup') s.startWorker(); + await vi.advanceTimersByTimeAsync(phase === 'startup' ? 0 : STATUS_REFRESH_MS); + expect(refresh).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledExactlyOnceWith( + '[worker] status fanout error:', + failure + ); - await vi.advanceTimersByTimeAsync(STATUS_REFRESH_MS); - expect(refresh).toHaveBeenCalledTimes(2); - expect(log).toHaveBeenCalledTimes(1); - } finally { - s.stopWorker(); s.background.resolve(); - await vi.advanceTimersByTimeAsync(0); - vi.useRealTimers(); - refresh.mockRestore(); log.mockRestore(); - await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); + await vi.advanceTimersByTimeAsync(STATUS_REFRESH_MS); + expect(refresh).toHaveBeenCalledTimes(2); + expect(log).toHaveBeenCalledTimes(1); + } finally { + s.stopWorker(); + s.background.resolve(); + await vi.advanceTimersByTimeAsync(0); + vi.useRealTimers(); + refresh.mockRestore(); + log.mockRestore(); + await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); + } } - }); + ); it('collects match chat on worker ticks without overlapping a slow collection', async () => { const s = await setup(); @@ -81,7 +112,9 @@ describe('independent provisioning scheduling', () => { await vi.advanceTimersByTimeAsync(15_000); expect(collectFleetChat).toHaveBeenCalledTimes(2); } finally { - s.stopWorker(); collection.resolve(); s.background.resolve(); + s.stopWorker(); + collection.resolve(); + s.background.resolve(); await vi.advanceTimersByTimeAsync(0); vi.useRealTimers(); await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); @@ -102,7 +135,8 @@ describe('independent provisioning scheduling', () => { expect(drainRetention).toHaveBeenCalledTimes(2); expect(s.process.mock.calls.length).toBeGreaterThan(2); } finally { - s.stopWorker(); s.background.resolve(); + s.stopWorker(); + s.background.resolve(); await vi.advanceTimersByTimeAsync(0); vi.useRealTimers(); await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); @@ -113,7 +147,10 @@ describe('independent provisioning scheduling', () => { const s = await setup(); const { drainRetention } = await import('../../src/lib/server/retention.js'); const cleanup = deferred(); - vi.mocked(drainRetention).mockImplementationOnce(async () => { await cleanup.promise; return 1501; }); + vi.mocked(drainRetention).mockImplementationOnce(async () => { + await cleanup.promise; + return 1501; + }); vi.useFakeTimers(); try { s.startWorker(); @@ -123,7 +160,9 @@ describe('independent provisioning scheduling', () => { cleanup.resolve(); await vi.advanceTimersByTimeAsync(0); } finally { - s.stopWorker(); cleanup.resolve(); s.background.resolve(); + s.stopWorker(); + cleanup.resolve(); + s.background.resolve(); await vi.advanceTimersByTimeAsync(0); vi.useRealTimers(); await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); @@ -132,7 +171,8 @@ describe('independent provisioning scheduling', () => { it('runs controller library cleanup on worker ticks without overlapping a slow cleanup', async () => { const s = await setup(); - const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js'); + const { cleanupControllerLibraries } = + await import('../../src/lib/server/game-files/index.js'); const cleanup = deferred(); vi.mocked(cleanupControllerLibraries).mockImplementationOnce(() => cleanup.promise); vi.useFakeTimers(); @@ -147,7 +187,9 @@ describe('independent provisioning scheduling', () => { await vi.advanceTimersByTimeAsync(15_000); expect(cleanupControllerLibraries).toHaveBeenCalledTimes(2); } finally { - s.stopWorker(); cleanup.resolve(); s.background.resolve(); + s.stopWorker(); + cleanup.resolve(); + s.background.resolve(); await vi.advanceTimersByTimeAsync(0); vi.useRealTimers(); await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); @@ -159,7 +201,9 @@ describe('independent provisioning scheduling', () => { try { await s.kick(); expect(s.process).toHaveBeenCalledTimes(1); - expect(s.workerActivity().map(work => work.key)).toEqual(expect.arrayContaining(['campaign', 'beta', 'maintenance'])); + expect(s.workerActivity().map((work) => work.key)).toEqual( + expect.arrayContaining(['campaign', 'beta', 'maintenance']) + ); await s.kick(); expect(s.process).toHaveBeenCalledTimes(2); expect(s.sweep).toHaveBeenCalledTimes(1); @@ -184,7 +228,8 @@ describe('independent provisioning scheduling', () => { await Promise.all([first, second, receipt]); expect(s.process).toHaveBeenCalledTimes(2); } finally { - busy.resolve(); s.background.resolve(); + busy.resolve(); + s.background.resolve(); await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); } }); diff --git a/vite.launch.config.ts b/vite.launch.config.ts index 79e9314..f74c124 100644 --- a/vite.launch.config.ts +++ b/vite.launch.config.ts @@ -3,13 +3,23 @@ import { resolve } from 'node:path'; // Reuse launch composition with an independent read-only database connection. export default defineConfig({ - plugins: [{ name: 'launch-database', enforce: 'pre', resolveId(source) { - if (source.endsWith('/db/index.js')) return resolve('src/lib/server/reports/database.ts'); - } }], + plugins: [ + { + name: 'launch-database', + enforce: 'pre', + resolveId(source) { + if (source.endsWith('/db/index.js')) + return resolve('src/lib/server/reports/database.ts'); + } + } + ], build: { - ssr: true, outDir: '.svelte-kit/launch', emptyOutDir: true, + ssr: true, + outDir: '.svelte-kit/launch', + emptyOutDir: true, rollupOptions: { - input: 'src/lib/server/provisioning/launch-worker.ts', external: ['better-sqlite3'], + input: 'src/lib/server/provisioning/launch-worker.ts', + external: ['better-sqlite3'], output: { entryFileNames: 'worker.mjs' } } } diff --git a/vite.reports.config.ts b/vite.reports.config.ts index 3f730c5..353465d 100644 --- a/vite.reports.config.ts +++ b/vite.reports.config.ts @@ -4,13 +4,23 @@ import { resolve } from 'node:path'; // Build the same report implementations for Node workers. Their database is // read-only; migrations and all application writes stay in the controller. export default defineConfig({ - plugins: [{ name: 'report-database', enforce: 'pre', resolveId(source) { - if (source.endsWith('/db/index.js')) return resolve('src/lib/server/reports/database.ts'); - } }], + plugins: [ + { + name: 'report-database', + enforce: 'pre', + resolveId(source) { + if (source.endsWith('/db/index.js')) + return resolve('src/lib/server/reports/database.ts'); + } + } + ], build: { - ssr: true, outDir: '.svelte-kit/reports', emptyOutDir: true, + ssr: true, + outDir: '.svelte-kit/reports', + emptyOutDir: true, rollupOptions: { - input: 'src/lib/server/reports/worker.ts', external: ['better-sqlite3'], + input: 'src/lib/server/reports/worker.ts', + external: ['better-sqlite3'], output: { entryFileNames: 'worker.mjs' } } }