josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.prettierignore
+5
View File
@@ -0,0 +1,5 @@
# Prettier also respects .gitignore.
package-lock.json
# Generated by agent/package_release.py, including its checksum manifest.
static/agent/
.prettierrc.json
+14
View File
@@ -0,0 +1,14 @@
{
"useTabs": false,
"tabWidth": 4,
"singleQuote": true,
"trailingComma": "none",
"printWidth": 100,
"plugins": ["prettier-plugin-svelte"],
"overrides": [
{
"files": "*.svelte",
"options": { "parser": "svelte" }
}
]
}
README.md
+5 -2
View File
@@ -7,7 +7,7 @@ a small Python agent.
![Fleet overview showing sample T5 and T6 lobbies, player counts, and match rounds](docs/assets/fleet.png)
*Fleet overview with sample data.*
_Fleet overview with sample data._
## Features
@@ -58,7 +58,7 @@ doesn't need a Docker socket or direct access to your game-server files.
## Development
The controller uses SvelteKit, TypeScript, and SQLite. The host agent uses the
Python 3 standard library. For local development, install Node.js 22.12+ and Python 3:
Python 3 standard library. For local development, install Node.js 22.13+ and Python 3:
```sh
npm ci
@@ -72,8 +72,11 @@ database defaults to `./data/cherry.db`; keep it separate from production data.
`npm run dev` packages the downloadable agent before starting the server.
```sh
npm run lint
npm run check
npm test
npm run test:agent
npm run build
```
Run `npm run format` to apply the repository's Prettier formatting.
docker-compose.yml
+2 -2
View File
@@ -6,7 +6,7 @@ services:
container_name: electric-cherry
restart: unless-stopped
ports:
- "3000:3000"
- '3000:3000'
volumes:
# Preserve the existing controller database by default. Set EC_DATA_DIR
# to a dedicated persistent directory for a new controller installation.
@@ -17,7 +17,7 @@ services:
EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-}
EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false}
# Large game files are streamed with manifest size and checksum limits.
BODY_SIZE_LIMIT: "Infinity"
BODY_SIZE_LIMIT: 'Infinity'
DB_PATH: /data/cherry.db
GAME_FILES_ROOT: /data/game-files
BETA_ARCHIVE_DIR: /data/beta-archive
docs/development/README.md
+17 -2
View File
@@ -10,7 +10,7 @@ does not require a Docker socket or direct access to host filesystems.
## Run locally
Use Node.js 22.12+ (the container uses Node.js 22), npm and Python 3.
Use Node.js 22.13+ (the container uses Node.js 22), npm and Python 3.
```sh
npm ci
@@ -76,12 +76,27 @@ remove the generated fixture directory.
## Checks and builds
```sh
npm run lint
npm run check
npm test
npm run test:agent
npm run build
```
Run `npm run format` to apply Prettier formatting before committing. It uses
four-space indentation and a 100-column target for supported source, config and
documentation files, including Svelte components. Python, shell and game scripts
are outside Prettier's supported languages. Generated agent downloads and the npm
lockfile keep their generator's formatting.
`npm run lint` checks formatting without changing files, then runs the recommended
JavaScript, TypeScript and Svelte ESLint rules. Generated build output and local
runtime data are excluded. Type checking remains part of `npm run check`.
The ESLint config documents exceptions for existing Svelte navigation, list and
reactivity patterns. Individual suppressions explain intentional validation and
test fixtures. Node.js 22.13 is the minimum supported 22.x version for the lint
toolchain.
The `package.json` overrides keep SvelteKit 2 on patched `devalue` 5.x (5.9.4 or
newer) and `cookie` 0.7.x releases until its dependency ranges include those fixes.
These dependencies ship in the server bundle even though SvelteKit is a
@@ -104,7 +119,7 @@ T5/T6 server validation use the [host acceptance checklist](../guides/native-hos
Paths below are relative to the repository root.
| Path | Responsibility |
| --- | --- |
| --------------------------------------- | ----------------------------------------------------------- |
| `src/routes` | Pages and API endpoints |
| `src/lib/ui` | Shared components, stores and [UI patterns](ui-patterns.md) |
| `src/lib/server/hosts` | Enrollment, readiness and command transport |
docs/development/ui-patterns.md
+1 -1
View File
@@ -75,7 +75,7 @@ logs. Static values use text or state colors; the accent marks links and control
### Product language
| Concept | Visible label |
| --- | --- |
| -------------------------- | ------------------------------------------------------- |
| A managed game instance | Server; use the friendly map and lobby name |
| Creating a server | Create server |
| Removing a server | Remove server |
docs/guides/authentication.md
+1 -1
View File
@@ -32,7 +32,7 @@ guessable token. An unset setup token holds setup closed.
## Roles
| Capability | Viewer | Operator | Administrator |
| --- | --- | --- | --- |
| --------------------------------------------------------------------------------------------- | -------------- | ----------- | ------------- |
| Fleet, metrics, matches and operation summaries | Yes | Yes | Yes |
| Raw configs, file diffs, logs, RCON and telemetry diagnostics | No | Yes | Yes |
| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes |
docs/guides/autoscaling.md
+1 -1
View File
@@ -13,7 +13,7 @@ Choose **Add policy**, select the game and scope, and enter an explicit maximum.
Choose **Save policy** to apply changes directly. Saving checks the current fleet, policy revision, ownership conflicts, and host readiness. If saving is blocked, the form keeps your settings and shows the reason. Saved policies show lobby counts, key availability, and the current status. An active job appears once with its latest stage and a **View job** link. Use **Lobbies** for per-map spare coverage and occupancy evidence, **Activity** for decision changes grouped by job, and **Metrics** for observed capacity and provisioning measurements.
| Autoscaling | Behavior |
| --- | --- |
| ----------- | ----------------------------------------------------------------------------------------------------- |
| Off | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled. |
| On | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. |
docs/guides/beta-test-plans.md
+38 -6
View File
@@ -57,7 +57,7 @@ There are four events. Send them whether or not a beta is active.
### Event fields
| Event | Fields |
| --- | --- |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------- |
| `game_start` | `server`, `map`, `players` with each player's `name` and `guid` |
| `round_change` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
| `heartbeat` | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats |
@@ -110,8 +110,24 @@ Send the new round and each player's current stats.
"map": "zm_transit",
"round": 25,
"players": [
{ "name": "Josh", "guid": "12345678901234567", "kills": 420, "headshots": 160, "downs": 1, "revives": 3, "score": 32000 },
{ "name": "Alex", "guid": "23456789012345678", "kills": 380, "headshots": 140, "downs": 3, "revives": 1, "score": 28000 }
{
"name": "Josh",
"guid": "12345678901234567",
"kills": 420,
"headshots": 160,
"downs": 1,
"revives": 3,
"score": 32000
},
{
"name": "Alex",
"guid": "23456789012345678",
"kills": 380,
"headshots": 140,
"downs": 3,
"revives": 1,
"score": 28000
}
]
}
```
@@ -127,8 +143,24 @@ Send the final round and each player's final stats when the game ends.
"map": "zm_transit",
"round": 26,
"players": [
{ "name": "Josh", "guid": "12345678901234567", "kills": 445, "headshots": 170, "downs": 2, "revives": 3, "score": 33500 },
{ "name": "Alex", "guid": "23456789012345678", "kills": 401, "headshots": 146, "downs": 4, "revives": 1, "score": 29750 }
{
"name": "Josh",
"guid": "12345678901234567",
"kills": 445,
"headshots": 170,
"downs": 2,
"revives": 3,
"score": 33500
},
{
"name": "Alex",
"guid": "23456789012345678",
"kills": 401,
"headshots": 146,
"downs": 4,
"revives": 1,
"score": 29750
}
]
}
```
@@ -214,7 +246,7 @@ an occupied server can still have no scoreboard report.
Requirements on a qualifying server are ANDed:
| Requirement | Evidence |
| --- | --- |
| --------------------------- | ------------------------------------------------------------------------------------------------ |
| Reach round X | A round change, heartbeat, or game end reports round >= X with the required player count |
| Complete qualifying matches | Game end at round 2 or later, optionally qualified by a higher round, player count, and duration |
| Populated minutes | Time between consecutive game reports with enough players at both ends |
docs/guides/getting-started.md
+1 -1
View File
@@ -27,7 +27,7 @@ openssl rand -hex 32
Set these values in `.env`:
| Variable | Purpose |
| --- | --- |
| -------------------- | ---------------------------------------------------------------------------------------------------------------- |
| `EC_PUBLIC_URL` | The external HTTPS address of the dashboard. |
| `EC_SETUP_TOKEN` | The generated secret, used to create the first administrator. |
| `EC_DATA_DIR` | A persistent directory for the database, game files, and beta archives. |
docs/guides/leaderboard.md
+7 -7
View File
@@ -88,7 +88,7 @@ of inferring one. GUIDs stay strings.
## Recorded statistics
| Statistic | Meaning |
| --- | --- |
| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Highest round | Highest round reported with that player present, including players who joined mid-game. |
| Total kills, headshots, revives, downs | Sum of each player's highest reported counter in each game. Round snapshots are cumulative, so they are never added together within a game. |
| Total rounds | Round progression during observed participation, added across games. A player observed at game start and then round 8 receives seven rounds. A player first seen at round 8 gets a highest-round record of 8, but zero total rounds; reaching round 9 then adds one. |
@@ -212,7 +212,7 @@ All parameters are optional. Names and enumerated values are case-sensitive.
Use URL encoding for map keys and search text.
| Parameter | Default | Accepted values and behavior |
| --- | --- | --- |
| --------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `game` | Both games | Exactly `t5` or `t6`. An empty value is invalid. |
| `map` | All maps | Opaque `maps[].key`, at most 1024 characters. Empty means no map filter. An unknown key or a game/map mismatch returns an empty selection, not an error. |
| `players` | All squad sizes | Exactly `1`, `2`, `3`, or `4`, matching the recorded starting player count. Unknown counts contribute only when this filter is absent. |
@@ -236,7 +236,7 @@ HTTP `200` returns the following top-level fields directly, with no `ok` or
`data` wrapper:
| Field | Type | Meaning |
| --- | --- | --- |
| ----------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------- |
| `entries` | Entry array | Requested page, ordered by the selected metric descending. |
| `maps` | Map array | Available completed-match map options, filtered only by `game`, independent of `map`, `q`, metric qualification and pagination. |
| `total` | number | Number of matching GUIDs after metric qualification and search, before pagination. |
@@ -248,7 +248,7 @@ Each entry combines one GUID's qualifying participation across the selected
games/maps. It contains all of these fields regardless of the selected metric:
| Entry field | Type | Meaning |
| --- | --- | --- |
| ------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------- |
| `guid` | string | Player identity. Preserve it as a string, including leading zeros. |
| `name` | string | Latest reported name across both games, even if that report is outside the selected scope or from an unfinished match. |
| `rank` | number | Rank within the scope after metric qualification, before search/pagination. Ties share rank and leave gaps: 1, 1, 3. |
@@ -263,7 +263,7 @@ search, not an exact-GUID lookup; a tool looking up one GUID must check equality
on returned `guid` values and account for pagination.
| Ratio | Calculation | Requirement to rank by this metric |
| --- | --- | --- |
| ---------------- | ------------------------------ | ------------------------------------- |
| `headshotRate` | `headshots / kills` | `gamesPlayed >= 5` and `kills >= 100` |
| `killsPerDown` | `kills / downs` | `gamesPlayed >= 5` and `downs >= 5` |
| `averageKills` | `kills / gamesPlayed` | `gamesPlayed >= 5` |
@@ -284,7 +284,7 @@ completed matches reaching round 2, so an option can exist without any qualifyin
players. Removed servers' maps remain available.
| Summary field | Type | Meaning |
| --- | --- | --- |
| ------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------- |
| `players` | number | Unique qualifying GUIDs, combined across games. |
| `matches` | number | Distinct completed matches with qualifying player participation; not the sum of `gamesPlayed`. |
| `kills`, `roundsSurvived` | number | Sums across qualifying players; rounds are player-rounds. |
@@ -302,7 +302,7 @@ scope as the entry, and ties choose the earliest completion, then lowest match
ID. The three records can refer to different matches.
| Evidence field | Type | Meaning |
| --- | --- | --- |
| ------------------------------------- | --------------- | ---------------------------------------------------------------------------------------------------- |
| `matchId`, `value` | number | Completed match ID and that personal-best value. Link to `/matches/{matchId}` for its results page. |
| `game`, `mapKey`, `mapName`, `server` | string | Recorded game, opaque map key, map display name and server name. |
| `startedAt` | string | First recorded match observation; only the game start when `startObserved` is true. |
docs/guides/metrics.md
+1 -1
View File
@@ -64,7 +64,7 @@ represents the current registration rather than every instance in the history.
Hosting Rules v1 deliberately offer experiments even with small samples:
| Recorded condition | Suggested experiment |
| --- | --- |
| ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------- |
| At least 20% of known capacity time had every lobby full | Trial one additional lobby during busy hours. |
| Capacity unavailable and players present for at least 80% of known server time | Tentatively trial an extra lobby using occupancy as a proxy. |
| Players present for at most 10% of known time, with under 1% measured pressure or unavailable capacity | Review one idle lobby or a shorter hosting window while preserving access to the map. |
docs/guides/plugin-releases.md
+1 -1
View File
@@ -52,7 +52,7 @@ GitHub release is imported from its detail page.
- `GET /api/plugins/github/release?url=...` previews a release and its DLL assets.
- `POST /api/plugins/github/import` accepts JSON `{ url, assetId, name,
scopeType?, scopeValue? }` for a new plugin, or `{ url, assetId, pluginId }`
scopeType?, scopeValue? }` for a new plugin, or `{ url, assetId, pluginId }`
for an existing plugin. Use the preview's exact release URL, not a mutable
`latest` URL. Returns plugin/version IDs, checksum and the beta builder URL.
- `GET /api/plugins/updates` checks all plugins; `?pluginId=123` checks one.
docs/guides/provisioning.md
+1 -1
View File
@@ -35,7 +35,7 @@ See [Collections](collections.md) for the review details and safe submission
retries when a response is lost.
| Action | Behavior |
| --- | --- |
| ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Create | Create a new instance using the latest promoted files and verify container health, game status, the selected map, and the Plutonium revision. |
| Start | Start the existing container and verify container health and RCON. |
| Restart | Wait until empty, restart the existing container, and verify container health and RCON. |
docs/guides/settings.md
+1 -1
View File
@@ -45,7 +45,7 @@ is a delay: 0 adds no delay. A positive age and version count together preserve
minimum count as well as versions younger than the age threshold.
| History or storage | Default |
| --- | --- |
| ---------------------------------------------- | ----------------------------------------------- |
| Campaigns | 7 days |
| Successful routine operations | 1 day |
| Provisioning workflows | Indefinite |
docs/plans/lobby-autoscaling-plan.md
+3 -3
View File
@@ -15,7 +15,7 @@ Update October 7, 2026: Dry run and Create only have been removed. Current polic
## Existing foundations and required extensions
| Foundation | Reuse and required work |
| --- | --- |
| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Background worker | Add a requester to [pipeline/worker.ts](../../src/lib/server/pipeline/worker.ts), which already reconciles background work every 15 seconds independently of Provisioning. Keep host execution inside Provisioning. |
| Occupancy evidence | Reuse [status evidence](../../src/lib/server/fleet/status-evidence.ts) and shared freshness rules. Monitoring runs on a 30-second cadence; accepted evidence currently expires after 90 seconds. Positive occupancy vetoes an empty claim. Unknown is never zero. |
| Safe creation | Use the durable creation intent and placement path in [provisioning/native.ts](../../src/lib/server/provisioning/native.ts). Preserve CPU, memory, port, host eligibility, library, and startup concurrency checks. A blocked create remains queued. |
@@ -32,7 +32,7 @@ Create already uses the promoted launch recipe and capacity-aware placement. Sta
Use one policy per `(game, mapSlug)`, preserving configured variants such as T6 start locations. Do not group ambiguous legacy records by engine map alone. Resolve or exclude ambiguous identities before activation.
| Setting | Proposed initial behavior |
| --- | --- |
| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
| Mode | Off, Dry run, Create only, or Create and destroy. Default Off. |
| Minimum running lobbies | Default 1; a floor for desired ready capacity, subject to available keys and host capacity. |
| Maximum lobbies | Required explicit limit, at most 20. Count occupied lobbies, registered identities, reservations, and unresolved creations toward admission. |
@@ -122,7 +122,7 @@ Complete and validate each phase in a separate commit before starting the next.
Required behavioral scenarios across these phases:
| Scenario | Expected result |
| --- | --- |
| ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Last spare fills, including a high-round partially filled session | One create when eligible; free player slots do not count as spares. |
| Create is queued, starting, or awaiting verification | Reservation counts once; it is not advertised as ready and no duplicate is queued. |
| Every eligible host is full or unavailable | Existing job waits; other policies and worker tasks continue. |
docs/plans/player-achievements-and-visual-refresh-plan.md
+3 -3
View File
@@ -22,7 +22,7 @@ its own.
## Scope and defaults
| Area | First release |
| --- | --- |
| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Leaderboard periods | All time, This week, and This month, with previous/next period navigation. Keep All time as the default. |
| Calendar | Weeks start Monday at 00:00 UTC; months start on the first at 00:00 UTC. Show the boundaries explicitly. Personal timezone preferences format timestamps without changing competition membership. |
| Personal bests | Highest round, best game kills, and highest reported score, compared within the same game and map variant. |
@@ -213,7 +213,7 @@ styling, and document their use.
### Information order and status consistency
| Surface | Presentation changes |
| --- | --- |
| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Fleet cards | Map/lobby/game first; health, player count, and round next; roster and match duration below; lower-priority timing and technical identity in supporting details. Preserve compact empty cards, search, pins, table mode, and the match spotlight. |
| Server detail | Add map/game identity to the shared header with health. Keep player counts, round, and timing in the Live content so Configuration and Diagnostics remain focused on their task. |
| Autoscaling | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Off/On setting, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. |
@@ -239,7 +239,7 @@ player-achievement workstream before starting the visual refresh. Commit any
unrelated fix separately.
| Phase | Reviewable result | Acceptance gate |
| --- | --- | --- |
| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| A1 Completion time and period queries | Stable completion field/migration, period parsing, scoped ranking/evidence/profile queries, and cache keys. | Boundary, correction, qualification, and all-time compatibility tests pass; query plans and representative timing are recorded. |
| A2 Period navigation | Week/month controls, history navigation, period metadata, profile context, and stable share URLs. | Direct links, back/forward, rollover, empty periods, and inspection-preserving refresh work on desktop and mobile. |
| A3 Personal best highlights | Derived milestones, profile section, and match annotations. | Strict improvements, ties, first results, variants, corrections, and undated history behave as specified; long player histories remain practical. |
docs/research/leaderboard-query-benchmark.md
+1 -1
View File
@@ -21,7 +21,7 @@ On the local Intel i5-8279U / Node 25.9.0 / SQLite 3.49.2 test environment, thre
measured runs after warmup gave these median query times on the large fixture:
| Request | Before | After |
| --- | ---: | ---: |
| ------------------------ | -------: | -----: |
| Highest round, all games | 1,717 ms | 829 ms |
| Total kills, all games | 1,555 ms | 858 ms |
| One game | 899 ms | 491 ms |
docs/research/t5-t6-idle-cpu-research.md
+2 -2
View File
@@ -27,7 +27,7 @@ The live host kernel, Wine version, image digest, installed scripts/plugins, CPU
## Options
| Option | Expected effect on idle CPU | Availability and gameplay tradeoff | Evidence and priority |
| --- | --- | --- | --- |
| --------------------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
| 1. Enable and verify NTSYNC | Potentially substantial if Wine synchronization dominates | Intended to retain normal joins and gameplay; compatibility testing required | Official Wine/kernel support; Plutonium operator anecdote. First trial |
| 2. Audit busy-wait settings, scripts, and helpers | Small to substantial, depending on the actual cause | Usually compatible with keeping lobbies online | Documented settings plus diagnostic experiments. First trial |
| 3. Apply CPU quotas only when empty | Enforceable ceiling on each empty container's CPU time | Excessive throttling can delay joins, RCON, and heartbeats | Docker-supported mechanism; automatic policy needs development |
@@ -156,7 +156,7 @@ First record the actual host/VM details, kernel, image digest, Wine version, Plu
Use one representative instance from each game, then expand successful changes to other maps. Suggested measurement windows are 10–15 minutes after startup settles, repeated after players join and all leave. These are proposed test windows, not product requirements.
| Test | What it resolves |
| --- | --- |
| ---------------------------------------------------------- | --------------------------------------------------------- |
| Normal deployment, empty immediately after startup | Initial baseline |
| Same instance after the last player disconnects | Whether work accumulates or persists after play |
| Minimal scripts/plugins on the same map | Contribution from custom behavior |
eslint.config.js
+64
View File
@@ -0,0 +1,64 @@
import js from '@eslint/js';
import { defineConfig, globalIgnores } from 'eslint/config';
import prettier from 'eslint-config-prettier';
import svelte from 'eslint-plugin-svelte';
import globals from 'globals';
import ts from 'typescript-eslint';
import svelteConfig from './svelte.config.js';
export default defineConfig(
globalIgnores([
'**/node_modules/',
'.svelte-kit/',
'build/',
'data/',
'test-fixtures/',
'static/agent/',
'.claude/',
'.gstack/'
]),
js.configs.recommended,
ts.configs.recommended,
svelte.configs.recommended,
prettier,
svelte.configs.prettier,
{
// The controller is deployed at the origin root, including imperative navigation in TS.
rules: { 'svelte/no-navigation-without-resolve': 'off' }
},
{
languageOptions: {
globals: { ...globals.browser, ...globals.node }
},
rules: {
'@typescript-eslint/no-unused-vars': [
'error',
{
argsIgnorePattern: '^_',
varsIgnorePattern: '^_',
caughtErrorsIgnorePattern: '^_',
ignoreRestSiblings: true
}
]
}
},
{
files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js'],
languageOptions: {
parserOptions: {
parser: ts.parser,
svelteConfig
}
},
rules: {
// Existing lists include intentionally positional rows and repeated primitive values.
'svelte/require-each-key': 'off',
// Local collection helpers do not need reactive replacements.
'svelte/prefer-svelte-reactivity': 'off',
// Preserve the existing state-reset behavior during tooling adoption.
'svelte/prefer-writable-derived': 'off',
// Explicit string interpolation preserves whitespace around inline controls and logs.
'svelte/no-useless-mustaches': 'off'
}
}
);
package-lock.json
Generated
+1566 -1
View File
File diff suppressed because it is too large Load Diff
package.json
+13 -3
View File
@@ -1,11 +1,11 @@
{
"name": "electric-cherry",
"version": "0.1.0",
"description": "T5/T6 Plutonium Zombies fleet manager \u2014 replaces push_scripts.sh with a live web UI",
"description": "T5/T6 Plutonium Zombies fleet manager — replaces push_scripts.sh with a live web UI",
"private": true,
"type": "module",
"engines": {
"node": "^22.12.0 || >=24.0.0"
"node": "^22.13.0 || >=24.0.0"
},
"scripts": {
"dev": "vite dev",
@@ -25,7 +25,9 @@
"predev": "npm run agent:package && npm run build:reports && npm run build:launch",
"pretest": "npm run build:reports && npm run build:launch",
"pretest:watch": "npm run build:reports && npm run build:launch",
"check:styles": "node scripts/check-ui-styles.mjs"
"check:styles": "node scripts/check-ui-styles.mjs",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"dependencies": {
"@fontsource/alfa-slab-one": "^5.3.0",
@@ -34,15 +36,23 @@
"better-sqlite3": "^11.3.0"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
"@sveltejs/adapter-node": "^5.2.9",
"@sveltejs/kit": "^2.70.3",
"@sveltejs/vite-plugin-svelte": "^6.2.4",
"@types/better-sqlite3": "^7.6.11",
"@types/node": "^22.7.0",
"eslint": "^10.12.0",
"eslint-config-prettier": "^10.1.8",
"eslint-plugin-svelte": "^3.23.1",
"globals": "^17.13.0",
"prettier": "^3.9.9",
"prettier-plugin-svelte": "^3.5.2",
"svelte": "^5.1.0",
"svelte-check": "^4.0.0",
"tsx": "^4.19.0",
"typescript": "^5.6.0",
"typescript-eslint": "^8.71.1",
"vite": "^7.3.7",
"vitest": "^4.1.11"
},
scripts/benchmark-leaderboard.mjs
+232 -63
View File
@@ -24,33 +24,51 @@ const allowed = new Set(['--scenario', '--runs', '--baseline-ref', '--json']);
for (let index = 0; index < arguments_.length; index += 2) {
const key = arguments_[index];
const value = arguments_[index + 1];
if (!allowed.has(key) || !value) throw new Error('Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH');
if (!allowed.has(key) || !value)
throw new Error(
'Usage: --scenario small|large|all --runs N --baseline-ref COMMIT --json PATH'
);
options[key.slice(2)] = value;
}
options.runs = Number(options.runs);
if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20) throw new Error('--runs must be an integer from 1 to 20');
if (!['small', 'large', 'all'].includes(options.scenario)) throw new Error('--scenario must be small, large, or all');
if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 20)
throw new Error('--runs must be an integer from 1 to 20');
if (!['small', 'large', 'all'].includes(options.scenario))
throw new Error('--scenario must be small, large, or all');
const baseline = options['baseline-ref']
? execFileSync('git', ['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`], { cwd: repo, encoding: 'utf8' }).trim()
? execFileSync(
'git',
['rev-parse', '--verify', '--end-of-options', `${options['baseline-ref']}^{commit}`],
{ cwd: repo, encoding: 'utf8' }
).trim()
: null;
const definitions = [
{ name: 'small', matches: 2000, players: 500 },
{ name: 'large', matches: 50000, players: 10000 }
].filter(item => options.scenario === 'all' || item.name === options.scenario);
].filter((item) => options.scenario === 'all' || item.name === options.scenario);
const temporary = mkdtempSync(join(tmpdir(), 'cherry-leaderboard-benchmark-'));
const source = (file, ref) => ref
? execFileSync('git', ['show', `${ref}:${file}`], { cwd: repo, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 })
const source = (file, ref) =>
ref
? execFileSync('git', ['show', `${ref}:${file}`], {
cwd: repo,
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024
})
: readFileSync(join(repo, file), 'utf8');
const percentile = (values, fraction) => [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1];
const percentile = (values, fraction) =>
[...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1];
function seed(handle, definition) {
handle.exec(source('src/lib/server/db/schema.sql'));
handle.exec("CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)");
handle.exec(
"CREATE TABLE report_revisions(topic TEXT PRIMARY KEY, revision INTEGER NOT NULL); INSERT INTO report_revisions VALUES ('leaderboard',0)"
);
const match = handle.prepare(`INSERT INTO leaderboard_matches
(id, server, game, map, map_key, map_name, started_at, ended_at, last_received_at,
status, start_observed, start_players, highest_round, last_event, last_fingerprint)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 4, ?, ?, ?)`);
const identity = handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
const identity =
handle.prepare(`INSERT INTO leaderboard_players (game, guid, name, first_seen_at, last_seen_at)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(game, guid) DO UPDATE SET
name = CASE WHEN excluded.last_seen_at >= last_seen_at THEN excluded.name ELSE name END,
last_seen_at = MAX(last_seen_at, excluded.last_seen_at)`);
@@ -60,19 +78,38 @@ function seed(handle, definition) {
first_observed_round, present_at_start)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
let state = 20260918;
const random = () => { state = (Math.imul(state, 1664525) + 1013904223) >>> 0; return state / 4294967296; };
const random = () => {
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
return state / 4294967296;
};
handle.transaction(() => {
for (let id = 1; id <= definition.matches; id++) {
const mapIndex = Math.floor(random() * 12);
const game = mapIndex < 6 ? 't5' : 't6';
const map = `map_${mapIndex}`;
const status = id % 20 === 0 ? 'active' : id % 20 === 1 ? 'interrupted' : 'completed';
const started = new Date(Date.UTC(2023, 0, 1) + Math.floor(id / definition.matches * 3 * 365 * 86400000)).toISOString();
const ended = new Date(Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000).toISOString();
const started = new Date(
Date.UTC(2023, 0, 1) + Math.floor((id / definition.matches) * 3 * 365 * 86400000)
).toISOString();
const ended = new Date(
Date.parse(started) + (20 + Math.floor(random() * 130)) * 60000
).toISOString();
const round = id % 101 === 0 ? 1 : 2 + Math.floor(random() * 60);
match.run(id, `server_${id % 24}`, game, map, `${game}:${map}`, `Map ${mapIndex}`,
started, status === 'completed' ? ended : null, ended, status, round,
status === 'completed' ? 'game_end' : 'heartbeat', `event_${id}`);
match.run(
id,
`server_${id % 24}`,
game,
map,
`${game}:${map}`,
`Map ${mapIndex}`,
started,
status === 'completed' ? ended : null,
ended,
status,
round,
status === 'completed' ? 'game_end' : 'heartbeat',
`event_${id}`
);
const roster = new Set();
while (roster.size < 4) roster.add(Math.floor(random() ** 2 * definition.players));
// Reserve one completed participation for an explicitly ineligible player.
@@ -84,16 +121,47 @@ function seed(handle, definition) {
const kills = Math.floor(round * 18 + random() * 800);
const presentAtStart = random() > 0.1;
const firstRound = presentAtStart ? 0 : Math.max(1, Math.floor(random() * round));
participation.run(id, guid, round, Math.max(0, round - Math.max(1, firstRound)), kills,
Math.floor(kills * random()), Math.floor(random() * 8), Math.floor(random() * 12),
Math.floor(random() * 100000), Math.max(0, round - 1), Number(status === 'active'),
Number(random() > 0.01), Number(status === 'completed' && random() > 0.15),
started, ended, firstRound, Number(presentAtStart));
participation.run(
id,
guid,
round,
Math.max(0, round - Math.max(1, firstRound)),
kills,
Math.floor(kills * random()),
Math.floor(random() * 8),
Math.floor(random() * 12),
Math.floor(random() * 100000),
Math.max(0, round - 1),
Number(status === 'active'),
Number(random() > 0.01),
Number(status === 'completed' && random() > 0.15),
started,
ended,
firstRound,
Number(presentAtStart)
);
}
if (id === definition.matches - 2) {
identity.run(game, 'benchmark-low-sample', 'Low sample', started, ended);
participation.run(id, 'benchmark-low-sample', 2, 1, 20, 5, 0, 0, 1000,
1, 0, 1, 1, started, ended, 0, 1);
participation.run(
id,
'benchmark-low-sample',
2,
1,
20,
5,
0,
0,
1000,
1,
0,
1,
1,
started,
ended,
0,
1
);
}
}
})();
@@ -104,15 +172,27 @@ function loadImplementation(handle, ref) {
const modules = new Map();
const hashes = {};
let recorded = null;
const database = { prepare(sql) {
const database = {
prepare(sql) {
const statement = handle.prepare(sql);
return Object.fromEntries(['all', 'get'].map(method => [method, (...parameters) => {
return Object.fromEntries(
['all', 'get'].map((method) => [
method,
(...parameters) => {
const start = performance.now();
const result = statement[method](...parameters);
if (recorded) recorded.push({ sql, parameters, milliseconds: performance.now() - start });
if (recorded)
recorded.push({
sql,
parameters,
milliseconds: performance.now() - start
});
return result;
}]));
} };
}
])
);
}
};
function load(file) {
if (file === 'src/lib/server/db/index.ts') return { db: () => database };
if (modules.has(file)) return modules.get(file);
@@ -123,10 +203,14 @@ function loadImplementation(handle, ref) {
}).outputText;
const exports = {};
modules.set(file, exports);
const localRequire = specifier => {
const localRequire = (specifier) => {
if (!specifier.startsWith('.')) return require(specifier);
const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(/\.js$/, '.ts');
if (!target.startsWith('src/lib/')) throw new Error(`Unexpected benchmark dependency: ${target}`);
const target = relative(repo, resolve(repo, dirname(file), specifier)).replace(
/\.js$/,
'.ts'
);
if (!target.startsWith('src/lib/'))
throw new Error(`Unexpected benchmark dependency: ${target}`);
return load(target);
};
new Function('require', 'exports', compiled)(localRequire, exports);
@@ -134,10 +218,14 @@ function loadImplementation(handle, ref) {
}
const api = load('src/lib/server/fleet/leaderboard.ts');
return {
name: ref ? 'baseline' : 'current', ref, hashes,
metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map(metric => metric.key),
name: ref ? 'baseline' : 'current',
ref,
hashes,
metrics: load('src/lib/leaderboard.ts').leaderboardMetrics.map((metric) => metric.key),
run(test) {
return test.guid ? api.getPlayerLeaderboardEntry(test.guid, test.options) : api.getLeaderboard(test.options);
return test.guid
? api.getPlayerLeaderboardEntry(test.guid, test.options)
: api.getLeaderboard(test.options);
},
measure(test) {
recorded = [];
@@ -152,8 +240,16 @@ function loadImplementation(handle, ref) {
}
const report = {
environment: { node: process.version, sqlite: null, platform: platform(), arch: arch(), cpu: cpus()[0]?.model },
baselineRef: baseline, runs: options.runs, seed: 20260918,
environment: {
node: process.version,
sqlite: null,
platform: platform(),
arch: arch(),
cpu: cpus()[0]?.model
},
baselineRef: baseline,
runs: options.runs,
seed: 20260918,
notes: 'Synthetic file-backed SQLite, one warmup per case, rotating execution order, warm OS cache. Includes SQL preparation and source-record lookup; excludes HTTP and browser rendering. Shared local CPU; timings are not production latency. Schema and indexes are current for both implementations.',
scenarios: []
};
@@ -161,77 +257,148 @@ try {
for (const definition of definitions) {
const handle = new Database(join(temporary, `${definition.name}.sqlite`));
try {
report.environment.sqlite = handle.prepare('SELECT sqlite_version() AS version').get().version;
report.environment.sqlite = handle
.prepare('SELECT sqlite_version() AS version')
.get().version;
seed(handle, definition);
const implementations = [...(baseline ? [loadImplementation(handle, baseline)] : []), loadImplementation(handle)];
const implementations = [
...(baseline ? [loadImplementation(handle, baseline)] : []),
loadImplementation(handle)
];
const cases = [
{ name: 'unfiltered', options: {} },
{ name: 'kills', options: { metric: 'kills' } },
{ name: 'game', options: { game: 't5', metric: 'kills' } },
{ name: 'map', options: { game: 't6', map: 't6:map_8', metric: 'kills' } },
{ name: 'search', options: { q: '00042', metric: 'kills' } },
{ name: 'deep-page', options: { metric: 'kills', offset: Math.floor(definition.players / 2) } },
{
name: 'deep-page',
options: { metric: 'kills', offset: Math.floor(definition.players / 2) }
},
{ name: 'efficiency', options: { metric: 'headshotRate' } },
{ name: 'exact-player', guid: '10000000000000042', options: { metric: 'kills' } },
{ name: 'empty-search', options: { q: 'No such player exists' } },
{ name: 'past-end', options: { offset: 999999 } },
{ name: 'empty-map', options: { map: 't5:missing' } }
];
const counts = handle.prepare(`SELECT COUNT(*) AS participationRows,
COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`).get();
const scenario = { ...definition, ...counts, maps: 12, servers: 24, completedShare: 0.9, results: [] };
const counts = handle
.prepare(
`SELECT COUNT(*) AS participationRows,
COUNT(DISTINCT guid) AS uniquePlayers FROM leaderboard_match_players`
)
.get();
const scenario = {
...definition,
...counts,
maps: 12,
servers: 24,
completedShare: 0.9,
results: []
};
report.scenarios.push(scenario);
// Broad parity is separate from timings so every ratio and edge case is
// checked without multiplying the larger fixture's timed request matrix.
if (baseline && definition.name === 'small') {
const scopes = [{}, { game: 't5' }, { game: 't6', map: 't6:map_8' }, { q: '00042' },
{ q: 'No such player exists' }, { offset: 999999 }, { map: 't5:missing' }];
const scopes = [
{},
{ game: 't5' },
{ game: 't6', map: 't6:map_8' },
{ q: '00042' },
{ q: 'No such player exists' },
{ offset: 999999 },
{ map: 't5:missing' }
];
let checked = 0;
for (const metric of implementations[1].metrics) {
for (const scope of scopes) {
const test = { options: { ...scope, metric } };
assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test),
`${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`);
assert.deepStrictEqual(
implementations[1].run(test),
implementations[0].run(test),
`${definition.name}/${metric}/${JSON.stringify(scope)}: full-response parity`
);
checked++;
}
for (const guid of ['10000000000000042', 'benchmark-low-sample', 'benchmark-missing']) {
for (const guid of [
'10000000000000042',
'benchmark-low-sample',
'benchmark-missing'
]) {
const test = { guid, options: { metric } };
assert.deepStrictEqual(implementations[1].run(test), implementations[0].run(test),
`${definition.name}/${metric}/${guid}: exact-player parity`);
assert.deepStrictEqual(
implementations[1].run(test),
implementations[0].run(test),
`${definition.name}/${metric}/${guid}: exact-player parity`
);
checked++;
}
}
scenario.parityCases = checked;
console.log(JSON.stringify({ scenario: definition.name, parityCases: checked, parity: true }));
console.log(
JSON.stringify({
scenario: definition.name,
parityCases: checked,
parity: true
})
);
}
for (const test of cases) {
const expected = implementations[0].run(test);
for (const implementation of implementations.slice(1)) {
assert.deepStrictEqual(implementation.run(test), expected, `${definition.name}/${test.name}: baseline parity`);
assert.deepStrictEqual(
implementation.run(test),
expected,
`${definition.name}/${test.name}: baseline parity`
);
}
const samples = Object.fromEntries(implementations.map(item => [item.name, []]));
const samples = Object.fromEntries(implementations.map((item) => [item.name, []]));
const latestStatements = {};
for (let index = 0; index < options.runs; index++) {
const rotation = index % implementations.length;
for (const implementation of [...implementations.slice(rotation), ...implementations.slice(0, rotation)]) {
for (const implementation of [
...implementations.slice(rotation),
...implementations.slice(0, rotation)
]) {
const measured = implementation.measure(test);
samples[implementation.name].push(measured.milliseconds);
latestStatements[implementation.name] = measured.statements;
}
}
const result = { ...test, parity: baseline ? true : null, implementations: implementations.map(implementation => ({
name: implementation.name, sourceHashes: implementation.hashes,
const result = {
...test,
parity: baseline ? true : null,
implementations: implementations.map((implementation) => ({
name: implementation.name,
sourceHashes: implementation.hashes,
medianMs: percentile(samples[implementation.name], 0.5),
p95Ms: percentile(samples[implementation.name], 0.95), samplesMs: samples[implementation.name],
statements: latestStatements[implementation.name].map(statement => ({ ...statement,
plan: handle.prepare(`EXPLAIN QUERY PLAN ${statement.sql}`).all(...statement.parameters)
p95Ms: percentile(samples[implementation.name], 0.95),
samplesMs: samples[implementation.name],
statements: latestStatements[implementation.name].map((statement) => ({
...statement,
plan: handle
.prepare(`EXPLAIN QUERY PLAN ${statement.sql}`)
.all(...statement.parameters)
}))
})) };
}))
};
scenario.results.push(result);
console.log(JSON.stringify({ scenario: definition.name, case: test.name, parity: result.parity,
medianMs: Object.fromEntries(result.implementations.map(item => [item.name, Math.round(item.medianMs)])) }));
console.log(
JSON.stringify({
scenario: definition.name,
case: test.name,
parity: result.parity,
medianMs: Object.fromEntries(
result.implementations.map((item) => [
item.name,
Math.round(item.medianMs)
])
)
})
);
}
} finally {
handle.close();
}
} finally { handle.close(); }
}
if (options.json) {
const target = resolve(options.json);
@@ -239,4 +406,6 @@ try {
writeFileSync(target, JSON.stringify(report, null, 2));
console.log(`Detailed results: ${target}`);
}
} finally { rmSync(temporary, { recursive: true, force: true }); }
} finally {
rmSync(temporary, { recursive: true, force: true });
}
scripts/check-ui-styles.mjs
+10 -4
View File
@@ -12,15 +12,21 @@ async function visit(directory) {
const source = await readFile(file, 'utf8');
for (const match of source.matchAll(/font-size\s*:\s*[\d.]+px\b/g)) {
const line = source.slice(0, match.index).split('\n').length;
failures.push(`${path.relative(process.cwd(), file)}:${line}: use a shared type token (12 px minimum): ${match[0]}`);
failures.push(
`${path.relative(process.cwd(), file)}:${line}: use a shared type token (12 px minimum): ${match[0]}`
);
}
}
}
}
await visit(root);
const tokens = await readFile(path.join(root, 'lib/ui/styles/tokens.css'), 'utf8');
for (const match of tokens.matchAll(/--text-(?:xs|sm|base|md|lg|xl|2xl|3xl|input):\s*([\d.]+)px/g)) {
for (const match of tokens.matchAll(
/--text-(?:xs|sm|base|md|lg|xl|2xl|3xl|input):\s*([\d.]+)px/g
)) {
if (Number(match[1]) < 12) failures.push(`Type tokens must be at least 12 px: ${match[0]}`);
}
if (failures.length) { console.error(failures.join('\n')); process.exitCode = 1; }
else console.log('UI styles: shared font sizes and 12 px minimum verified.');
if (failures.length) {
console.error(failures.join('\n'));
process.exitCode = 1;
} else console.log('UI styles: shared font sizes and 12 px minimum verified.');
scripts/database-diagnostics.mjs
+27 -10
View File
@@ -4,13 +4,30 @@ import { statSync } from 'node:fs';
const path = process.argv[2] ?? process.env.DB_PATH ?? './data/cherry.db';
const db = new Database(path, { readonly: true, fileMustExist: true });
try {
console.table(['', '-wal', '-shm'].map(suffix => {
try { return { file: path + suffix, bytes: statSync(path + suffix).size }; }
catch (error) { if (error.code !== 'ENOENT') throw error; return { file: path + suffix, bytes: 0 }; }
}));
console.table(db.prepare(`SELECT name, ROUND(SUM(pgsize)/1048576.0, 2) AS mib
FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC LIMIT 20`).all());
console.table(['page_size', 'page_count', 'freelist_count', 'auto_vacuum'].map(pragma => ({
pragma, value: db.pragma(pragma, { simple: true })
})));
} finally { db.close(); }
console.table(
['', '-wal', '-shm'].map((suffix) => {
try {
return { file: path + suffix, bytes: statSync(path + suffix).size };
} catch (error) {
if (error.code !== 'ENOENT') throw error;
return { file: path + suffix, bytes: 0 };
}
})
);
console.table(
db
.prepare(
`SELECT name, ROUND(SUM(pgsize)/1048576.0, 2) AS mib
FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC LIMIT 20`
)
.all()
);
console.table(
['page_size', 'page_count', 'freelist_count', 'auto_vacuum'].map((pragma) => ({
pragma,
value: db.pragma(pragma, { simple: true })
}))
);
} finally {
db.close();
}
scripts/fixture-data.ts
+511 -121
View File
@@ -9,7 +9,14 @@ const DAY = 24 * 60 * MINUTE;
const GIB = 1024 ** 3;
const image = 'fixture.invalid/electric-cherry/preview:local';
type Scenario = { game: 't5' | 't6'; slug: string; players: number; round: number; minutes: number; state: 'active' | 'idle' | 'stopped' | 'stale' | 'starting' | 'host-offline' };
type Scenario = {
game: 't5' | 't6';
slug: string;
players: number;
round: number;
minutes: number;
state: 'active' | 'idle' | 'stopped' | 'stale' | 'starting' | 'host-offline';
};
const scenarios: Scenario[] = [
{ game: 't5', slug: 'theater', players: 4, round: 27, minutes: 76, state: 'active' },
{ game: 't5', slug: 'cosmodrome', players: 2, round: 16, minutes: 39, state: 'active' },
@@ -25,211 +32,594 @@ const scenarios: Scenario[] = [
{ game: 't6', slug: 'highrise', players: 0, round: 0, minutes: 0, state: 'host-offline' }
];
export const fixtureServers = scenarios.map((scenario, index) => ({ ...scenario,
name: `${scenario.game}_zombie_${scenario.slug}_1`, host: `fixture-host-${index === 11 ? 3 : index % 2 + 1}`,
instance: `fixture-instance-${index + 1}`, port: 28960 + index,
export const fixtureServers = scenarios.map((scenario, index) => ({
...scenario,
name: `${scenario.game}_zombie_${scenario.slug}_1`,
host: `fixture-host-${index === 11 ? 3 : (index % 2) + 1}`,
instance: `fixture-instance-${index + 1}`,
port: 28960 + index,
map: zombieMapBySlug(scenario.game, scenario.slug)!
}));
const names = ['Jamie Cross', 'Alex', 'Sam', 'Morgan', 'Riley', 'Taylor', 'Jordan', 'Casey', 'Quinn', 'Avery', 'Drew', 'Kai'];
const names = [
'Jamie Cross',
'Alex',
'Sam',
'Morgan',
'Riley',
'Taylor',
'Jordan',
'Casey',
'Quinn',
'Avery',
'Drew',
'Kai'
];
const iso = (at: number) => new Date(at).toISOString();
const player = (index: number) => ({ guid: index === 0 ? 'fixture-jamie' : `fixture-player-${index}`,
name: index === 0 ? names[0] : `${names[index % names.length]} ${Math.floor(index / names.length) + 1}` });
const rosterFor = (server: typeof fixtureServers[number], index: number) => Array.from({ length: server.players }, (_, slot) => ({ ...player(slot === 0 ? 0 : index * 4 + slot),
ping: 29 + slot * 7, kills: (slot + 1) * 137, headshots: (slot + 1) * 41, downs: slot % 2, revives: slot, score: (slot + 1) * 4100 }));
const seenAt = (server: Scenario, now: number) => iso(now - (['stopped', 'host-offline'].includes(server.state) ? 15 * MINUTE : server.state === 'stale' ? 105_000 : 0));
const containerState = (server: Scenario) => server.state === 'stopped' ? 'exited' : server.state === 'starting' ? 'starting' : 'running';
const instanceStates = (host: string, now: number) => fixtureServers.filter(server => server.host === host).map(server => ({
instanceId: server.instance, generation: 1, state: containerState(server), healthy: !['stopped', 'starting'].includes(server.state),
const player = (index: number) => ({
guid: index === 0 ? 'fixture-jamie' : `fixture-player-${index}`,
name:
index === 0
? names[0]
: `${names[index % names.length]} ${Math.floor(index / names.length) + 1}`
});
const rosterFor = (server: (typeof fixtureServers)[number], index: number) =>
Array.from({ length: server.players }, (_, slot) => ({
...player(slot === 0 ? 0 : index * 4 + slot),
ping: 29 + slot * 7,
kills: (slot + 1) * 137,
headshots: (slot + 1) * 41,
downs: slot % 2,
revives: slot,
score: (slot + 1) * 4100
}));
const seenAt = (server: Scenario, now: number) =>
iso(
now -
(['stopped', 'host-offline'].includes(server.state)
? 15 * MINUTE
: server.state === 'stale'
? 105_000
: 0)
);
const containerState = (server: Scenario) =>
server.state === 'stopped' ? 'exited' : server.state === 'starting' ? 'starting' : 'running';
const instanceStates = (host: string, now: number) =>
fixtureServers
.filter((server) => server.host === host)
.map((server) => ({
instanceId: server.instance,
generation: 1,
state: containerState(server),
healthy: !['stopped', 'starting'].includes(server.state),
observedAt: iso(now - (host === 'fixture-host-3' ? 10 * MINUTE : 0))
}));
}));
/** Uses only a caller-owned, fully migrated database. No host or worker code is imported. */
export function seedFixtureData(handle: Database.Database, now: number): void {
if (!Number.isFinite(now)) throw new Error('Invalid fixture time.');
const occupied = ['servers', 'leaderboard_matches', 'ec_hosts', 'auth_users', 'scripts', 'server_deployments', 'deployments', 'betas', 'game_file_libraries']
.some(table => handle.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get());
if (occupied || handle.prepare('SELECT 1 FROM settings WHERE key=?').get(FIXTURE_KEY)) throw new Error('Fixture seeding requires a new database.');
const occupied = [
'servers',
'leaderboard_matches',
'ec_hosts',
'auth_users',
'scripts',
'server_deployments',
'deployments',
'betas',
'game_file_libraries'
].some((table) => handle.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get());
if (occupied || handle.prepare('SELECT 1 FROM settings WHERE key=?').get(FIXTURE_KEY))
throw new Error('Fixture seeding requires a new database.');
const insert = (table: string, values: Record<string, unknown>) => {
const keys = Object.keys(values);
handle.prepare(`INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})`).run(...Object.values(values));
handle
.prepare(
`INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})`
)
.run(...Object.values(values));
};
const started = iso(now - 30 * DAY);
const rememberPlayer = handle.prepare(`INSERT INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?)
const rememberPlayer =
handle.prepare(`INSERT INTO leaderboard_players(game,guid,name,first_seen_at,last_seen_at) VALUES(?,?,?,?,?)
ON CONFLICT(game,guid) DO UPDATE SET first_seen_at=MIN(first_seen_at,excluded.first_seen_at),last_seen_at=MAX(last_seen_at,excluded.last_seen_at)`);
handle.transaction(() => {
insert('settings', { key: FIXTURE_KEY, value: FIXTURE_VERSION });
// Preview observations must never opt into automatic lifecycle changes.
for (const [key, value] of Object.entries({ maintenance_paused: '1', maintenance_health: '0', maintenance_drift: '0', maintenance_updates: '0', watchdog_enabled: '0', autoscaling_paused: '1' })) {
handle.prepare('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value);
for (const [key, value] of Object.entries({
maintenance_paused: '1',
maintenance_health: '0',
maintenance_drift: '0',
maintenance_updates: '0',
watchdog_enabled: '0',
autoscaling_paused: '1'
})) {
handle
.prepare(
'INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value'
)
.run(key, value);
}
for (const [index, name] of ['Chicago · Primary', 'Dallas · Overflow', 'Frankfurt · Europe'].entries()) {
for (const [index, name] of [
'Chicago · Primary',
'Dallas · Overflow',
'Frankfurt · Europe'
].entries()) {
const id = `fixture-host-${index + 1}`;
insert('ec_hosts', { id, name, hostname: `host-${index + 1}.fixture.invalid`, created_at: started, enrolled_at: started,
last_heartbeat: iso(now - (index === 2 ? 10 * MINUTE : 0)), inventory: JSON.stringify({ protocolVersion: 1, agentVersion: '1.7.0', cpuCount: 16,
cpuUsagePercent: 24 + index * 9, memoryTotal: 32 * GIB, memoryAvailable: (20 - index * 3) * GIB,
storageTotal: 500 * GIB, storageAvailable: 240 * GIB, dockerRunning: true, directoriesWritable: true,
runtimeImages: [image], libraries: [{ id: 'fixture-t5', state: 'ready' }, { id: 'fixture-t6', state: 'ready' }],
capabilities: ['instance.heartbeat-state'], instanceStates: instanceStates(id, now) }) });
insert('ec_hosts', {
id,
name,
hostname: `host-${index + 1}.fixture.invalid`,
created_at: started,
enrolled_at: started,
last_heartbeat: iso(now - (index === 2 ? 10 * MINUTE : 0)),
inventory: JSON.stringify({
protocolVersion: 1,
agentVersion: '1.7.0',
cpuCount: 16,
cpuUsagePercent: 24 + index * 9,
memoryTotal: 32 * GIB,
memoryAvailable: (20 - index * 3) * GIB,
storageTotal: 500 * GIB,
storageAvailable: 240 * GIB,
dockerRunning: true,
directoriesWritable: true,
runtimeImages: [image],
libraries: [
{ id: 'fixture-t5', state: 'ready' },
{ id: 'fixture-t6', state: 'ready' }
],
capabilities: ['instance.heartbeat-state'],
instanceStates: instanceStates(id, now)
})
});
// Deliberately not a token hash; no bearer token can authenticate as these agents.
insert('ec_host_credentials', { id: `fixture-credential-${index + 1}`, host_id: id, token_hash: `invalid-fixture-${index + 1}`, created_at: started });
insert('ec_host_credentials', {
id: `fixture-credential-${index + 1}`,
host_id: id,
token_hash: `invalid-fixture-${index + 1}`,
created_at: started
});
for (let minute = 0; minute < 7 * 1440; minute += 15) {
insert('ec_host_resource_samples', { host_id: id, sampled_at: iso(now - minute * MINUTE), cpu_count: 16,
cpu_usage_percent: 28 + index * 6 + 10 * Math.sin(minute / 95), memory_total: 32 * GIB,
memory_available: Math.round((18 - index * 2 + Math.sin(minute / 110)) * GIB) });
insert('ec_host_resource_samples', {
host_id: id,
sampled_at: iso(now - minute * MINUTE),
cpu_count: 16,
cpu_usage_percent: 28 + index * 6 + 10 * Math.sin(minute / 95),
memory_total: 32 * GIB,
memory_available: Math.round((18 - index * 2 + Math.sin(minute / 110)) * GIB)
});
}
}
let liveId = 495;
for (const [index, server] of fixtureServers.entries()) {
const beginning = iso(now - server.minutes * MINUTE - 12_000);
const roster = rosterFor(server, index);
insert('servers', { name: server.name, game: server.game, map: server.map.canonical, map_slug: server.slug, game_mode: 'zombie',
game_port: server.port, game_id: server.game === 't5' ? 't5sp' : 't6zm', image,
created_by: 'development fixture', created_at: started, last_players: server.players, last_map: server.map.canonical, last_gametype: 'zclassic',
last_roster: JSON.stringify(roster), last_seen_at: seenAt(server, now),
last_checked_at: iso(now), last_error: server.state === 'stale' ? 'timeout: no reply from game server' : null,
insert('servers', {
name: server.name,
game: server.game,
map: server.map.canonical,
map_slug: server.slug,
game_mode: 'zombie',
game_port: server.port,
game_id: server.game === 't5' ? 't5sp' : 't6zm',
image,
created_by: 'development fixture',
created_at: started,
last_players: server.players,
last_map: server.map.canonical,
last_gametype: 'zclassic',
last_roster: JSON.stringify(roster),
last_seen_at: seenAt(server, now),
last_checked_at: iso(now),
last_error: server.state === 'stale' ? 'timeout: no reply from game server' : null,
rcon_failures: server.state === 'stale' ? 2 : 0,
container_state: containerState(server),
container_checked_at: iso(now), desired_state: server.state === 'stopped' ? 'stopped' : 'running',
active_since: server.players ? beginning : null, inactive_since: server.players ? null : iso(now - 5 * MINUTE),
game_started_at: server.players ? beginning : null });
insert('native_instances', { id: server.instance, server: server.name, host_id: server.host,
library_id: `fixture-${server.game}`, image, port: server.port, created_at: started });
insert('plutonium_versions', { server: server.name, revision: 4878, checked_at: iso(now), error: null });
container_checked_at: iso(now),
desired_state: server.state === 'stopped' ? 'stopped' : 'running',
active_since: server.players ? beginning : null,
inactive_since: server.players ? null : iso(now - 5 * MINUTE),
game_started_at: server.players ? beginning : null
});
insert('native_instances', {
id: server.instance,
server: server.name,
host_id: server.host,
library_id: `fixture-${server.game}`,
image,
port: server.port,
created_at: started
});
insert('plutonium_versions', {
server: server.name,
revision: 4878,
checked_at: iso(now),
error: null
});
if (server.state === 'active' || server.state === 'idle') {
const matchId = server.players ? liveId++ : null;
const payload = { server: server.name, map: server.map.canonical, event: 'heartbeat', round: server.round, players: roster, receivedAt: iso(now),
timing: { matchId, players: Object.fromEntries(roster.map(p => [p.guid, { serverJoinedAt: beginning, matchJoinedAt: beginning, roundStarted: 1 }])) } };
insert('server_telemetry', { server: server.name, round: server.round, players: server.players, received_at: iso(now), payload: JSON.stringify(payload) });
const payload = {
server: server.name,
map: server.map.canonical,
event: 'heartbeat',
round: server.round,
players: roster,
receivedAt: iso(now),
timing: {
matchId,
players: Object.fromEntries(
roster.map((p) => [
p.guid,
{
serverJoinedAt: beginning,
matchJoinedAt: beginning,
roundStarted: 1
}
])
)
}
};
insert('server_telemetry', {
server: server.name,
round: server.round,
players: server.players,
received_at: iso(now),
payload: JSON.stringify(payload)
});
if (matchId !== null) {
insert('leaderboard_matches', { id: matchId, server: server.name, game: server.game, map: server.map.canonical,
map_key: server.slug, map_name: server.map.friendly, started_at: beginning, last_received_at: iso(now), status: 'active',
start_observed: 1, start_players: server.players, highest_round: server.round, last_event: 'heartbeat', last_fingerprint: `fixture-live-${index}` });
insert('leaderboard_matches', {
id: matchId,
server: server.name,
game: server.game,
map: server.map.canonical,
map_key: server.slug,
map_name: server.map.friendly,
started_at: beginning,
last_received_at: iso(now),
status: 'active',
start_observed: 1,
start_players: server.players,
highest_round: server.round,
last_event: 'heartbeat',
last_fingerprint: `fixture-live-${index}`
});
for (const p of roster) {
rememberPlayer.run(server.game, p.guid, p.name, beginning, iso(now));
insert('leaderboard_match_players', { match_id: matchId, guid: p.guid, highest_round: server.round, rounds_survived: server.round,
kills: p.kills, headshots: p.headshots, downs: p.downs, revives: p.revives, best_score: p.score,
last_round: server.round, present: 1, has_stats: 1, completed: 0, first_seen_at: beginning, last_seen_at: iso(now),
first_observed_round: 1, present_at_start: 1 });
insert('leaderboard_match_players', {
match_id: matchId,
guid: p.guid,
highest_round: server.round,
rounds_survived: server.round,
kills: p.kills,
headshots: p.headshots,
downs: p.downs,
revives: p.revives,
best_score: p.score,
last_round: server.round,
present: 1,
has_stats: 1,
completed: 0,
first_seen_at: beginning,
last_seen_at: iso(now),
first_observed_round: 1,
present_at_start: 1
});
}
}
}
for (const phase of ['startup', 'running']) {
for (let minute = 0; minute < 60; minute++) insert('ec_instance_resource_samples', {
host_id: server.host, instance_id: server.instance, generation: 1, phase, bucket: iso(now - minute * MINUTE),
for (let minute = 0; minute < 60; minute++)
insert('ec_instance_resource_samples', {
host_id: server.host,
instance_id: server.instance,
generation: 1,
phase,
bucket: iso(now - minute * MINUTE),
cpu_cores: phase === 'startup' ? 0.7 + index / 40 : 0.25 + index / 50,
memory_bytes: Math.round((phase === 'startup' ? 1.1 : 0.7) * GIB) });
memory_bytes: Math.round((phase === 'startup' ? 1.1 : 0.7) * GIB)
});
}
}
for (let index = 0; index < 494; index++) {
// A large Kino duo cohort exercises ranking and profile pagination.
const server = fixtureServers[index < 64 ? 0 : index % fixtureServers.length];
const squad = index < 64 ? 2 : Math.floor(index / 12) % 4 + 1;
const squad = index < 64 ? 2 : (Math.floor(index / 12) % 4) + 1;
const unknownStart = index >= 64 && index % 37 === 0;
const interrupted = index >= 64 && index % 31 === 0;
const end = now - (180 + index * 37) * MINUTE;
const beginning = end - (30 + index % 100) * MINUTE;
const round = 10 + index * 7 % 56;
insert('leaderboard_matches', { id: index + 1, server: server.name, game: server.game, map: server.map.canonical,
map_key: server.slug, map_name: server.map.friendly, started_at: iso(beginning), ended_at: iso(end), last_received_at: iso(end),
status: interrupted ? 'interrupted' : 'completed', start_observed: unknownStart ? 0 : 1, start_players: unknownStart ? null : squad,
highest_round: round, last_event: interrupted ? 'interrupted' : 'game_end', last_fingerprint: `fixture-match-${index + 1}` });
const beginning = end - (30 + (index % 100)) * MINUTE;
const round = 10 + ((index * 7) % 56);
insert('leaderboard_matches', {
id: index + 1,
server: server.name,
game: server.game,
map: server.map.canonical,
map_key: server.slug,
map_name: server.map.friendly,
started_at: iso(beginning),
ended_at: iso(end),
last_received_at: iso(end),
status: interrupted ? 'interrupted' : 'completed',
start_observed: unknownStart ? 0 : 1,
start_players: unknownStart ? null : squad,
highest_round: round,
last_event: interrupted ? 'interrupted' : 'game_end',
last_fingerprint: `fixture-match-${index + 1}`
});
for (let slot = 0; slot < squad; slot++) {
const p = player(slot === 0 ? 0 : (index + slot * 19) % 80 + 1);
const p = player(slot === 0 ? 0 : ((index + slot * 19) % 80) + 1);
rememberPlayer.run(server.game, p.guid, p.name, iso(beginning), iso(end));
insert('leaderboard_match_players', { match_id: index + 1, guid: p.guid, highest_round: round, rounds_survived: round - (unknownStart ? 4 : 0),
kills: round * 35 + slot * 71, headshots: round * 9, downs: index % 4, revives: slot + index % 3,
best_score: round * 2100 + slot * 1000, last_round: round, present: slot > 0 && index % 9 === 0 ? 0 : 1, has_stats: 1,
completed: !interrupted && !(slot > 0 && index % 9 === 0) ? 1 : 0, first_seen_at: iso(beginning), last_seen_at: iso(end),
first_observed_round: unknownStart ? 5 : 1, present_at_start: unknownStart ? null : 1 });
insert('leaderboard_match_players', {
match_id: index + 1,
guid: p.guid,
highest_round: round,
rounds_survived: round - (unknownStart ? 4 : 0),
kills: round * 35 + slot * 71,
headshots: round * 9,
downs: index % 4,
revives: slot + (index % 3),
best_score: round * 2100 + slot * 1000,
last_round: round,
present: slot > 0 && index % 9 === 0 ? 0 : 1,
has_stats: 1,
completed: !interrupted && !(slot > 0 && index % 9 === 0) ? 1 : 0,
first_seen_at: iso(beginning),
last_seen_at: iso(end),
first_observed_round: unknownStart ? 5 : 1,
present_at_start: unknownStart ? null : 1
});
}
}
const metric = handle.prepare('INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,?,?,?,?,4)');
const metric = handle.prepare(
'INSERT INTO metric_samples(at,server,game,map,players,capacity) VALUES(?,?,?,?,?,4)'
);
for (let minute = 0; minute < 7 * 1440; minute++) {
for (const [index, server] of fixtureServers.entries()) {
const count = Math.floor((Math.sin(minute / 90 + index) + 1) * 2.49);
metric.run(now - minute * MINUTE, server.name, server.game, server.map.friendly,
minute % 720 < 8 && index === 5 ? null : count);
metric.run(
now - minute * MINUTE,
server.name,
server.game,
server.map.friendly,
minute % 720 < 8 && index === 5 ? null : count
);
}
}
for (const scope of ['*', ...[1, 2, 3].map(index => `fixture-host-${index}`)]) for (const phase of ['startup', 'running']) {
insert('ec_resource_allowances', { scope, phase, cpu_cores: phase === 'startup' ? 1 : 0.5,
memory_bytes: Math.round((phase === 'startup' ? 1.2 : 0.8) * GIB), samples: 240,
window_start: iso(now - DAY), observed_at: iso(now), computed_at: iso(now), usage_margin: 1 });
for (const scope of ['*', ...[1, 2, 3].map((index) => `fixture-host-${index}`)])
for (const phase of ['startup', 'running']) {
insert('ec_resource_allowances', {
scope,
phase,
cpu_cores: phase === 'startup' ? 1 : 0.5,
memory_bytes: Math.round((phase === 'startup' ? 1.2 : 0.8) * GIB),
samples: 240,
window_start: iso(now - DAY),
observed_at: iso(now),
computed_at: iso(now),
usage_margin: 1
});
}
// Real artifact blobs, so details and downloads work without game files.
for (const [index, kind] of ['script', 'baseline', 'config'].entries()) {
const content = Buffer.from(kind === 'script' ? '// Synthetic UI fixture; no gameplay behavior.\n' : '// Synthetic UI fixture configuration.\nset sv_maxclients 4\n');
const content = Buffer.from(
kind === 'script'
? '// Synthetic UI fixture; no gameplay behavior.\n'
: '// Synthetic UI fixture configuration.\nset sv_maxclients 4\n'
);
const hash = createHash('sha256').update(content).digest('hex');
handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(hash, content.length, content);
insert('scripts', { id: index + 1, name: ['Fixture telemetry', 'Fixture Black Ops I baseline', 'Fixture lobby settings'][index],
target_rel_path: index === 0 ? 'fixture.gsc' : index === 1 ? 'dedicated_sp.cfg' : 'fixture.cfg',
scope_type: 'game', scope_value: 't5', current_version_id: index + 1, kind, created_at: started });
insert('script_versions', { id: index + 1, script_id: index + 1, version: 1, sha256: hash, content_hash: hash,
size: content.length, notes: 'Synthetic preview content.', state: 'tested', kind, uploaded_at: started });
handle
.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)')
.run(hash, content.length, content);
insert('scripts', {
id: index + 1,
name: [
'Fixture telemetry',
'Fixture Black Ops I baseline',
'Fixture lobby settings'
][index],
target_rel_path:
index === 0 ? 'fixture.gsc' : index === 1 ? 'dedicated_sp.cfg' : 'fixture.cfg',
scope_type: 'game',
scope_value: 't5',
current_version_id: index + 1,
kind,
created_at: started
});
insert('script_versions', {
id: index + 1,
script_id: index + 1,
version: 1,
sha256: hash,
content_hash: hash,
size: content.length,
notes: 'Synthetic preview content.',
state: 'tested',
kind,
uploaded_at: started
});
}
for (let index = 0; index < 36; index++) {
const failed = index % 7 === 0;
const server = fixtureServers[failed ? 5 : index % fixtureServers.length];
const id = `fixture-operation-${index + 1}`;
const state = failed ? 'failed' : index % 11 === 0 ? 'cancelled' : 'done';
insert('server_deployments', { id, server: server.name, action: ['restart', 'start', 'rebuild', 'stop'][index % 4], state,
stage: failed ? 'Verification failed' : state === 'done' ? 'Completed' : 'Cancelled before dispatch',
started_at: iso(now - (60 + index * 2) * MINUTE), finished_at: iso(now - (59 + index * 2) * MINUTE), triggered_by: 'Fixture operator',
error: failed ? 'Synthetic verification timeout. No host command was sent.' : null, warnings: '[]' });
insert('native_provisioning', { deployment_id: id, instance_id: server.instance, generation: 1 });
insert('server_deployment_events', { deployment_id: id, created_at: iso(now - (59 + index * 2) * MINUTE), message: 'Synthetic retained history; no command was sent.' });
insert('server_deployments', {
id,
server: server.name,
action: ['restart', 'start', 'rebuild', 'stop'][index % 4],
state,
stage: failed
? 'Verification failed'
: state === 'done'
? 'Completed'
: 'Cancelled before dispatch',
started_at: iso(now - (60 + index * 2) * MINUTE),
finished_at: iso(now - (59 + index * 2) * MINUTE),
triggered_by: 'Fixture operator',
error: failed ? 'Synthetic verification timeout. No host command was sent.' : null,
warnings: '[]'
});
insert('native_provisioning', {
deployment_id: id,
instance_id: server.instance,
generation: 1
});
insert('server_deployment_events', {
deployment_id: id,
created_at: iso(now - (59 + index * 2) * MINUTE),
message: 'Synthetic retained history; no command was sent.'
});
}
for (let index = 0; index < 8; index++) {
const server = fixtureServers[index % fixtureServers.length];
insert('ec_host_operations', { id: `fixture-host-command-${index + 1}`, host_id: server.host, instance_id: server.instance,
server_name: server.name, kind: index % 2 ? 'instance.inspect' : 'instance.restart', payload: '{}',
state: index % 3 ? 'completed' : 'failed', result: '{}', error: index % 3 ? null : 'Synthetic command failure.',
created_at: iso(now - (90 + index * 3) * MINUTE), updated_at: iso(now - (89 + index * 3) * MINUTE), source: 'Fixture operator' });
insert('ec_host_operations', {
id: `fixture-host-command-${index + 1}`,
host_id: server.host,
instance_id: server.instance,
server_name: server.name,
kind: index % 2 ? 'instance.inspect' : 'instance.restart',
payload: '{}',
state: index % 3 ? 'completed' : 'failed',
result: '{}',
error: index % 3 ? null : 'Synthetic command failure.',
created_at: iso(now - (90 + index * 3) * MINUTE),
updated_at: iso(now - (89 + index * 3) * MINUTE),
source: 'Fixture operator'
});
}
for (let index = 0; index < 3; index++) {
const server = fixtureServers[index === 0 ? 1 : index === 1 ? 5 : index];
insert('maintenance_jobs', { id: index + 1, server: server.name, state: index === 1 ? 'failed' : 'done', target_revision: 4878,
requested_at: iso(now - (120 + index * 4) * MINUTE), finished_at: iso(now - (119 + index * 4) * MINUTE),
detail: index === 1 ? 'Synthetic update could not be verified.' : 'Synthetic scheduled restart completed.', kind: 'schedule', origin: 'manual',
provisioning_id: index === 0 ? 'fixture-operation-2' : null });
insert('watchdog_incidents', { id: `fixture-recovery-${index + 1}`, server: server.name, kind: 'health', state: 'resolved',
summary: 'Synthetic recovery finding resolved.', details: '{}', first_seen_at: iso(now - (130 + index * 4) * MINUTE),
last_seen_at: iso(now - (129 + index * 4) * MINUTE), finished_at: iso(now - (129 + index * 4) * MINUTE) });
insert('deployments', { id: `fixture-campaign-${index + 1}`, action: 'drift', started_at: iso(now - (140 + index * 4) * MINUTE),
finished_at: iso(now - (139 + index * 4) * MINUTE), triggered_by: 'Fixture operator' });
insert('deployment_servers', { deployment_id: `fixture-campaign-${index + 1}`, server: server.name, files_changed: 1, restart_state: 'done', players: 0 });
insert('maintenance_jobs', {
id: index + 1,
server: server.name,
state: index === 1 ? 'failed' : 'done',
target_revision: 4878,
requested_at: iso(now - (120 + index * 4) * MINUTE),
finished_at: iso(now - (119 + index * 4) * MINUTE),
detail:
index === 1
? 'Synthetic update could not be verified.'
: 'Synthetic scheduled restart completed.',
kind: 'schedule',
origin: 'manual',
provisioning_id: index === 0 ? 'fixture-operation-2' : null
});
insert('watchdog_incidents', {
id: `fixture-recovery-${index + 1}`,
server: server.name,
kind: 'health',
state: 'resolved',
summary: 'Synthetic recovery finding resolved.',
details: '{}',
first_seen_at: iso(now - (130 + index * 4) * MINUTE),
last_seen_at: iso(now - (129 + index * 4) * MINUTE),
finished_at: iso(now - (129 + index * 4) * MINUTE)
});
insert('deployments', {
id: `fixture-campaign-${index + 1}`,
action: 'drift',
started_at: iso(now - (140 + index * 4) * MINUTE),
finished_at: iso(now - (139 + index * 4) * MINUTE),
triggered_by: 'Fixture operator'
});
insert('deployment_servers', {
deployment_id: `fixture-campaign-${index + 1}`,
server: server.name,
files_changed: 1,
restart_state: 'done',
players: 0
});
}
for (let index = 0; index < 2; index++) {
insert('betas', { id: index + 1, script_id: 1, script_version_id: 1, trigger_kind: 'ui', state: index ? 'rolled-back' : 'promoted',
opened_at: iso(now - (150 + index * 3) * MINUTE), closed_at: iso(now - (149 + index * 3) * MINUTE), notes: 'Synthetic completed beta.' });
insert('beta_scripts', { beta_id: index + 1, script_id: 1, script_version_id: 1, kind: 'script' });
insert('beta_servers', { beta_id: index + 1, server: fixtureServers[index].name, had_original: 0, stage: 'ready', updated_at: iso(now - (149 + index * 3) * MINUTE) });
insert('betas', {
id: index + 1,
script_id: 1,
script_version_id: 1,
trigger_kind: 'ui',
state: index ? 'rolled-back' : 'promoted',
opened_at: iso(now - (150 + index * 3) * MINUTE),
closed_at: iso(now - (149 + index * 3) * MINUTE),
notes: 'Synthetic completed beta.'
});
insert('beta_scripts', {
beta_id: index + 1,
script_id: 1,
script_version_id: 1,
kind: 'script'
});
insert('beta_servers', {
beta_id: index + 1,
server: fixtureServers[index].name,
had_original: 0,
stage: 'ready',
updated_at: iso(now - (149 + index * 3) * MINUTE)
});
}
insert('provisioning_sources', { operation_id: 'fixture-operation-2', kind: 'maintenance', source_id: '1', server: fixtureServers[1].name });
insert('provisioning_sources', { operation_id: 'fixture-operation-3', kind: 'autoscaling', source_id: 'fixture-paused-plan', server: fixtureServers[2].name });
insert('provisioning_sources', {
operation_id: 'fixture-operation-2',
kind: 'maintenance',
source_id: '1',
server: fixtureServers[1].name
});
insert('provisioning_sources', {
operation_id: 'fixture-operation-3',
kind: 'autoscaling',
source_id: 'fixture-paused-plan',
server: fixtureServers[2].name
});
})();
}
/** Refreshes existing synthetic observations only; never queues work or writes real files. */
export function refreshFixtureData(handle: Database.Database, now: number): void {
const marker = handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY) as { value: string } | undefined;
if (marker?.value !== FIXTURE_VERSION) throw new Error('Refusing to refresh an unmarked database.');
const marker = handle.prepare('SELECT value FROM settings WHERE key=?').get(FIXTURE_KEY) as
{ value: string } | undefined;
if (marker?.value !== FIXTURE_VERSION)
throw new Error('Refusing to refresh an unmarked database.');
if (!Number.isFinite(now)) throw new Error('Invalid fixture time.');
handle.transaction(() => {
for (const [index, server] of fixtureServers.entries()) {
handle.prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,container_checked_at=?,container_state=?,
last_players=?,last_map=?,last_roster=?,last_error=?,rcon_failures=?,last_reply_snippet=NULL WHERE name=?`)
.run(seenAt(server, now), iso(now), iso(now), containerState(server), server.players, server.map.canonical,
JSON.stringify(rosterFor(server, index)), server.state === 'stale' ? 'timeout: no reply from game server' : null,
server.state === 'stale' ? 2 : 0, server.name);
handle.prepare(`UPDATE server_telemetry SET received_at=?,payload=json_set(payload,'$.receivedAt',?) WHERE server=?`).run(iso(now), iso(now), server.name);
handle.prepare("UPDATE leaderboard_matches SET last_received_at=? WHERE server=? AND status='active' AND id BETWEEN 495 AND 500").run(iso(now), server.name);
handle.prepare('UPDATE plutonium_versions SET checked_at=? WHERE server=?').run(iso(now), server.name);
handle
.prepare(
`UPDATE servers SET last_seen_at=?,last_checked_at=?,container_checked_at=?,container_state=?,
last_players=?,last_map=?,last_roster=?,last_error=?,rcon_failures=?,last_reply_snippet=NULL WHERE name=?`
)
.run(
seenAt(server, now),
iso(now),
iso(now),
containerState(server),
server.players,
server.map.canonical,
JSON.stringify(rosterFor(server, index)),
server.state === 'stale' ? 'timeout: no reply from game server' : null,
server.state === 'stale' ? 2 : 0,
server.name
);
handle
.prepare(
`UPDATE server_telemetry SET received_at=?,payload=json_set(payload,'$.receivedAt',?) WHERE server=?`
)
.run(iso(now), iso(now), server.name);
handle
.prepare(
"UPDATE leaderboard_matches SET last_received_at=? WHERE server=? AND status='active' AND id BETWEEN 495 AND 500"
)
.run(iso(now), server.name);
handle
.prepare('UPDATE plutonium_versions SET checked_at=? WHERE server=?')
.run(iso(now), server.name);
}
for (let index = 1; index <= 3; index++) {
const id = `fixture-host-${index}`;
handle.prepare("UPDATE ec_hosts SET last_heartbeat=?,inventory=json_set(inventory,'$.instanceStates',json(?)) WHERE id=?")
.run(iso(now - (index === 3 ? 10 * MINUTE : 0)), JSON.stringify(instanceStates(id, now)), id);
handle
.prepare(
"UPDATE ec_hosts SET last_heartbeat=?,inventory=json_set(inventory,'$.instanceStates',json(?)) WHERE id=?"
)
.run(
iso(now - (index === 3 ? 10 * MINUTE : 0)),
JSON.stringify(instanceStates(id, now)),
id
);
}
})();
}
scripts/fixture-options.ts
+36 -11
View File
@@ -1,29 +1,49 @@
import { mkdir, open, realpath, rm } from 'node:fs/promises';
import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
export interface FixtureOptions { dbPath: string; now: number; watch: boolean }
export interface FixtureOptions {
dbPath: string;
now: number;
watch: boolean;
}
export function parseFixtureOptions(args: string[], clock = Date.now()): FixtureOptions {
let dbPath = '', now = clock, watch = false, explicitTime = false;
let dbPath = '',
now = clock,
watch = false,
explicitTime = false;
const seen = new Set<string>();
for (let index = 0; index < args.length; index++) {
const option = args[index];
if (seen.has(option)) throw new Error(`Duplicate option: ${option}`);
seen.add(option);
if (option === '--watch') { watch = true; continue; }
if (option === '--watch') {
watch = true;
continue;
}
if (option !== '--db' && option !== '--now') throw new Error(`Unknown option: ${option}`);
const value = args[++index];
if (!value || value.startsWith('--')) throw new Error(`Missing value for ${option}.`);
if (option === '--db') dbPath = value;
else {
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(value)
|| !Number.isFinite(Date.parse(value))) throw new Error('--now must be an ISO timestamp with a timezone.');
now = Date.parse(value); explicitTime = true;
if (
!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(
value
) ||
!Number.isFinite(Date.parse(value))
)
throw new Error('--now must be an ISO timestamp with a timezone.');
now = Date.parse(value);
explicitTime = true;
}
}
if (!dbPath || !isAbsolute(dbPath)) throw new Error('Provide --db with an absolute path inside a new fixture directory. DB_PATH is never used.');
if (!dbPath || !isAbsolute(dbPath))
throw new Error(
'Provide --db with an absolute path inside a new fixture directory. DB_PATH is never used.'
);
if (!Number.isFinite(now)) throw new Error('Invalid fixture clock.');
if (watch && explicitTime) throw new Error('Use --watch for live time or --now for a fixed snapshot, separately.');
if (watch && explicitTime)
throw new Error('Use --watch for live time or --now for a fixed snapshot, separately.');
return { dbPath: resolve(dbPath), now, watch };
}
@@ -36,9 +56,14 @@ export async function reserveFixture(dbPath: string): Promise<{ root: string; db
// but never follow a pre-existing fixture directory or database symlink.
const parent = await realpath(dirname(requestedRoot));
const root = join(parent, basename(requestedRoot));
try { await mkdir(root, { mode: 0o700 }); }
catch (cause) {
if ((cause as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('Fixture directory already exists. Choose a new directory; existing data is never overwritten.');
try {
await mkdir(root, { mode: 0o700 });
} catch (cause) {
if ((cause as NodeJS.ErrnoException).code === 'EEXIST')
throw new Error(
'Fixture directory already exists. Choose a new directory; existing data is never overwritten.',
{ cause }
);
throw cause;
}
const target = join(root, basename(requested));
scripts/reset-admin.mjs
+46 -17
View File
@@ -18,38 +18,67 @@ function secret(prompt) {
process.stdout.write('\n');
};
const listener = (text, key) => {
if (key?.ctrl && key.name === 'c') { cleanup(); reject(new Error('Cancelled.')); }
else if (key?.name === 'return' || key?.name === 'enter') { cleanup(); resolve(value); }
else if (key?.name === 'backspace') value = [...value].slice(0,-1).join('');
else if (text && !key?.ctrl && !key?.meta && !/[\x00-\x1f\x7f]/.test(text) && [...value].length < 128) value += text;
if (key?.ctrl && key.name === 'c') {
cleanup();
reject(new Error('Cancelled.'));
} else if (key?.name === 'return' || key?.name === 'enter') {
cleanup();
resolve(value);
} else if (key?.name === 'backspace') value = [...value].slice(0, -1).join('');
else if (
text &&
!key?.ctrl &&
!key?.meta &&
// eslint-disable-next-line no-control-regex -- Reject terminal control characters while collecting password input.
!/[\x00-\x1f\x7f]/.test(text) &&
[...value].length < 128
)
value += text;
};
process.stdin.on('keypress',listener);
process.stdin.on('keypress', listener);
});
}
try {
const username = process.argv[2]?.toLowerCase();
if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username)) throw new Error('Usage: node scripts/reset-admin.mjs <administrator-username>');
if (!process.stdin.isTTY) throw new Error('Run this command in an interactive terminal. Passwords are never accepted as command arguments.');
const database = new Database(process.env.DB_PATH || './data/cherry.db',{fileMustExist:true});
if (process.argv.length !== 3 || !username || !/^[a-z0-9][a-z0-9_.-]{2,63}$/.test(username))
throw new Error('Usage: node scripts/reset-admin.mjs <administrator-username>');
if (!process.stdin.isTTY)
throw new Error(
'Run this command in an interactive terminal. Passwords are never accepted as command arguments.'
);
const database = new Database(process.env.DB_PATH || './data/cherry.db', {
fileMustExist: true
});
database.pragma('busy_timeout=5000');
const user = database.prepare("SELECT id FROM auth_users WHERE username=? AND role='admin' AND disabled=0").get(username);
const user = database
.prepare("SELECT id FROM auth_users WHERE username=? AND role='admin' AND disabled=0")
.get(username);
if (!user) throw new Error('An enabled administrator with that username was not found.');
const password = await secret('New administrator password (8–128 characters): ');
const confirmation = await secret('Repeat password: ');
if (password !== confirmation) throw new Error('Passwords did not match.');
const passwordHash = await hashPassword(password);
database.transaction(() => {
const changed = database.prepare("UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0")
.run(passwordHash,Date.now(),user.id);
if (changed.changes !== 1) throw new Error('Administrator changed during recovery. Try again.');
database
.transaction(() => {
const changed = database
.prepare(
"UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=? AND role='admin' AND disabled=0"
)
.run(passwordHash, Date.now(), user.id);
if (changed.changes !== 1)
throw new Error('Administrator changed during recovery. Try again.');
database.prepare('DELETE FROM auth_sessions WHERE user_id=?').run(user.id);
database.prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)')
.run('password.recovered',user.id,'{"source":"local-cli"}',Date.now());
}).immediate();
database
.prepare(
'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(NULL,?,?,?,?)'
)
.run('password.recovered', user.id, '{"source":"local-cli"}', Date.now());
})
.immediate();
database.close();
console.log('Administrator password reset. Existing sessions have been revoked.');
} catch (cause) {
console.error(cause instanceof Error ? cause.message : 'Administrator recovery failed.');
process.exitCode=1;
process.exitCode = 1;
}
scripts/seed-fixture.ts
+127 -34
View File
@@ -5,7 +5,13 @@ import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createServer, type ViteDevServer } from 'vite';
import type Database from 'better-sqlite3';
import { FIXTURE_KEY, FIXTURE_VERSION, fixtureServers, refreshFixtureData, seedFixtureData } from './fixture-data.js';
import {
FIXTURE_KEY,
FIXTURE_VERSION,
fixtureServers,
refreshFixtureData,
seedFixtureData
} from './fixture-data.js';
import { parseFixtureOptions, reserveFixture } from './fixture-options.js';
const help = `Create an isolated synthetic UI preview database.
@@ -22,7 +28,10 @@ No web server, worker, game agent or network request is started by this command.
async function main() {
const args = process.argv.slice(2);
if (args.length === 1 && ['--help', '-h'].includes(args[0])) { console.log(help); return; }
if (args.length === 1 && ['--help', '-h'].includes(args[0])) {
console.log(help);
return;
}
const options = parseFixtureOptions(args);
process.umask(0o077);
const destination = await reserveFixture(options.dbPath);
@@ -30,70 +39,154 @@ async function main() {
let closeDb: (() => void) | undefined;
let complete = false;
try {
const paths = { DB_PATH: destination.dbPath, SERVERS_ROOT: join(destination.root, 't5'), T6_SERVERS_ROOT: join(destination.root, 't6'),
BETA_ARCHIVE_DIR: join(destination.root, 'beta-archive'), GAME_FILES_ROOT: join(destination.root, 'game-files'), SQLITE_TMPDIR: join(destination.root, 'sqlite-tmp') };
const paths = {
DB_PATH: destination.dbPath,
SERVERS_ROOT: join(destination.root, 't5'),
T6_SERVERS_ROOT: join(destination.root, 't6'),
BETA_ARCHIVE_DIR: join(destination.root, 'beta-archive'),
GAME_FILES_ROOT: join(destination.root, 'game-files'),
SQLITE_TMPDIR: join(destination.root, 'sqlite-tmp')
};
for (const [key, value] of Object.entries(paths)) {
process.env[key] = value;
if (key !== 'DB_PATH') await mkdir(value, { mode: 0o700 });
}
Object.assign(process.env, { RCON_PASSWORD: '', RCON_HOST: '127.0.0.1', EC_AUTH_ALLOW_HTTP: 'true',
DEPLOY_KEEP_DAYS: '7', DEPLOYMENT_RESTART_CONCURRENCY: '1', DEPLOYMENT_CREATE_CONCURRENCY: '1',
EC_SQLITE_MMAP_MB: '0', PLUTAINER_IMAGE: 'fixture.invalid/electric-cherry/preview:local' });
Object.assign(process.env, {
RCON_PASSWORD: '',
RCON_HOST: '127.0.0.1',
EC_AUTH_ALLOW_HTTP: 'true',
DEPLOY_KEEP_DAYS: '7',
DEPLOYMENT_RESTART_CONCURRENCY: '1',
DEPLOYMENT_CREATE_CONCURRENCY: '1',
EC_SQLITE_MMAP_MB: '0',
PLUTAINER_IMAGE: 'fixture.invalid/electric-cherry/preview:local'
});
const repository = resolve(dirname(fileURLToPath(import.meta.url)), '..');
vite = await createServer({ root: repository, cacheDir: join(destination.root, 'vite-cache'), configFile: false, envFile: false, appType: 'custom',
vite = await createServer({
root: repository,
cacheDir: join(destination.root, 'vite-cache'),
configFile: false,
envFile: false,
appType: 'custom',
// Vite 5 otherwise opens its HMR WebSocket port even with hmr:false.
// Attach it to an HTTP server that is deliberately never listened on.
server: { middlewareMode: true, hmr: { server: createHttpServer() }, watch: null }, optimizeDeps: { noDiscovery: true },
resolve: { alias: { $lib: join(repository, 'src/lib'), $ui: join(repository, 'src/lib/ui') } },
logLevel: 'error' });
server: { middlewareMode: true, hmr: { server: createHttpServer() }, watch: null },
optimizeDeps: { noDiscovery: true },
resolve: {
alias: { $lib: join(repository, 'src/lib'), $ui: join(repository, 'src/lib/ui') }
},
logLevel: 'error'
});
// Load the real migrations through Vite's ?raw support. Importing hooks
// would start queues and agents; this script never loads the app router.
const database = await vite.ssrLoadModule('/src/lib/server/db/index.ts') as { db: () => Database.Database; closeDb: () => void };
const database = (await vite.ssrLoadModule('/src/lib/server/db/index.ts')) as {
db: () => Database.Database;
closeDb: () => void;
};
closeDb = database.closeDb;
const handle = database.db();
const hosts = await vite.ssrLoadModule('/src/lib/server/hosts/schema.ts');
const auth = await vite.ssrLoadModule('/src/lib/server/auth/schema.ts');
const files = await vite.ssrLoadModule('/src/lib/server/game-files/schema.ts');
hosts.ensureHostSchema(handle); auth.ensureAuthSchema(handle); files.ensureGameFileSchema(handle);
hosts.ensureHostSchema(handle);
auth.ensureAuthSchema(handle);
files.ensureGameFileSchema(handle);
seedFixtureData(handle, options.now);
const password = randomBytes(24).toString('base64url');
const { hashPassword } = await vite.ssrLoadModule('/src/lib/server/auth/passwords.js');
handle.prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,created_at,updated_at)
VALUES('fixture-admin','demo','Demo operator',?,'admin',?,?)`).run(await hashPassword(password), options.now, options.now);
if ((handle.pragma('foreign_key_check') as unknown[]).length || handle.pragma('integrity_check', { simple: true }) !== 'ok') {
handle
.prepare(
`INSERT INTO auth_users(id,username,display_name,password_hash,role,created_at,updated_at)
VALUES('fixture-admin','demo','Demo operator',?,'admin',?,?)`
)
.run(await hashPassword(password), options.now, options.now);
if (
(handle.pragma('foreign_key_check') as unknown[]).length ||
handle.pragma('integrity_check', { simple: true }) !== 'ok'
) {
throw new Error('Fixture integrity validation failed.');
}
const environment = { ...paths, RCON_PASSWORD: '', RCON_HOST: '127.0.0.1', EC_AUTH_ALLOW_HTTP: 'true',
PLUTAINER_IMAGE: process.env.PLUTAINER_IMAGE! };
const environment = {
...paths,
RCON_PASSWORD: '',
RCON_HOST: '127.0.0.1',
EC_AUTH_ALLOW_HTTP: 'true',
PLUTAINER_IMAGE: process.env.PLUTAINER_IMAGE!
};
const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
await writeFile(join(destination.root, 'preview.env'), Object.entries(environment).map(([key, value]) => `export ${key}=${shellQuote(value)}`).join('\n') + '\n', { flag: 'wx', mode: 0o600 });
await writeFile(join(destination.root, 'fixture.json'), JSON.stringify({ format: FIXTURE_VERSION, marker: FIXTURE_KEY,
dbPath: destination.dbPath, createdAt: new Date(options.now).toISOString(), username: 'demo', password,
hosts: 3, servers: fixtureServers.map(server => server.name), matches: 500,
note: 'Synthetic local data only. Host credentials cannot authenticate; game images and library IDs are fictitious.' }, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
await writeFile(
join(destination.root, 'preview.env'),
Object.entries(environment)
.map(([key, value]) => `export ${key}=${shellQuote(value)}`)
.join('\n') + '\n',
{ flag: 'wx', mode: 0o600 }
);
await writeFile(
join(destination.root, 'fixture.json'),
JSON.stringify(
{
format: FIXTURE_VERSION,
marker: FIXTURE_KEY,
dbPath: destination.dbPath,
createdAt: new Date(options.now).toISOString(),
username: 'demo',
password,
hosts: 3,
servers: fixtureServers.map((server) => server.name),
matches: 500,
note: 'Synthetic local data only. Host credentials cannot authenticate; game images and library IDs are fictitious.'
},
null,
2
) + '\n',
{ flag: 'wx', mode: 0o600 }
);
handle.pragma('wal_checkpoint(TRUNCATE)');
complete = true;
console.log(`Created ${destination.dbPath}\n3 hosts · 12 servers · 500 matches · 7 days of metrics\nLocal sign-in details: ${join(destination.root, 'fixture.json')}\nPreview environment: ${join(destination.root, 'preview.env')}`);
console.log(
`Created ${destination.dbPath}\n3 hosts · 12 servers · 500 matches · 7 days of metrics\nLocal sign-in details: ${join(destination.root, 'fixture.json')}\nPreview environment: ${join(destination.root, 'preview.env')}`
);
if (options.watch) {
refreshFixtureData(handle, Date.now());
console.log('Refreshing synthetic observations every 15 seconds. Leave this terminal open; Ctrl+C stops the feeder.');
console.log(
'Refreshing synthetic observations every 15 seconds. Leave this terminal open; Ctrl+C stops the feeder.'
);
await new Promise<void>((resolvePromise, reject) => {
const finish = (error?: unknown) => { clearInterval(timer); process.off('SIGINT', stop); process.off('SIGTERM', stop); if (error) reject(error); else resolvePromise(); };
const finish = (error?: unknown) => {
clearInterval(timer);
process.off('SIGINT', stop);
process.off('SIGTERM', stop);
if (error) reject(error);
else resolvePromise();
};
const stop = () => finish();
const timer = setInterval(() => { try { refreshFixtureData(handle, Date.now()); } catch (error) { finish(error); } }, 15_000);
process.on('SIGINT', stop); process.on('SIGTERM', stop);
const timer = setInterval(() => {
try {
refreshFixtureData(handle, Date.now());
} catch (error) {
finish(error);
}
}, 15_000);
process.on('SIGINT', stop);
process.on('SIGTERM', stop);
});
}
} finally {
try { closeDb?.(); }
finally {
try { await vite?.close(); }
finally { if (!complete) await rm(destination.root, { recursive: true, force: true }); }
try {
closeDb?.();
} finally {
try {
await vite?.close();
} finally {
if (!complete) await rm(destination.root, { recursive: true, force: true });
}
}
}
}
main().catch(error => {
console.error(`Fixture creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
main().catch((error) => {
console.error(
`Fixture creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`
);
process.exitCode = 1;
});
src/hooks.server.ts
+24 -6
View File
@@ -13,7 +13,7 @@ let rejectionLoggerRegistered = false;
function boot(): void {
if (booted) return;
if (!rejectionLoggerRegistered) {
process.on('unhandledRejection', reason => {
process.on('unhandledRejection', (reason) => {
console.error('[process] unhandled rejection:', reason);
});
rejectionLoggerRegistered = true;
@@ -30,13 +30,31 @@ function boot(): void {
export const handle: Handle = async (input) => {
boot();
const started = performance.now();
const response = await withHostRequestContext(() => withOperationSource(`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
() => authenticationHandle(input)));
const response = await withHostRequestContext(() =>
withOperationSource(
`${input.event.request.method} ${input.event.route.id ?? 'Web request'}`,
() => authenticationHandle(input)
)
);
const elapsed = performance.now() - started;
response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {
console.info('[request]', { route: input.event.route.id, method: input.event.request.method, status: response.status, elapsedMs: Math.round(elapsed) });
if (
process.env.EC_PERF_LOG === 'true' &&
elapsed > 250 &&
!input.event.route.id?.startsWith('/api/agent/')
) {
console.info('[request]', {
route: input.event.route.id,
method: input.event.request.method,
status: response.status,
elapsedMs: Math.round(elapsed)
});
}
if (input.event.locals.user && !['GET','HEAD','OPTIONS'].includes(input.event.request.method) && response.status < 400) invalidateFleet();
if (
input.event.locals.user &&
!['GET', 'HEAD', 'OPTIONS'].includes(input.event.request.method) &&
response.status < 400
)
invalidateFleet();
return response;
};
src/lib/activity.ts
+38 -11
View File
@@ -1,17 +1,37 @@
export const activityTypes = {
create: 'Create server', start: 'Start server', restart: 'Restart server', rebuild: 'Rebuild server',
stop: 'Stop server', remove: 'Destroy server', update: 'Game update', files: 'File changes',
health: 'Health finding', drift: 'File drift', host: 'Host operation', beta: 'Beta test'
create: 'Create server',
start: 'Start server',
restart: 'Restart server',
rebuild: 'Rebuild server',
stop: 'Stop server',
remove: 'Destroy server',
update: 'Game update',
files: 'File changes',
health: 'Health finding',
drift: 'File drift',
host: 'Host operation',
beta: 'Beta test'
} as const;
export type ActivityType = keyof typeof activityTypes;
export const activitySources = {
provisioning: 'Server operations', maintenance: 'Maintenance & recovery', recovery: 'Recovery',
campaign: 'Campaigns', beta: 'Betas', host: 'Hosts', autoscaling: 'Autoscaling'
provisioning: 'Server operations',
maintenance: 'Maintenance & recovery',
recovery: 'Recovery',
campaign: 'Campaigns',
beta: 'Betas',
host: 'Hosts',
autoscaling: 'Autoscaling'
} as const;
export type ActivitySource = keyof typeof activitySources;
export const activityStates = {
queued: 'Queued', running: 'In progress', succeeded: 'Completed', warning: 'Completed with warnings',
failed: 'Failed', cancelled: 'Cancelled', attention: 'Needs attention', unknown: 'Unknown'
queued: 'Queued',
running: 'In progress',
succeeded: 'Completed',
warning: 'Completed with warnings',
failed: 'Failed',
cancelled: 'Cancelled',
attention: 'Needs attention',
unknown: 'Unknown'
} as const;
export type ActivityState = keyof typeof activityStates;
@@ -54,8 +74,11 @@ export interface ActivityPage {
query: ActivityQuery;
}
function selection<T extends string>(value: string | null, labels: Record<T, string>): T | undefined {
return value !== null && Object.hasOwn(labels, value) ? value as T : undefined;
function selection<T extends string>(
value: string | null,
labels: Record<T, string>
): T | undefined {
return value !== null && Object.hasOwn(labels, value) ? (value as T) : undefined;
}
function integer(value: string | null, fallback: number, min: number, max: number): number {
if (value === null || !/^\d+$/.test(value)) return fallback;
@@ -72,7 +95,10 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery {
if (params.get('excludeProvisioning') === 'true') query.excludeProvisioning = true;
if (params.get('excludeMaintenance') === 'true') query.excludeMaintenance = true;
for (const key of ['server', 'host', 'q'] as const) {
const value = params.get(key)?.trim().slice(0, key === 'q' ? 200 : 255);
const value = params
.get(key)
?.trim()
.slice(0, key === 'q' ? 200 : 255);
if (value) query[key] = value;
}
query.type = selection(params.get('type'), activityTypes);
@@ -86,6 +112,7 @@ export function parseActivityQuery(params: URLSearchParams): ActivityQuery {
export function normalizeActivityQuery(query: ActivityQuery): ActivityQuery {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(query)) if (value !== undefined) params.set(key, String(value));
for (const [key, value] of Object.entries(query))
if (value !== undefined) params.set(key, String(value));
return parseActivityQuery(params);
}
src/lib/artifact-version.ts
+3 -1
View File
@@ -1,5 +1,7 @@
/** Keep the upload number visible: two builds may share a publisher's release tag. */
export function artifactVersionLabel(version: { version: number; release_version?: string | null } | null | undefined): string {
export function artifactVersionLabel(
version: { version: number; release_version?: string | null } | null | undefined
): string {
if (!version) return '—';
return version.release_version
? `${version.release_version} (upload ${version.version})`
src/lib/auth.ts
+3 -1
View File
@@ -16,5 +16,7 @@ export interface AuthUser {
}
export const roleLabels: Record<UserRole, string> = {
admin: 'Administrator', operator: 'Operator', viewer: 'Viewer'
admin: 'Administrator',
operator: 'Operator',
viewer: 'Viewer'
};
src/lib/autoscaling.ts
+95 -23
View File
@@ -1,56 +1,128 @@
import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
export const autoscalingModes = ['off', 'create-destroy'] as const;
export type AutoscalingMode = typeof autoscalingModes[number];
export type AutoscalingMode = (typeof autoscalingModes)[number];
export interface ScalingSettings {
game: ZombieGame;
/** '*' applies the targets independently to every map. */
mapSlug: string;
mode: AutoscalingMode;
minimum: number; maximum: number; spares: number; emptyMinutes: number;
minimum: number;
maximum: number;
spares: number;
emptyMinutes: number;
}
export interface ScalingPolicy extends ScalingSettings {
id: string; revision: number; pausedReason: string | null; updatedAt: string;
id: string;
revision: number;
pausedReason: string | null;
updatedAt: string;
}
export interface ScalingMember {
name: string; mapSlug?: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
players: number | null; observedAt: number | null; source: string | null;
emptySince: number | null; blocker: string | null; operationId: string | null;
name: string;
mapSlug?: string;
identity: string;
state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
players: number | null;
observedAt: number | null;
source: string | null;
emptySince: number | null;
blocker: string | null;
operationId: string | null;
emptyStartedAt?: number | null;
}
export interface ScalingSnapshot {
members: ScalingMember[]; reservedNames: string[]; keyPool: { total: number; available: number }; blockers: string[];
members: ScalingMember[];
reservedNames: string[];
keyPool: { total: number; available: number };
blockers: string[];
}
export interface ScalingDecision {
action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null;
desired: number; deficit: number; occupied: number; empty: number; pending: number; unknown: number;
action: 'create' | 'destroy' | 'wait';
reason: string;
server: string | null;
desired: number;
deficit: number;
occupied: number;
empty: number;
pending: number;
unknown: number;
}
export interface ScalingEvaluation {
policy: ScalingPolicy;
snapshot: ScalingSnapshot;
decision: ScalingDecision;
blockers: string[];
}
export interface ScalingHistory {
id: string;
at: number;
lastSeenAt: number;
decision: ScalingDecision;
operationId: string | null;
operationState: string | null;
}
export interface ScalingEvaluation { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; blockers: string[] }
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
export interface ScalingView extends ScalingEvaluation {
history: ScalingHistory[]; lastEvaluatedAt: number | null;
totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null };
history: ScalingHistory[];
lastEvaluatedAt: number | null;
totals: {
observedMinutes: number;
emptyLobbyHours: number;
deficitMinutes: number;
unknownMinutes: number;
};
operations: {
creates: number;
removals: number;
cancelled: number;
failed: number;
readinessSamples: number;
readinessMeanSeconds: number | null;
readinessMaxSeconds: number | null;
};
ownsMap: boolean;
unsettled: { id: string; action: string; state: string; stage: string }[];
}
export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
export interface ScalingReport {
policies: ScalingView[];
globalPaused: boolean;
liveAvailable: boolean;
removalAvailable: boolean;
}
export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-destroy';
export function scalingSettings(input: unknown): ScalingSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide an autoscaling policy.');
const value = input as Record<string, unknown>;
const { game, mapSlug } = value;
if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)) throw new Error('Choose a supported game and map variant, or all maps.');
if (
(game !== 't5' && game !== 't6') ||
typeof mapSlug !== 'string' ||
(mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)
)
throw new Error('Choose a supported game and map variant, or all maps.');
const mode = value.mode ?? 'off';
if (!autoscalingModes.includes(mode as AutoscalingMode)) throw new Error('Choose a valid autoscaling mode.');
if (!autoscalingModes.includes(mode as AutoscalingMode))
throw new Error('Choose a valid autoscaling mode.');
const integer = (key: string, fallback: number | undefined, max: number) => {
const n = value[key] ?? fallback;
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max) throw new Error(`${key} must be an integer from 1 to ${max}.`);
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max)
throw new Error(`${key} must be an integer from 1 to ${max}.`);
return n;
};
const minimum = integer('minimum', 1, 20), maximum = integer('maximum', undefined, 20), spares = integer('spares', 1, 20);
if (minimum > maximum || spares > maximum) throw new Error('Minimum and spare target cannot exceed the maximum.');
return { game, mapSlug, mode: mode as AutoscalingMode, minimum, maximum, spares,
emptyMinutes: integer('emptyMinutes', 15, 1440) };
const minimum = integer('minimum', 1, 20),
maximum = integer('maximum', undefined, 20),
spares = integer('spares', 1, 20);
if (minimum > maximum || spares > maximum)
throw new Error('Minimum and spare target cannot exceed the maximum.');
return {
game,
mapSlug,
mode: mode as AutoscalingMode,
minimum,
maximum,
spares,
emptyMinutes: integer('emptyMinutes', 15, 1440)
};
}
src/lib/beta-draft.ts
+120 -34
View File
@@ -2,9 +2,12 @@ import { requirementKinds, type BetaPlan } from './beta-plan.js';
export const betaDraftKey = 'electric-cherry:beta-draft:v1';
export interface BetaDraft {
version: 1; savedAt: string; step: number;
version: 1;
savedAt: string;
step: number;
picks: Array<{ scriptId: number | null; versionId: number | null }>;
servers: string[]; testPlan: BetaPlan;
servers: string[];
testPlan: BetaPlan;
selectionMode: 'automatic' | 'manual';
instanceCount: number | null;
label?: string;
@@ -14,55 +17,138 @@ export function readBetaDraft(raw: string | null): BetaDraft | null {
if (!raw) return null;
try {
const d = JSON.parse(raw);
if (d?.version !== 1 || !Array.isArray(d.picks) || !d.picks.length || d.picks.length > 100
|| !d.picks.every((p: BetaDraft['picks'][number]) => p && [p.scriptId, p.versionId].every(v => v === null || Number.isSafeInteger(v) && v! > 0))
|| !Array.isArray(d.servers) || !d.servers.every((s: unknown) => typeof s === 'string')
|| d.testPlan?.version !== 1 || !Array.isArray(d.testPlan.requirements) || d.testPlan.requirements.length > 30
|| !d.testPlan.requirements.every((r: BetaPlan['requirements'][number]) => r && requirementKinds.includes(r.kind) && typeof r.id === 'string' && /^[a-zA-Z0-9_-]{1,64}$/.test(r.id) && typeof r.label === 'string')) return null;
if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode)) return null;
if (d.instancesPerGame !== undefined && (!d.instancesPerGame || typeof d.instancesPerGame !== 'object'
|| Array.isArray(d.instancesPerGame) || !Object.entries(d.instancesPerGame).every(([game, count]) =>
['t5', 't6'].includes(game) && (count === null || typeof count === 'number' && Number.isFinite(count))))) return null;
if (d.instanceCount !== undefined && d.instanceCount !== null
&& (typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))) return null;
if (
d?.version !== 1 ||
!Array.isArray(d.picks) ||
!d.picks.length ||
d.picks.length > 100 ||
!d.picks.every(
(p: BetaDraft['picks'][number]) =>
p &&
[p.scriptId, p.versionId].every(
(v) => v === null || (Number.isSafeInteger(v) && v! > 0)
)
) ||
!Array.isArray(d.servers) ||
!d.servers.every((s: unknown) => typeof s === 'string') ||
d.testPlan?.version !== 1 ||
!Array.isArray(d.testPlan.requirements) ||
d.testPlan.requirements.length > 30 ||
!d.testPlan.requirements.every(
(r: BetaPlan['requirements'][number]) =>
r &&
requirementKinds.includes(r.kind) &&
typeof r.id === 'string' &&
/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) &&
typeof r.label === 'string'
)
)
return null;
if (d.selectionMode !== undefined && !['automatic', 'manual'].includes(d.selectionMode))
return null;
if (
d.instancesPerGame !== undefined &&
(!d.instancesPerGame ||
typeof d.instancesPerGame !== 'object' ||
Array.isArray(d.instancesPerGame) ||
!Object.entries(d.instancesPerGame).every(
([game, count]) =>
['t5', 't6'].includes(game) &&
(count === null || (typeof count === 'number' && Number.isFinite(count)))
))
)
return null;
if (
d.instanceCount !== undefined &&
d.instanceCount !== null &&
(typeof d.instanceCount !== 'number' || !Number.isFinite(d.instanceCount))
)
return null;
const previousCounts = Object.values(d.instancesPerGame ?? {}) as Array<number | null>;
const instanceCount = d.instanceCount !== undefined ? d.instanceCount
: previousCounts.includes(null) ? null : previousCounts.length ? Math.max(...previousCounts as number[]) : 1;
const instanceCount =
d.instanceCount !== undefined
? d.instanceCount
: previousCounts.includes(null)
? null
: previousCounts.length
? Math.max(...(previousCounts as number[]))
: 1;
// Existing drafts represent explicit selections and must not silently switch modes.
const selectionMode = d.selectionMode ?? (d.servers.length ? 'manual' : 'automatic');
// Revisit server setup when old per-game counts cannot be retained as one value.
const changedCounts = selectionMode === 'automatic' && d.instanceCount === undefined && new Set(previousCounts).size > 1;
return { ...d, selectionMode,
instanceCount, step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0)) };
} catch { return null; }
const changedCounts =
selectionMode === 'automatic' &&
d.instanceCount === undefined &&
new Set(previousCounts).size > 1;
return {
...d,
selectionMode,
instanceCount,
step: Math.max(0, Math.min(changedCounts ? 1 : 3, Number(d.step) || 0))
};
} catch {
return null;
}
}
interface BetaFileScope { scope_type: 'game' | 'map' | null; scope_value: string | null }
interface BetaServerScope { name: string; game: string | null; map: string | null }
interface BetaFileScope {
scope_type: 'game' | 'map' | null;
scope_value: string | null;
}
interface BetaServerScope {
name: string;
game: string | null;
map: string | null;
}
export function betaScopeMatches(file: BetaFileScope, server: BetaServerScope): boolean {
return !file.scope_type || (file.scope_type === 'game' ? server.game : server.map) === file.scope_value;
return (
!file.scope_type ||
(file.scope_type === 'game' ? server.game : server.map) === file.scope_value
);
}
/** A bundle installs each file only on its own matching participants. */
export function betaCandidateServers<T extends BetaServerScope>(files: readonly BetaFileScope[], servers: readonly T[]): T[] {
return servers.filter(server => files.some(file => betaScopeMatches(file, server)));
export function betaCandidateServers<T extends BetaServerScope>(
files: readonly BetaFileScope[],
servers: readonly T[]
): T[] {
return servers.filter((server) => files.some((file) => betaScopeMatches(file, server)));
}
export function betaApplicableGames(files: readonly BetaFileScope[], servers: readonly BetaServerScope[]): string[] {
return [...new Set([
...files.filter(file => file.scope_type === 'game').map(file => file.scope_value),
...betaCandidateServers(files, servers).map(server => server.game)
].filter((game): game is string => game === 't5' || game === 't6'))].sort();
export function betaApplicableGames(
files: readonly BetaFileScope[],
servers: readonly BetaServerScope[]
): string[] {
return [
...new Set(
[
...files
.filter((file) => file.scope_type === 'game')
.map((file) => file.scope_value),
...betaCandidateServers(files, servers).map((server) => server.game)
].filter((game): game is string => game === 't5' || game === 't6')
)
].sort();
}
/** Expand the shared count only to games represented by the selected files. */
export function betaInstanceCounts(games: readonly string[], count: number | null): Record<string, number> {
export function betaInstanceCounts(
games: readonly string[],
count: number | null
): Record<string, number> {
if (count === null || !Number.isSafeInteger(count) || count < 1 || count > 1000) return {};
return Object.fromEntries(games.map(game => [game, count]));
return Object.fromEntries(games.map((game) => [game, count]));
}
export function compatibleBetaServers(selected: Iterable<string>, candidates: readonly string[]): { kept: Set<string>; removed: string[] } {
const allowed = new Set(candidates), names = [...selected];
return { kept: new Set(names.filter(name => allowed.has(name))), removed: names.filter(name => !allowed.has(name)) };
export function compatibleBetaServers(
selected: Iterable<string>,
candidates: readonly string[]
): { kept: Set<string>; removed: string[] } {
const allowed = new Set(candidates),
names = [...selected];
return {
kept: new Set(names.filter((name) => allowed.has(name))),
removed: names.filter((name) => !allowed.has(name))
};
}
src/lib/beta-evidence.ts
+39 -14
View File
@@ -1,23 +1,35 @@
import type { BetaRequirement, BetaTestView } from './beta-plan.js';
import { needsGameTelemetry } from './beta-plan-summary.js';
export function betaWaitingCondition(requirement: BetaRequirement, observation?: BetaTestView['servers'][number]['observation']): string {
if (requirement.kind === 'manual') return 'Waiting for a tester to record their name and observation.';
if (requirement.kind === 'recovery') return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
if (requirement.kind === 'healthy-minutes') return observation?.healthy === true
export function betaWaitingCondition(
requirement: BetaRequirement,
observation?: BetaTestView['servers'][number]['observation']
): string {
if (requirement.kind === 'manual')
return 'Waiting for a tester to record their name and observation.';
if (requirement.kind === 'recovery')
return 'The other checks must pass before the next restart when empty; each cycle needs fresh evidence.';
if (requirement.kind === 'healthy-minutes')
return observation?.healthy === true
? 'Collecting consecutive fresh healthy observations; an outage or monitoring gap resets this duration.'
: 'Waiting for fresh container and RCON health observations. Unhealthy or stale observations reset this duration.';
if (requirement.kind === 'failure-budget') return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
if (requirement.kind === 'failure-budget')
return 'An observed health outage exceeded the budget. Investigate before starting a fresh attempt.';
if (needsGameTelemetry(requirement)) {
if (!observation?.lastGameplayAt) return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
if (!observation?.lastGameplayAt)
return 'No gameplay report received for this attempt. Check the game reporter and Telemetry API requests.';
if (observation.gameEnded) return 'Waiting for the next game and its qualifying reports.';
if (requirement.kind === 'population') return `Waiting for a game report containing at least ${requirement.target} players.`;
if (requirement.kind === 'population')
return `Waiting for a game report containing at least ${requirement.target} players.`;
const players = `at least ${requirement.minPlayers} player${requirement.minPlayers === 1 ? '' : 's'}`;
if ((observation.players ?? 0) < requirement.minPlayers) return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
if (requirement.kind === 'matches') return requirement.minMinutes && !observation.matchStarted
if ((observation.players ?? 0) < requirement.minPlayers)
return `The latest report has ${observation.players ?? 0} players; this check needs ${players}.`;
if (requirement.kind === 'matches')
return requirement.minMinutes && !observation.matchStarted
? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.'
: `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`;
if (requirement.kind === 'round') return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
if (requirement.kind === 'round')
return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
return `Time is credited when the next report arrives, with ${players} at both ends of the interval. It does not advance continuously between reports.`;
}
return 'Waiting for qualifying evidence.';
@@ -25,13 +37,26 @@ export function betaWaitingCondition(requirement: BetaRequirement, observation?:
export function betaEvidenceSummary(event: BetaTestView['evidence'][number]): string {
let payload: Record<string, unknown>;
try { payload = JSON.parse(event.payload); } catch { return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); }
try {
payload = JSON.parse(event.payload);
} catch {
return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
}
if (!payload || typeof payload !== 'object') return event.kind;
if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) {
const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : event.kind === 'heartbeat' ? 'Heartbeat' : 'Round changed';
const label =
event.kind === 'game_start'
? 'Game started'
: event.kind === 'game_end'
? 'Game ended'
: event.kind === 'heartbeat'
? 'Heartbeat'
: 'Round changed';
return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`;
}
if (event.kind === 'manual-sign-off') return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
if (event.kind === 'health' || event.kind === 'status-sample') return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
if (event.kind === 'manual-sign-off')
return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
if (event.kind === 'health' || event.kind === 'status-sample')
return `${payload.healthy ? 'Healthy observation' : 'Unhealthy observation'}${payload.reason ? `: ${payload.reason}` : '.'}`;
return event.kind.replaceAll('-', ' ').replaceAll('_', ' ');
}
src/lib/beta-plan-summary.ts
+48 -16
View File
@@ -1,34 +1,66 @@
import type { BetaPlan, BetaRequirement } from './beta-plan.js';
const count = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`;
const duration = (minutes: number) => minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
const duration = (minutes: number) =>
minutes >= 60 && minutes % 60 === 0 ? count(minutes / 60, 'hour') : count(minutes, 'minute');
export const presetDescriptions: Record<string, string> = {
'Quick smoke test': 'A quick check that the server starts and stays healthy.',
'Gameplay validation': 'Prove the build through real matches and populated playtime.',
'Long soak': 'Exercise stability and participation over a longer test.'
};
export function requirementSummary(r: BetaRequirement): string {
const players = r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
const players =
r.minPlayers > 1 ? ` with at least ${r.minPlayers} players` : ' with players present';
switch (r.kind) {
case 'round': return `Reach round ${r.target}${players}`;
case 'matches': return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
case 'populated-minutes': return `Accumulate ${duration(r.target)}${players}`;
case 'player-hours': return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
case 'population': return `Reach ${count(r.target, 'player')} online`;
case 'sustained-population': return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
case 'healthy-minutes': return `Stay continuously healthy for ${duration(r.target)}`;
case 'failure-budget': return r.target === 0 ? 'Allow no health outages' : `Allow at most ${count(r.target, 'health outage')}`;
case 'recovery': return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
case 'manual': return r.label;
case 'round':
return `Reach round ${r.target}${players}`;
case 'matches':
return `Complete ${count(r.target, 'match').replace('matchs', 'matches')}${r.minRound ? ` reaching round ${r.minRound}` : ''}${r.minPlayers > 1 ? players : ''}${r.minMinutes ? ` lasting at least ${duration(r.minMinutes)} each` : ''}`;
case 'populated-minutes':
return `Accumulate ${duration(r.target)}${players}`;
case 'player-hours':
return `Accumulate ${r.target} player-hours${r.minPlayers > 1 ? players : ''}`;
case 'population':
return `Reach ${count(r.target, 'player')} online`;
case 'sustained-population':
return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
case 'healthy-minutes':
return `Stay continuously healthy for ${duration(r.target)}`;
case 'failure-budget':
return r.target === 0
? 'Allow no health outages'
: `Allow at most ${count(r.target, 'health outage')}`;
case 'recovery':
return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
case 'manual':
return r.label;
}
}
export function coverageSummary(plan: BetaPlan): string {
return plan.scope === 'all' ? 'Every selected server' : plan.scope === 'quorum' ? `At least ${count(plan.quorum, 'selected server')}`
: plan.scope === 'map' ? 'One server per selected map' : plan.scope === 'game' ? 'One server per selected game' : 'One server per game / map';
return plan.scope === 'all'
? 'Every selected server'
: plan.scope === 'quorum'
? `At least ${count(plan.quorum, 'selected server')}`
: plan.scope === 'map'
? 'One server per selected map'
: plan.scope === 'game'
? 'One server per selected game'
: 'One server per game / map';
}
export function needsGameTelemetry(r: BetaRequirement): boolean {
return ['round', 'matches', 'population', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind);
return [
'round',
'matches',
'population',
'populated-minutes',
'player-hours',
'sustained-population'
].includes(r.kind);
}
export function requirementCategory(r: BetaRequirement): string {
return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget'].includes(r.kind) ? 'Continuous check' : 'Milestone';
return r.kind === 'manual'
? 'Tester sign-off'
: ['healthy-minutes', 'failure-budget'].includes(r.kind)
? 'Continuous check'
: 'Milestone';
}
src/lib/beta-plan.ts
+280 -54
View File
@@ -1,6 +1,16 @@
export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population',
'sustained-population', 'healthy-minutes', 'failure-budget', 'recovery', 'manual'] as const;
export type RequirementKind = typeof requirementKinds[number];
export const requirementKinds = [
'round',
'matches',
'populated-minutes',
'player-hours',
'population',
'sustained-population',
'healthy-minutes',
'failure-budget',
'recovery',
'manual'
] as const;
export type RequirementKind = (typeof requirementKinds)[number];
export interface BetaRequirement {
id: string;
kind: RequirementKind;
@@ -21,90 +31,306 @@ export interface BetaPlan {
waveSize: number; // 0 deploys all selected servers; otherwise pass each wave before expanding.
}
export const requirementLabels: Record<RequirementKind, string> = {
round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes',
'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)',
'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages',
recovery: 'Successful recovery cycles', manual: 'Tester checklist'
round: 'Reach round',
matches: 'Complete qualifying matches',
'populated-minutes': 'Populated minutes',
'player-hours': 'Player-hours',
population: 'Reach player count',
'sustained-population': 'Sustained population (minutes)',
'healthy-minutes': 'Continuous healthy minutes',
'failure-budget': 'Maximum health outages',
recovery: 'Successful recovery cycles',
manual: 'Tester checklist'
};
export function requirement(kind: RequirementKind, id: string): BetaRequirement {
return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
minRound: 0, minPlayers: 1, minMinutes: 0 };
return {
id,
kind,
label: requirementLabels[kind],
target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
minRound: 0,
minPlayers: 1,
minMinutes: 0
};
}
export function defaultPlan(): BetaPlan {
return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1,
timeoutHours: 24, failureAction: 'fail', rollbackRestart: 'when-empty', waveSize: 0 };
return {
version: 1,
requirements: [requirement('healthy-minutes', 'health')],
scope: 'all',
quorum: 1,
timeoutHours: 24,
failureAction: 'fail',
rollbackRestart: 'when-empty',
waveSize: 0
};
}
export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [
{ name: 'Quick smoke test', plan: defaultPlan() },
{ name: 'Gameplay validation', plan: { ...defaultPlan(), requirements: [
{
name: 'Gameplay validation',
plan: {
...defaultPlan(),
requirements: [
{ ...requirement('matches', 'matches'), target: 2, minRound: 25 },
{ ...requirement('populated-minutes', 'playtime'), target: 90 }, requirement('failure-budget', 'outages')
] } },
{ name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [
{ ...requirement('populated-minutes', 'playtime'), target: 90 },
requirement('failure-budget', 'outages')
]
}
},
{
name: 'Long soak',
plan: {
...defaultPlan(),
timeoutHours: 72,
requirements: [
{ ...requirement('healthy-minutes', 'health'), target: 360 },
{ ...requirement('player-hours', 'participation'), target: 10 }, requirement('failure-budget', 'outages')
] } }
{ ...requirement('player-hours', 'participation'), target: 10 },
requirement('failure-budget', 'outages')
]
}
}
];
/** Strict allow-list validation at every boundary (including stored presets). */
export function parseBetaPlan(value: unknown): BetaPlan {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Test plan must be an object');
if (!value || typeof value !== 'object' || Array.isArray(value))
throw new Error('Test plan must be an object');
const p = value as BetaPlan;
if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) throw new Error('Invalid test plan version or scope');
if (!['fail', 'pause', 'rollback'].includes(p.failureAction) || !['manual', 'when-empty'].includes(p.rollbackRestart)) throw new Error('Invalid failure policy');
for (const [key, min, max] of [['quorum', 1, 1000], ['timeoutHours', 0.01, 8760], ['waveSize', 0, 1000]] as const) {
if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) throw new Error(`Invalid ${key}`);
if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope))
throw new Error('Invalid test plan version or scope');
if (
!['fail', 'pause', 'rollback'].includes(p.failureAction) ||
!['manual', 'when-empty'].includes(p.rollbackRestart)
)
throw new Error('Invalid failure policy');
for (const [key, min, max] of [
['quorum', 1, 1000],
['timeoutHours', 0.01, 8760],
['waveSize', 0, 1000]
] as const) {
if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max)
throw new Error(`Invalid ${key}`);
}
if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) throw new Error('Quorum and wave size must be integers');
if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) throw new Error('Choose 1–30 requirements');
if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize))
throw new Error('Quorum and wave size must be integers');
if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30)
throw new Error('Choose 1–30 requirements');
const ids = new Set<string>();
const requirements = p.requirements.map(r => {
if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) throw new Error('Unknown requirement');
if (typeof r.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || r.id in Object.prototype || ids.has(r.id)) throw new Error('Requirement IDs must be unique, non-reserved identifiers');
const requirements = p.requirements.map((r) => {
if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind))
throw new Error('Unknown requirement');
if (
typeof r.id !== 'string' ||
!/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) ||
r.id in Object.prototype ||
ids.has(r.id)
)
throw new Error('Requirement IDs must be unique, non-reserved identifiers');
ids.add(r.id);
if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) throw new Error('Requirement needs a label (up to 200 characters)');
if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200)
throw new Error('Requirement needs a label (up to 200 characters)');
for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) {
if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`);
if (
typeof r[key] !== 'number' ||
!Number.isFinite(r[key]) ||
r[key] < 0 ||
r[key] > 1_000_000
)
throw new Error(`Invalid requirement ${key}`);
}
if (r.kind !== 'failure-budget' && r.target <= 0) throw new Error('Requirement target must be positive');
if (['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers');
if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers');
if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item');
return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound,
minPlayers: r.minPlayers, minMinutes: r.minMinutes };
if (r.kind !== 'failure-budget' && r.target <= 0)
throw new Error('Requirement target must be positive');
if (
['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(
r.kind
) &&
!Number.isInteger(r.target)
)
throw new Error('Count targets must be integers');
if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers))
throw new Error('Round and player qualifiers must be integers');
if (r.kind === 'manual' && r.target !== 1)
throw new Error('Add a separate requirement for each checklist item');
return {
id: r.id,
kind: r.kind,
label: r.label.trim(),
target: r.target,
minRound: r.minRound,
minPlayers: r.minPlayers,
minMinutes: r.minMinutes
};
});
return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours,
failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize };
return {
version: 1,
requirements,
scope: p.scope,
quorum: p.quorum,
timeoutHours: p.timeoutHours,
failureAction: p.failureAction,
rollbackRestart: p.rollbackRestart,
waveSize: p.waveSize
};
}
export interface RequirementProgress { id: string; label: string; current: number; target: number; passed: boolean; failed: boolean; detail: string }
export interface RequirementProgress {
id: string;
label: string;
current: number;
target: number;
passed: boolean;
failed: boolean;
detail: string;
}
export interface BetaTestView {
plan: BetaPlan;
outcome: string;
reason: string | null;
servers: Array<{ server: string; server_deleted_at: string | null; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null;
observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>;
evidence: Array<{ id: number; server: string; server_deleted_at: string | null; received_at: string; kind: string; payload: string }>;
servers: Array<{
server: string;
server_deleted_at: string | null;
phase: number;
passed: boolean;
progress: RequirementProgress[];
observingAt?: string | null;
observation?: {
lastGameplayAt: string | null;
lastEvent: string | null;
players: number | null;
round: number | null;
matchStarted: boolean;
gameEnded: boolean;
healthy: boolean | null;
healthCheckedAt: string | null;
};
}>;
evidence: Array<{
id: number;
server: string;
server_deleted_at: string | null;
received_at: string;
kind: string;
payload: string;
}>;
}
export interface BetaPlanIssue { field: string; message: string; requirementId?: string }
export interface BetaPlanIssue {
field: string;
message: string;
requirementId?: string;
}
export function betaPlanIssues(plan: BetaPlan, serverCount?: number): BetaPlanIssue[] {
const issues: BetaPlanIssue[] = [];
const check = (field: string, value: number, min: number, max: number, integer: boolean, message: string, requirementId?: string) => {
if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) issues.push({ field, message, requirementId });
const check = (
field: string,
value: number,
min: number,
max: number,
integer: boolean,
message: string,
requirementId?: string
) => {
if (
typeof value !== 'number' ||
!Number.isFinite(value) ||
value < min ||
value > max ||
(integer && !Number.isInteger(value))
)
issues.push({ field, message, requirementId });
};
check('timeoutHours', plan.timeoutHours, .01, 8760, false, 'Enter a testing window between 0.01 and 8,760 hours.');
check('waveSize', plan.waveSize, 0, 1000, true, 'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.');
if (plan.scope === 'quorum') check('quorum', plan.quorum, 1, serverCount ?? 1000, true, `Choose between 1 and ${serverCount ?? 1000} qualifying servers.`);
if (!plan.requirements.length) issues.push({ field: 'requirements', message: 'Add at least one requirement to continue.' });
check(
'timeoutHours',
plan.timeoutHours,
0.01,
8760,
false,
'Enter a testing window between 0.01 and 8,760 hours.'
);
check(
'waveSize',
plan.waveSize,
0,
1000,
true,
'Enter a whole number from 0 to 1,000; 0 deploys to all selected servers.'
);
if (plan.scope === 'quorum')
check(
'quorum',
plan.quorum,
1,
serverCount ?? 1000,
true,
`Choose between 1 and ${serverCount ?? 1000} qualifying servers.`
);
if (!plan.requirements.length)
issues.push({
field: 'requirements',
message: 'Add at least one requirement to continue.'
});
for (const r of plan.requirements) {
if (!r.label?.trim()) issues.push({ field: `${r.id}-label`, message: 'Describe what this requirement checks.', requirementId: r.id });
const integral = ['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind);
check(`${r.id}-target`, r.target, r.kind === 'failure-budget' ? 0 : .000001, 1_000_000, integral, `Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`, r.id);
if (['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)) check(`${r.id}-minPlayers`, r.minPlayers, 1, 1_000_000, true, 'Enter at least one player, using a whole number.', r.id);
if (!r.label?.trim())
issues.push({
field: `${r.id}-label`,
message: 'Describe what this requirement checks.',
requirementId: r.id
});
const integral = [
'round',
'matches',
'population',
'failure-budget',
'recovery',
'manual'
].includes(r.kind);
check(
`${r.id}-target`,
r.target,
r.kind === 'failure-budget' ? 0 : 0.000001,
1_000_000,
integral,
`Enter ${integral ? 'a whole number' : 'a number'} ${r.kind === 'failure-budget' ? 'of zero or more' : 'greater than zero'} for ${requirementLabels[r.kind].toLowerCase()}.`,
r.id
);
if (
[
'round',
'matches',
'populated-minutes',
'player-hours',
'sustained-population'
].includes(r.kind)
)
check(
`${r.id}-minPlayers`,
r.minPlayers,
1,
1_000_000,
true,
'Enter at least one player, using a whole number.',
r.id
);
if (r.kind === 'matches') {
check(`${r.id}-minRound`, r.minRound, 0, 1_000_000, true, 'Enter a whole round number; 0 allows any round.', r.id);
check(`${r.id}-minMinutes`, r.minMinutes, 0, 1_000_000, false, 'Enter a duration of zero or more minutes.', r.id);
check(
`${r.id}-minRound`,
r.minRound,
0,
1_000_000,
true,
'Enter a whole round number; 0 allows any round.',
r.id
);
check(
`${r.id}-minMinutes`,
r.minMinutes,
0,
1_000_000,
false,
'Enter a duration of zero or more minutes.',
r.id
);
}
}
return issues;
src/lib/beta-promotion.ts
+16 -4
View File
@@ -3,12 +3,24 @@ export interface BetaPromotionPreview {
token: string;
createdAt: string;
files: Array<{
id: number; name: string; kind: string; path: string; versionId: number;
version: number; release_version: string | null; sha256: string;
scope: string; servers: string[];
id: number;
name: string;
kind: string;
path: string;
versionId: number;
version: number;
release_version: string | null;
sha256: string;
scope: string;
servers: string[];
}>;
servers: Array<{ name: string; tested: boolean; game: string | null; map: string | null }>;
configWrites: Array<{ server: string; path: string; sha256: string | null; error: string | null }>;
configWrites: Array<{
server: string;
path: string;
sha256: string | null;
error: string | null;
}>;
blockers: string[];
restartPolicy: 'additional-when-empty';
}
src/lib/beta-revision.ts
+16 -4
View File
@@ -1,11 +1,23 @@
import type { BetaTestView } from './beta-plan.js';
export interface BetaRevisionFile {
script_id: number; name: string; kind: string; path: string;
version_id: number; version: number; release_version: string | null; sha256: string;
script_id: number;
name: string;
kind: string;
path: string;
version_id: number;
version: number;
release_version: string | null;
sha256: string;
}
export interface BetaRevision {
revision: number; started_at: string; notes: string | null;
revision: number;
started_at: string;
notes: string | null;
files: BetaRevisionFile[];
results: { state: string; test: BetaTestView | null; servers: Array<{ server: string; stage: string; error: string | null }> } | null;
results: {
state: string;
test: BetaTestView | null;
servers: Array<{ server: string; stage: string; error: string | null }>;
} | null;
}
src/lib/capacity.ts
+15 -4
View File
@@ -2,12 +2,19 @@ import type { CapacityCalibration, HostReadiness } from './hosts.js';
export type CapacityRange = '1h' | '24h' | '7d';
export type CapacityGame = 'all' | 't5' | 't6';
export interface CapacityQuery { range: CapacityRange; hostId: string | null; game: CapacityGame }
export interface CapacityQuery {
range: CapacityRange;
hostId: string | null;
game: CapacityGame;
}
export function parseCapacityQuery(params: URLSearchParams): CapacityQuery {
const range = params.get('range');
const game = params.get('game');
return { range: range === '1h' || range === '7d' ? range : '24h',
game: game === 't5' || game === 't6' ? game : 'all', hostId: params.get('host')?.trim().slice(0, 100) || null };
return {
range: range === '1h' || range === '7d' ? range : '24h',
game: game === 't5' || game === 't6' ? game : 'all',
hostId: params.get('host')?.trim().slice(0, 100) || null
};
}
export interface CapacityHistoryPoint {
@@ -60,4 +67,8 @@ export interface CapacityCurrent {
hostOptions: { id: string; name: string }[];
warnings: string[];
}
export interface CapacityData { query: CapacityQuery; current: CapacityCurrent; history: CapacityHistory }
export interface CapacityData {
query: CapacityQuery;
current: CapacityCurrent;
history: CapacityHistory;
}
src/lib/config-workbench.ts
+28 -11
View File
@@ -27,33 +27,45 @@ export interface ConfigInspection {
reason?: string;
}
export interface ConfigServer { name: string; game: string | null; map: string | null; }
export interface ConfigServer {
name: string;
game: string | null;
map: string | null;
}
/** The same lowercase `set` grammar that the overlay composer accepts. */
export function configLines(text: string) {
return text.split(/\r?\n/).flatMap((line, index) => {
const match = /^(\s*set\s+)(\S+)(\s+)(.*)$/.exec(line);
return match ? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }] : [];
return match
? [{ index, key: match[2], value: match[4], prefix: match[1] + match[2] + match[3] }]
: [];
});
}
export function changeConfigValue(text: string, index: number, value: string): string {
// Keep every separator, comment and unrelated line byte-for-byte intact.
const parts = text.split(/(\r?\n)/);
const entry = configLines(text).find(line => line.index === index);
const entry = configLines(text).find((line) => line.index === index);
if (entry) parts[index * 2] = entry.prefix + value.replace(/[\r\n]/g, '');
return parts.join('');
}
/** Add one assignment without changing existing commands or line endings. */
export function addConfigSetting(text: string, key: string, value: string): string {
key = key.trim(); value = value.trim();
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) throw new Error('Use a setting name containing letters, numbers, and underscores.');
if (!value || /[\r\n]/.test(value) || value.startsWith('//')) throw new Error('Enter a value on one line.');
if (configLines(text).some(entry => entry.key.toLowerCase() === key.toLowerCase())) throw new Error('This setting already exists. Find it above to edit its value.');
key = key.trim();
value = value.trim();
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key))
throw new Error('Use a setting name containing letters, numbers, and underscores.');
if (!value || /[\r\n]/.test(value) || value.startsWith('//'))
throw new Error('Enter a value on one line.');
if (configLines(text).some((entry) => entry.key.toLowerCase() === key.toLowerCase()))
throw new Error('This setting already exists. Find it above to edit its value.');
const newline = text.includes('\r\n') ? '\r\n' : '\n';
const formatted = /\s/.test(value) && !value.startsWith('"') ? JSON.stringify(value) : value;
return text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline;
return (
text + (text && !text.endsWith('\n') ? newline : '') + `set ${key} ${formatted}` + newline
);
}
export function configWarnings(text: string, overlay: boolean): string[] {
@@ -61,11 +73,16 @@ export function configWarnings(text: string, overlay: boolean): string[] {
const seen = new Map<string, number>();
for (const entry of configLines(text)) {
const key = entry.key.toLowerCase();
if (seen.has(key)) warnings.push(`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`);
if (seen.has(key))
warnings.push(
`Line ${entry.index + 1}: ${entry.key} is also assigned on line ${seen.get(key)}.`
);
seen.set(key, entry.index + 1);
if (!entry.value.trim() || entry.value.trim().startsWith('//')) warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
if (!entry.value.trim() || entry.value.trim().startsWith('//'))
warnings.push(`Line ${entry.index + 1}: ${entry.key} has no value.`);
}
if (overlay) text.split(/\r?\n/).forEach((line, index) => {
if (overlay)
text.split(/\r?\n/).forEach((line, index) => {
if (line.trim() && !/^\s*\/\//.test(line) && !/^\s*set\s+\S+\s+/.test(line)) {
warnings.push(`Line ${index + 1} is ignored by overlays. Use set KEY VALUE.`);
}
src/lib/deployment-plan.ts
+41 -7
View File
@@ -55,9 +55,21 @@ export interface CollectionPreview {
mode: CollectionMode;
retainedServers: string[];
removals: { name: string; reason: string }[];
entries: { count: number; existing: number; missing: number; existingServers: string[];
coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
entries: {
count: number;
existing: number;
missing: number;
existingServers: string[];
coverage: { name: string; health: string; href: string }[];
recipe: LaunchRecipeSummary;
}[];
servers: {
native?: import('./hosts.js').NativePlacement;
entryIndex: number;
name: string | null;
port: number | null;
hasSavedKey: boolean;
}[];
blockers: string[];
warnings?: string[];
availability?: LaunchAvailability;
@@ -87,10 +99,32 @@ export interface CollectionDetail {
canCancel: boolean;
retainedServers: string[];
retainedServerDeletedAt?: Record<string, string>;
queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number };
servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string;
started_at: string; finished_at: string | null; error: string | null; server_exists: number; server_deleted_at?: string | null; queuePosition: number | null;
cancellation: { cleanupId: string; state: string; stage: string; error: string | null } | null }[];
queue: {
concurrency: number;
active: { id: string; server: string; state: string; stage: string }[];
queued: number;
};
servers: {
id: string;
server: string;
action: 'create' | 'destroy';
port: number | null;
map_slug: string | null;
state: string;
stage: string;
started_at: string;
finished_at: string | null;
error: string | null;
server_exists: number;
server_deleted_at?: string | null;
queuePosition: number | null;
cancellation: {
cleanupId: string;
state: string;
stage: string;
error: string | null;
} | null;
}[];
}
export interface LaunchPreview {
src/lib/fleet-overview.ts
+77 -20
View File
@@ -9,7 +9,10 @@ import { mapTitle, serverTitle } from './ui/server-labels.js';
export interface FleetServer {
plutonium?: import('./plutonium.js').PlutoniumVersion;
host_id?: string | null;
host?: Pick<import('./hosts.js').Host, 'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat'> | null;
host?: Pick<
import('./hosts.js').Host,
'id' | 'name' | 'connectionState' | 'connected' | 'lastHeartbeat'
> | null;
container_state?: string | null;
container_checked_at?: string | null;
last_checked_at?: string | null;
@@ -56,49 +59,103 @@ export type FleetFilter = 'all' | 'occupied' | 'empty' | 'attention' | 'testing'
export type FleetSort = 'name' | 'players' | 'round' | 'duration';
export function freshServer(server: FleetServer, now: number): boolean {
return server.health === 'online' && !server.last_error && freshStatusTimestamp(server.last_seen_at, now);
return (
server.health === 'online' &&
!server.last_error &&
freshStatusTimestamp(server.last_seen_at, now)
);
}
export function fleetIdentity(server: FleetServer) {
const parsed = parseServerName(server.name);
const game = server.game ?? parsed?.game ?? '';
const mode = server.game_mode ?? parsed?.gameMode;
const selection = !mode || mode === 'zombie'
? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '') : undefined;
const selection =
!mode || mode === 'zombie'
? zombieMapBySlug(game, server.map_slug ?? parsed?.mapSlug ?? '')
: undefined;
const map = server.last_map ?? server.map;
// A live map change must not keep the registered map's launch shortcut.
const matchingSelection = selection && (!map || selection.canonical === map) ? selection : undefined;
const matchingSelection =
selection && (!map || selection.canonical === map) ? selection : undefined;
return {
game,
title: serverTitle(server.name),
map: matchingSelection?.friendly ?? mapTitle(map),
group: `${game}:${matchingSelection?.slug ?? map ?? 'unknown'}`,
launch: matchingSelection && (game === 't5' || game === 't6')
? { game: game as ZombieGame, slug: matchingSelection.slug } : null
launch:
matchingSelection && (game === 't5' || game === 't6')
? { game: game as ZombieGame, slug: matchingSelection.slug }
: null
};
}
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
export function currentRound(
server: FleetServer,
report: GameTelemetry | undefined,
now: number
): number | null {
if (
server.telemetry &&
(!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))
)
report = server.telemetry;
if (
!report ||
(report.event !== 'round_change' && report.event !== 'heartbeat') ||
!freshServer(server, now) ||
!server.active_since ||
!server.game_started_at ||
server.game_ended_at ||
(server.last_players ?? 0) === 0
)
return null;
const received = Date.parse(report.receivedAt);
return freshTelemetry(report, now, server.telemetry_after) && received >= Date.parse(server.game_started_at)
&& received >= Date.parse(server.active_since) ? report.round : null;
return freshTelemetry(report, now, server.telemetry_after) &&
received >= Date.parse(server.game_started_at) &&
received >= Date.parse(server.active_since)
? report.round
: null;
}
export function matchesFleet(server: FleetServer, query: string, filter: FleetFilter, game: string,
pinned: readonly string[], operations: readonly FleetOperation[], now: number): boolean {
export function matchesFleet(
server: FleetServer,
query: string,
filter: FleetFilter,
game: string,
pinned: readonly string[],
operations: readonly FleetOperation[],
now: number
): boolean {
const identity = fleetIdentity(server);
if (game && identity.game !== game) return false;
const fresh = freshServer(server, now);
if (filter === 'occupied' && !(fresh && (server.last_players ?? 0) > 0)) return false;
if (filter === 'empty' && !(fresh && server.last_players === 0)) return false;
if (filter === 'attention' && !(server.health === 'offline' || server.health === 'stale'
|| (server.health === 'online' && !fresh) || operations.some(op => op.server === server.name && op.state === 'attention'))) return false;
if (
filter === 'attention' &&
!(
server.health === 'offline' ||
server.health === 'stale' ||
(server.health === 'online' && !fresh) ||
operations.some((op) => op.server === server.name && op.state === 'attention')
)
)
return false;
if (filter === 'testing' && !server.held.length) return false;
if (filter === 'pinned' && !pinned.includes(server.name)) return false;
const text = [server.name, identity.title, identity.map, identity.game,
...(server.roster ?? []).map(player => player.name)].join(' ').toLocaleLowerCase();
return query.trim().toLocaleLowerCase().split(/\s+/).every(term => text.includes(term));
const text = [
server.name,
identity.title,
identity.map,
identity.game,
...(server.roster ?? []).map((player) => player.name)
]
.join(' ')
.toLocaleLowerCase();
return query
.trim()
.toLocaleLowerCase()
.split(/\s+/)
.every((term) => text.includes(term));
}
src/lib/fleet-settings.ts
+186 -53
View File
@@ -33,7 +33,6 @@ export interface FleetSettings {
lifecycleConcurrency: number;
rebuildConcurrency: number;
preparationConcurrency: number;
}
export const defaultFleetSettings: FleetSettings = {
@@ -70,7 +69,7 @@ export const defaultFleetSettings: FleetSettings = {
destroyConcurrencyPerHost: 0,
lifecycleConcurrency: 0,
rebuildConcurrency: 0,
preparationConcurrency: 0,
preparationConcurrency: 0
};
export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
@@ -107,12 +106,12 @@ export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
destroyConcurrencyPerHost: 'Concurrent server removals per host',
lifecycleConcurrency: 'Combined lifecycle limit',
rebuildConcurrency: 'Concurrent rebuilds',
preparationConcurrency: 'Hosts preparing game files simultaneously',
preparationConcurrency: 'Hosts preparing game files simultaneously'
};
export function validateFleetSettings(input: unknown): FleetSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide fleet settings.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide fleet settings.');
const values = input as Record<string, unknown>;
for (const key of Object.keys(values)) {
if (!Object.hasOwn(defaultFleetSettings, key)) throw new Error(`Unknown setting: ${key}`);
@@ -121,20 +120,68 @@ export function validateFleetSettings(input: unknown): FleetSettings {
const value = values[key];
const minimum = settingMinimum(key);
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) {
throw new Error(`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`);
throw new Error(
`${fleetSettingLabels[key]} must be ${minimum === 0 ? 'zero or a positive whole number' : 'a positive whole number'}.`
);
}
if (key.endsWith('Days') && value > 36500) {
throw new Error(`${fleetSettingLabels[key]} must be at most 36500 days.`);
}
}
return Object.fromEntries(Object.keys(defaultFleetSettings).map(key => [key, values[key]])) as unknown as FleetSettings;
return Object.fromEntries(
Object.keys(defaultFleetSettings).map((key) => [key, values[key]])
) as unknown as FleetSettings;
}
export const additionalConcurrencyKeys: (keyof FleetSettings)[] = ["restartConcurrencyPerHost", "createConcurrencyPerHost", "destroyConcurrencyPerHost", "lifecycleConcurrency", "rebuildConcurrency", "preparationConcurrency"];
export const concurrencyKeys: (keyof FleetSettings)[] = ['restartConcurrency', 'createConcurrency', 'destroyConcurrency', ...additionalConcurrencyKeys];
export const retentionKeys: (keyof FleetSettings)[] = ['deployKeepDays', 'observationKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'failureKeepDays', 'maintenanceKeepDays', 'recoveryKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays', 'metricsKeepDays', 'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'matchKeepDays', 'betaKeepDays', 'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays', 'auditKeepDays', 'historyKeepCount'];
export const additionalConcurrencyKeys: (keyof FleetSettings)[] = [
'restartConcurrencyPerHost',
'createConcurrencyPerHost',
'destroyConcurrencyPerHost',
'lifecycleConcurrency',
'rebuildConcurrency',
'preparationConcurrency'
];
export const concurrencyKeys: (keyof FleetSettings)[] = [
'restartConcurrency',
'createConcurrency',
'destroyConcurrency',
...additionalConcurrencyKeys
];
export const retentionKeys: (keyof FleetSettings)[] = [
'deployKeepDays',
'observationKeepDays',
'provisioningKeepDays',
'actionKeepDays',
'failureKeepDays',
'maintenanceKeepDays',
'recoveryKeepDays',
'hostResourceKeepDays',
'instanceResourceKeepDays',
'metricsKeepDays',
'telemetryKeepDays',
'telemetryMaxPerServer',
'telemetryMaxTotal',
'matchKeepDays',
'betaKeepDays',
'fileContentKeepDays',
'payloadKeepDays',
'versionKeepDays',
'versionKeepCount',
'gameFileGraceDays',
'gameFileKeepCount',
'uploadKeepDays',
'auditKeepDays',
'historyKeepCount'
];
export function settingMinimum(key: keyof FleetSettings): number {
return ['restartConcurrency','createConcurrency','destroyConcurrency','gameFileKeepCount'].includes(key) ? 1 : 0;
return [
'restartConcurrency',
'createConcurrency',
'destroyConcurrency',
'gameFileKeepCount'
].includes(key)
? 1
: 0;
}
export interface SettingsGroup {
@@ -146,34 +193,94 @@ export interface SettingsGroup {
/** Presentation groups do not change the saved schema or the section PATCH API. */
export const settingsGroups: SettingsGroup[] = [
{ id: 'concurrency', title: 'Concurrency', description: 'Limit simultaneous server operations and game-file preparation.', keys: concurrencyKeys },
{ id: 'operation-history', title: 'Operation history', description: 'Campaigns, server operations, maintenance, and routine checks.', keys: [
'deployKeepDays', 'provisioningKeepDays', 'actionKeepDays', 'observationKeepDays', 'failureKeepDays',
'maintenanceKeepDays', 'recoveryKeepDays', 'betaKeepDays', 'historyKeepCount'
] },
{ id: 'matches-players', title: 'Matches & players', description: 'Saved matches, player records, and leaderboard history.', keys: ['matchKeepDays'] },
{ id: 'telemetry-metrics', title: 'Telemetry & metrics', description: 'Game diagnostics, fleet activity, and host resource samples.', keys: [
'telemetryKeepDays', 'telemetryMaxPerServer', 'telemetryMaxTotal', 'metricsKeepDays', 'hostResourceKeepDays', 'instanceResourceKeepDays'
] },
{ id: 'files-versions', title: 'Files & versions', description: 'File snapshots, saved versions, game files, and abandoned uploads.', keys: [
'fileContentKeepDays', 'payloadKeepDays', 'versionKeepDays', 'versionKeepCount', 'gameFileGraceDays', 'gameFileKeepCount', 'uploadKeepDays'
] },
{ id: 'accounts-audit', title: 'Accounts & audit', description: 'Sign-in and account administration records.', keys: ['auditKeepDays'] }
{
id: 'concurrency',
title: 'Concurrency',
description: 'Limit simultaneous server operations and game-file preparation.',
keys: concurrencyKeys
},
{
id: 'operation-history',
title: 'Operation history',
description: 'Campaigns, server operations, maintenance, and routine checks.',
keys: [
'deployKeepDays',
'provisioningKeepDays',
'actionKeepDays',
'observationKeepDays',
'failureKeepDays',
'maintenanceKeepDays',
'recoveryKeepDays',
'betaKeepDays',
'historyKeepCount'
]
},
{
id: 'matches-players',
title: 'Matches & players',
description: 'Saved matches, player records, and leaderboard history.',
keys: ['matchKeepDays']
},
{
id: 'telemetry-metrics',
title: 'Telemetry & metrics',
description: 'Game diagnostics, fleet activity, and host resource samples.',
keys: [
'telemetryKeepDays',
'telemetryMaxPerServer',
'telemetryMaxTotal',
'metricsKeepDays',
'hostResourceKeepDays',
'instanceResourceKeepDays'
]
},
{
id: 'files-versions',
title: 'Files & versions',
description: 'File snapshots, saved versions, game files, and abandoned uploads.',
keys: [
'fileContentKeepDays',
'payloadKeepDays',
'versionKeepDays',
'versionKeepCount',
'gameFileGraceDays',
'gameFileKeepCount',
'uploadKeepDays'
]
},
{
id: 'accounts-audit',
title: 'Accounts & audit',
description: 'Sign-in and account administration records.',
keys: ['auditKeepDays']
}
];
/** Zero has a different meaning for age limits, count caps, and minimums. */
export function settingZeroMode(key: keyof FleetSettings): string | null {
if (settingMinimum(key) > 0) return null;
if (additionalConcurrencyKeys.includes(key) || key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal') return 'No limit';
if (
additionalConcurrencyKeys.includes(key) ||
key === 'telemetryMaxPerServer' ||
key === 'telemetryMaxTotal'
)
return 'No limit';
if (key === 'versionKeepDays') return 'No age expiry';
if (key === 'gameFileGraceDays') return 'No delay';
if (key === 'versionKeepCount' || key === 'historyKeepCount') return 'No minimum';
return key.endsWith('Days') ? 'Forever' : null;
}
export function settingUnit(key: keyof FleetSettings): 'days' | 'records' | 'versions' | 'operations' | 'hosts' {
export function settingUnit(
key: keyof FleetSettings
): 'days' | 'records' | 'versions' | 'operations' | 'hosts' {
if (key.endsWith('Days')) return 'days';
if (key === 'telemetryMaxPerServer' || key === 'telemetryMaxTotal' || key === 'historyKeepCount') return 'records';
if (
key === 'telemetryMaxPerServer' ||
key === 'telemetryMaxTotal' ||
key === 'historyKeepCount'
)
return 'records';
if (key === 'versionKeepCount' || key === 'gameFileKeepCount') return 'versions';
return key === 'preparationConcurrency' ? 'hosts' : 'operations';
}
@@ -188,38 +295,64 @@ export function settingDisplayValue(key: keyof FleetSettings, value: number): st
}
export const settingHelp: Partial<Record<keyof FleetSettings, string>> = {
provisioningKeepDays: 'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.',
actionKeepDays: 'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.',
failureKeepDays: 'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.',
maintenanceKeepDays: 'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
recoveryKeepDays: 'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
provisioningKeepDays:
'Completed lifecycle history. Recovery, source workflows and cancellation dependencies remain protected.',
actionKeepDays:
'Completed agent actions. Referenced commands and retry evidence remain protected. Expired actions keep compact receipts to prevent duplicate execution.',
failureKeepDays:
'Explicitly resolved failed operations and campaigns, measured from resolution. Unresolved failures remain protected.',
maintenanceKeepDays:
'Finished non-update maintenance. Update attempts remain durable to prevent duplicate automatic retries.',
recoveryKeepDays:
'Resolved watchdog incidents with no unfinished recovery. Dismissed incidents remain as suppression evidence.',
hostResourceKeepDays: 'Host CPU and memory history.',
instanceResourceKeepDays: 'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
instanceResourceKeepDays:
'Minute resource samples. Capacity learning continues to use its independent 24-hour window.',
metricsKeepDays: 'Historical fleet occupancy and utilization samples.',
telemetryKeepDays: 'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.',
telemetryKeepDays:
'Raw received telemetry diagnostics. Independent of saved matches. Count limits also apply.',
telemetryMaxPerServer: 'Count cap per server, in addition to the age limit.',
telemetryMaxTotal: 'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.',
matchKeepDays: 'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
betaKeepDays: 'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
fileContentKeepDays: 'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
payloadKeepDays: 'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.',
versionKeepDays: 'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.',
versionKeepCount: 'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
gameFileGraceDays: 'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.',
gameFileKeepCount: 'Keep this many recent published versions per game, in addition to every referenced or protected version.',
uploadKeepDays: 'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.',
telemetryMaxTotal:
'Fleet-wide count cap for telemetry diagnostics, in addition to the age limit.',
matchKeepDays:
'Completed and interrupted matches and player details. Expiry changes leaderboards and player totals; no lifetime aggregates are preserved.',
betaKeepDays:
'Closed betas, revisions and test evidence, after linked campaigns and recovery work no longer need them.',
fileContentKeepDays:
'Expired campaign file bodies are removed while summaries remain. Rollback becomes unavailable for those campaigns. Active beta snapshots remain protected.',
payloadKeepDays:
'Expire file bodies in completed launch recipes, commands and receipts while keeping their history. Unfinished work and protected records keep their files.',
versionKeepDays:
'Unused script, plugin, configuration and baseline versions. With no age expiry, the version count policy may still remove older unused versions. Current and referenced versions are always protected.',
versionKeepCount:
'Keep at least this many recent versions per artifact. With no age expiry, a positive count removes older unused versions.',
gameFileGraceDays:
'Delay removal after a version becomes superseded. Latest files, referenced versions and offline host copies remain protected.',
gameFileKeepCount:
'Keep this many recent published versions per game, in addition to every referenced or protected version.',
uploadKeepDays:
'Remove incomplete browser uploads after this many days without activity. Active transfers and publication are protected.',
auditKeepDays: 'Recorded authentication and account-administration events.',
historyKeepCount: 'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.',
historyKeepCount:
'Keep at least this many recent records per history category. Does not apply to metrics, resource samples, telemetry count caps or game-file storage.',
restartConcurrency: 'Fleet recovery slots stay occupied until container and game health are verified.',
createConcurrency: 'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
restartConcurrency:
'Fleet recovery slots stay occupied until container and game health are verified.',
createConcurrency:
'Fleet startup slots cover creation and rebuilds through health and selected-map verification.',
destroyConcurrency: 'Fleet removals hold capacity until teardown and cleanup finish.',
restartConcurrencyPerHost: 'Additional recovery limit on each host, including starts and rebuilds.',
restartConcurrencyPerHost:
'Additional recovery limit on each host, including starts and rebuilds.',
createConcurrencyPerHost: 'Additional startup limit on each host, including rebuilds.',
destroyConcurrencyPerHost: 'Additional removal limit on each host, including migration source removal.',
lifecycleConcurrency: 'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.',
destroyConcurrencyPerHost:
'Additional removal limit on each host, including migration source removal.',
lifecycleConcurrency:
'Combined fleet limit for creates, starts, stops, restarts, rebuilds and removals. Each workflow counts once.',
rebuildConcurrency: 'Additional fleet limit for rebuilds, including host drain migrations.',
preparationConcurrency: 'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
deployKeepDays: 'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.',
observationKeepDays: 'Successful monitoring and file-read records. Failures and actions are retained.'
preparationConcurrency:
'Maximum hosts downloading or verifying game files at once. Interrupted transfers keep their slot until reconciled.',
deployKeepDays:
'Eligible campaign records and file diffs. Pending, unresolved failed, held and recovery-linked campaigns are protected.',
observationKeepDays:
'Successful monitoring and file-read records. Failures and actions are retained.'
};
src/lib/game-chat.ts
+82 -24
View File
@@ -45,12 +45,20 @@ function logSeconds(time: string): number {
}
function plainText(value: string): string {
return value.replace(/\^[0-9]/g, '').replace(/[\x00-\x1f\x7f]/g, '').trim();
return (
value
.replace(/\^[0-9]/g, '')
// eslint-disable-next-line no-control-regex -- Strip control characters from displayed chat text.
.replace(/[\x00-\x1f\x7f]/g, '')
.trim()
);
}
/** Preserve log order: elapsed timestamps can reset between matches. */
export function parseGameChat(lines: readonly string[]): ChatMessage[] {
return parseGameChatTimeline(lines).flatMap(entry => entry.type === 'message' ? [entry.message] : []);
return parseGameChatTimeline(lines).flatMap((entry) =>
entry.type === 'message' ? [entry.message] : []
);
}
/** Keep match boundaries alongside messages, including when the log clock resets. */
@@ -89,8 +97,16 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde
if (!match) continue;
const message = plainText(match[5]);
if (!message) continue;
push({ type: 'message', message: { time: match[1], guid: match[2], slot: Number(match[3]),
player: plainText(match[4]) || 'Unknown player', message } });
push({
type: 'message',
message: {
time: match[1],
guid: match[2],
slot: Number(match[3]),
player: plainText(match[4]) || 'Unknown player',
message
}
});
}
return entries;
}
@@ -100,11 +116,14 @@ function indexedTimeline(lines: readonly string[]): { entry: ChatEntry; lineInde
* segment to the file's last write, then join observed gameplay at that time.
* Resets have unknown downtime, so earlier segments cannot share this anchor.
*/
export function enrichGameChatTimeline(lines: readonly string[], options: {
export function enrichGameChatTimeline(
lines: readonly string[],
options: {
mtime: string | null;
matches: readonly ChatMatch[];
reports: readonly ChatRoundReport[];
}): ContextualChatEntry[] {
}
): ContextualChatEntry[] {
let segmentStart = 0;
let previousSeconds: number | null = null;
let lastLine = -1;
@@ -115,30 +134,43 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
const event = line.slice(timestamp?.[0].length ?? 0).trimStart();
const lifecycle = /^(?:InitGame|ExitLevel|ShutdownGame)(?:\s|:|$)/.test(event);
if (!timestamp) {
if (lifecycle) { segmentStart = index + 1; previousSeconds = null; }
if (lifecycle) {
segmentStart = index + 1;
previousSeconds = null;
}
continue;
}
const seconds = logSeconds(timestamp[1]);
if (previousSeconds !== null && (seconds < previousSeconds
|| (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))) segmentStart = index;
if (
previousSeconds !== null &&
(seconds < previousSeconds || (seconds === 0 && /^InitGame(?:\s|:|$)/.test(event)))
)
segmentStart = index;
previousSeconds = seconds;
}
// An untimed final write cannot locate the preceding timed event in real time.
const finalTimestamp = lastLine < 0 ? null : logTimestamp(lines[lastLine]);
const writtenAt = options.mtime ? Date.parse(options.mtime) : NaN;
const offset = finalTimestamp ? writtenAt - logSeconds(finalTimestamp[1]) * 1000 : NaN;
const reports = options.reports.map(report => ({ ...report, at: Date.parse(report.receivedAt) }))
const reports = options.reports
.map((report) => ({ ...report, at: Date.parse(report.receivedAt) }))
.sort((a, b) => b.at - a.at);
const matches = options.matches.map(match => ({ ...match, start: Date.parse(match.startedAt) }))
const matches = options.matches
.map((match) => ({ ...match, start: Date.parse(match.startedAt) }))
.sort((a, b) => b.start - a.start)
.map((match, index, sorted) => {
const nextStart = sorted[index - 1]?.start ?? Infinity;
// Identical heartbeats are deduplicated by the leaderboard, but still
// establish that this match was running when the message was sent.
const latest = reports.find(report => report.at >= match.start && report.at < nextStart);
const latest = reports.find(
(report) => report.at >= match.start && report.at < nextStart
);
const lastSeen = Math.max(Date.parse(match.lastReceivedAt), latest?.at ?? -Infinity);
const end = match.endedAt ? Date.parse(match.endedAt)
: match.status === 'active' ? lastSeen + 120_000 : Date.parse(match.lastReceivedAt);
const end = match.endedAt
? Date.parse(match.endedAt)
: match.status === 'active'
? lastSeen + 120_000
: Date.parse(match.lastReceivedAt);
return { ...match, end: Math.min(end, nextStart) };
});
let logStartedAt: number | null = null;
@@ -147,8 +179,10 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
return indexedTimeline(lines).map(({ entry, lineIndex }) => {
const context: ChatContext = { realTime: null, matchElapsedMs: null, round: null };
const time = entry.type === 'message' ? entry.message.time : entry.time;
const at = time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN;
if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime())) context.realTime = new Date(at).toISOString();
const at =
time !== null && lineIndex >= segmentStart ? offset + logSeconds(time) * 1000 : NaN;
if (Number.isFinite(at) && Number.isFinite(new Date(at).getTime()))
context.realTime = new Date(at).toISOString();
if (entry.type === 'boundary' && entry.event !== 'end') {
// The first event after an implicit reset can be several seconds into
// the new process. Its clock origin is the lower bound, not that event.
@@ -156,20 +190,44 @@ export function enrichGameChatTimeline(lines: readonly string[], options: {
logStartedAt = context.realTime ? (entry.event === 'reset' ? offset : at) : Infinity;
logEnded = false;
}
const match = context.realTime && !logEnded ? matches.find(match => at >= match.start && at <= match.end
const match =
context.realTime && !logEnded
? matches.find(
(match) =>
at >= match.start &&
at <= match.end &&
// Allow the log's one-second precision at a start, but never cross a later InitGame.
&& (logStartedAt === null || match.start + 1000 >= logStartedAt
|| reports.some(report => report.event === 'game_start' && report.playerCount > 0
&& report.at >= match.start && report.at <= at && report.at + 1000 >= logStartedAt!))) : undefined;
(logStartedAt === null ||
match.start + 1000 >= logStartedAt ||
reports.some(
(report) =>
report.event === 'game_start' &&
report.playerCount > 0 &&
report.at >= match.start &&
report.at <= at &&
report.at + 1000 >= logStartedAt!
))
)
: undefined;
if (match) {
const report = reports.find(report => report.at <= at && report.at >= match.start);
const report = reports.find((report) => report.at <= at && report.at >= match.start);
if (!report || (report.event !== 'game_end' && report.playerCount > 0)) {
// A repeated start updates the fleet timer even if the leaderboard
// deduplicates it. Use that same observed start for later messages.
const start = reports.find(report => report.event === 'game_start' && report.playerCount > 0
&& report.at >= match.start && report.at <= at)?.at ?? (match.startObserved ? match.start : null);
const start =
reports.find(
(report) =>
report.event === 'game_start' &&
report.playerCount > 0 &&
report.at >= match.start &&
report.at <= at
)?.at ?? (match.startObserved ? match.start : null);
if (start !== null) context.matchElapsedMs = at - start;
if (report && at - report.at <= 120_000 && (report.event === 'round_change' || report.event === 'heartbeat')) {
if (
report &&
at - report.at <= 120_000 &&
(report.event === 'round_change' || report.event === 'heartbeat')
) {
context.round = report.round;
}
}
src/lib/game-files.ts
+48 -16
View File
@@ -4,33 +4,65 @@ export const GAME_FILE_GAMES = [
{ id: 't6', name: 'Black Ops II' }
] as const;
export type GameLibraryState = 'importing' | 'ready' | 'failed';
export type LibraryCacheState = 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
export type LibraryCacheState =
'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
export interface GameFileEntry { path: string; size: number; sha256: string }
export interface GameFileEntry {
path: string;
size: number;
sha256: string;
}
export interface LibraryTransferProgress {
phase: 'downloading' | 'verifying'; startedAt: string;
elapsedSeconds: number; phaseElapsedSeconds: number;
bytesPerSecond: number | null; etaSeconds: number | null;
phase: 'downloading' | 'verifying';
startedAt: string;
elapsedSeconds: number;
phaseElapsedSeconds: number;
bytesPerSecond: number | null;
etaSeconds: number | null;
}
export interface HostLibraryCache {
hostId: string; libraryVersionId: string; status: LibraryCacheState;
bytesDownloaded: number; bytesTotal: number; manifestSha256: string | null;
error: string | null; updatedAt: string; operationId: string | null;
bytesVerified?: number; transfer?: LibraryTransferProgress;
hostId: string;
libraryVersionId: string;
status: LibraryCacheState;
bytesDownloaded: number;
bytesTotal: number;
manifestSha256: string | null;
error: string | null;
updatedAt: string;
operationId: string | null;
bytesVerified?: number;
transfer?: LibraryTransferProgress;
}
export interface GameLibraryVersion {
id: string; libraryId: string; game: GameLibraryGame; version: string;
description: string; state: GameLibraryState; sizeBytes: number;
fileCount: number; manifestSha256: string | null; files: GameFileEntry[];
createdAt: string; publishedAt: string | null; storageLocation: string;
sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
id: string;
libraryId: string;
game: GameLibraryGame;
version: string;
description: string;
state: GameLibraryState;
sizeBytes: number;
fileCount: number;
manifestSha256: string | null;
files: GameFileEntry[];
createdAt: string;
publishedAt: string | null;
storageLocation: string;
sourceHostId: string | null;
sourceVolume: string | null;
error: string | null;
operationId: string | null;
caches: HostLibraryCache[];
filesRemoved?: boolean;
filesRemoving?: boolean;
}
export interface GameLibrary {
id: string; name: string; game: GameLibraryGame; description: string;
createdAt: string; latestVersionId: string | null; versions: GameLibraryVersion[];
id: string;
name: string;
game: GameLibraryGame;
description: string;
createdAt: string;
latestVersionId: string | null;
versions: GameLibraryVersion[];
}
export function formatGameBytes(bytes: number): string {
src/lib/host-operation-labels.ts
+24 -10
View File
@@ -1,14 +1,28 @@
export const hostOperationTitles: Record<string, string> = {
'instance.create': 'Create server container', 'instance.rebuild': 'Rebuild server container',
'instance.start': 'Start server container', 'instance.restart': 'Restart server container',
'instance.stop': 'Stop server container', 'instance.remove': 'Destroy server container',
'instance.cancel-create': 'Cancel server creation', 'instance.inspect': 'Check server container',
'host.inspect': 'Check host', 'host.decommission': 'Remove host', 'runtime.pull': 'Prepare container runtime',
'library.download': 'Download game files', 'library.import': 'Import game files',
'plutonium.prepare': 'Prepare game update', 'files.stage': 'Stage server files',
'files.apply': 'Apply server files', 'files.read': 'Read server files', 'files.list': 'List server files',
'files.stat': 'Check server files', 'files.manifest': 'Check file versions', 'files.remove': 'Remove server files',
'rcon.status-batch': 'Check game status', 'rcon.execute': 'Game command', 'logs.read': 'Read server logs'
'instance.create': 'Create server container',
'instance.rebuild': 'Rebuild server container',
'instance.start': 'Start server container',
'instance.restart': 'Restart server container',
'instance.stop': 'Stop server container',
'instance.remove': 'Destroy server container',
'instance.cancel-create': 'Cancel server creation',
'instance.inspect': 'Check server container',
'host.inspect': 'Check host',
'host.decommission': 'Remove host',
'runtime.pull': 'Prepare container runtime',
'library.download': 'Download game files',
'library.import': 'Import game files',
'plutonium.prepare': 'Prepare game update',
'files.stage': 'Stage server files',
'files.apply': 'Apply server files',
'files.read': 'Read server files',
'files.list': 'List server files',
'files.stat': 'Check server files',
'files.manifest': 'Check file versions',
'files.remove': 'Remove server files',
'rcon.status-batch': 'Check game status',
'rcon.execute': 'Game command',
'logs.read': 'Read server logs'
};
export function hostOperationTitle(kind: string): string {
src/lib/hosts.ts
+204 -58
View File
@@ -1,31 +1,67 @@
import type { HostLibraryCache } from './game-files.js';
export type HostMode = 'active' | 'draining' | 'disabled';
export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export type HostReadiness =
| 'preparing'
| 'needs-runtime'
| 'needs-game-files'
| 'ready'
| 'degraded'
| 'draining'
| 'disabled'
| 'deleting';
export type HostOperationState =
'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export function hostReadinessLabel(readiness: HostReadiness): string {
return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
return {
preparing: 'Preparing',
'needs-runtime': 'Needs runtime',
'needs-game-files': 'Needs game files',
ready: 'Ready',
degraded: 'Degraded',
draining: 'Draining',
disabled: 'Disabled',
deleting: 'Deleting'
}[readiness];
}
export interface HostDeletion { allowed: boolean; blockers: string[] }
export interface HostDeletion {
allowed: boolean;
blockers: string[];
}
export interface HostDecommission {
operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
operationId: string;
stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
requestedAt: string;
updatedAt: string;
error: string | null;
retryable: boolean;
}
export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
if (!host.decommission) return '';
switch (host.decommission.stage) {
case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
case 'completed': return 'Host files and agent removal confirmed.';
case 'queued':
return host.connected
? 'Host cleanup is queued.'
: 'Waiting for the host to reconnect before removing its files and agent.';
case 'running':
return 'Removing host files, cached game libraries, and the agent installation.';
case 'cleaned':
return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
case 'failed':
return 'Cleanup needs attention. Complete removal has not been confirmed.';
case 'completed':
return 'Host files and agent removal confirmed.';
}
}
export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
export interface HostLibrary {
id: string;
state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated';
progress?: number;
error?: string;
}
export interface HostAgentUpdate {
enabled: boolean;
state: 'current' | 'checking' | 'deferred' | 'updating' | 'updated' | 'failed';
@@ -37,50 +73,93 @@ export interface HostAgentUpdate {
export interface HostAgentLane {
name: 'health-1' | 'health-2' | 'read' | 'write';
alive: boolean; busy: boolean; lastPollAt: string | null; pollAgeSeconds: number;
alive: boolean;
busy: boolean;
lastPollAt: string | null;
pollAgeSeconds: number;
}
export function hostAgentLaneLabel(name: HostAgentLane['name']): string {
return { 'health-1': 'Health check 1', 'health-2': 'Health check 2', read: 'File reads', write: 'Actions' }[name];
return {
'health-1': 'Health check 1',
'health-2': 'Health check 2',
read: 'File reads',
write: 'Actions'
}[name];
}
export function hostAgentLaneState(lane: HostAgentLane): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
export function hostAgentLaneState(
lane: HostAgentLane
): 'stopped' | 'stalled' | 'busy' | 'starting' | 'polling' {
if (!lane.alive) return 'stopped';
// Reads and mutations can legitimately take minutes. Health work has a
// bounded budget; an old poll there indicates a blocked worker even if busy.
if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-'))) return 'stalled';
if (lane.pollAgeSeconds > 90 && (!lane.busy || lane.name.startsWith('health-')))
return 'stalled';
return lane.busy ? 'busy' : lane.lastPollAt === null ? 'starting' : 'polling';
}
export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean {
return inventory.agentUpdate?.enabled === true && inventory.capabilities?.includes('agent.update') === true;
return (
inventory.agentUpdate?.enabled === true &&
inventory.capabilities?.includes('agent.update') === true
);
}
export function hostAgentUpdateLabel(inventory: HostInventory): string {
if (!hostAgentUpdatesEnabled(inventory)) return 'Setup required';
const update = inventory.agentUpdate!;
if (update.state === 'current' && !update.checkedAt) return 'Awaiting first check';
return { current: 'Current', checking: 'Checking', deferred: 'Deferred',
updating: 'Updating', updated: 'Updated', failed: 'Needs attention' }[update.state];
return {
current: 'Current',
checking: 'Checking',
deferred: 'Deferred',
updating: 'Updating',
updated: 'Updated',
failed: 'Needs attention'
}[update.state];
}
export function hostAgentUpdateMessage(inventory: HostInventory): string {
if (!hostAgentUpdatesEnabled(inventory)) return 'Run the current installer once on this host to enable automatic agent updates.';
if (!hostAgentUpdatesEnabled(inventory))
return 'Run the current installer once on this host to enable automatic agent updates.';
const update = inventory.agentUpdate!;
switch (update.state) {
case 'current': return update.checkedAt ? 'The last check found no newer compatible agent release.' : 'The host timer will check for a compatible agent release automatically.';
case 'checking': return 'The host is checking the controller release and verifying its download.';
case 'deferred': return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
case 'updating': return 'The host is replacing agent code and checking that the new process connects successfully.';
case 'updated': return 'The new agent connected successfully. Automatic checks continue.';
case 'failed': return 'The update did not complete successfully. Review the reported error before retrying setup.';
case 'current':
return update.checkedAt
? 'The last check found no newer compatible agent release.'
: 'The host timer will check for a compatible agent release automatically.';
case 'checking':
return 'The host is checking the controller release and verifying its download.';
case 'deferred':
return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
case 'updating':
return 'The host is replacing agent code and checking that the new process connects successfully.';
case 'updated':
return 'The new agent connected successfully. Automatic checks continue.';
case 'failed':
return 'The update did not complete successfully. Review the reported error before retrying setup.';
}
}
export interface HostInventory {
hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string;
cpu?: string; cpuCount?: number; cpuUsagePercent?: number; memoryTotal?: number; memoryAvailable?: number;
storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
currentOperation?: string | null; [key: string]: unknown;
hostname?: string;
agentVersion?: string;
protocolVersion?: number;
os?: string;
architecture?: string;
cpu?: string;
cpuCount?: number;
cpuUsagePercent?: number;
memoryTotal?: number;
memoryAvailable?: number;
storageTotal?: number;
storageAvailable?: number;
dockerVersion?: string;
dockerRunning?: boolean;
directoriesWritable?: boolean;
runtimeImages?: string[];
libraries?: HostLibrary[];
currentOperation?: string | null;
[key: string]: unknown;
capabilities?: string[];
plutoniumCache?: { revision: number; bytes: number };
agentUpdate?: HostAgentUpdate;
@@ -90,37 +169,82 @@ export interface HostInventory {
instanceStates?: InstanceState[];
}
export interface InstanceState {
instanceId: string; generation: number; state: string; healthy: boolean; observedAt: string;
instanceId: string;
generation: number;
state: string;
healthy: boolean;
observedAt: string;
}
export interface InstanceResources {
instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;
instanceId: string;
generation: number;
uptimeSeconds: number;
cpuCores: number;
memoryBytes: number;
}
export interface ResourceAllowance {
cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap';
samples: number; observedAt: string | null;
cpuCores: number;
memoryBytes: number;
source: 'host' | 'fleet' | 'bootstrap';
samples: number;
observedAt: string | null;
}
export interface CapacityCalibration {
running: ResourceAllowance;
startup: ResourceAllowance;
}
export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance }
export interface CapacitySample {
hostId: string; hostName: string | null; instanceId: string; server: string | null;
generation: number; phase: 'startup' | 'running'; timestamp: string; cpuCores: number; memoryBytes: number;
hostId: string;
hostName: string | null;
instanceId: string;
server: string | null;
generation: number;
phase: 'startup' | 'running';
timestamp: string;
cpuCores: number;
memoryBytes: number;
}
export interface CapacitySamplePage {
samples: CapacitySample[]; total: number; page: number; pageSize: number;
samples: CapacitySample[];
total: number;
page: number;
pageSize: number;
}
export interface Host {
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string;
lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
enrolled: boolean; credentialActive: boolean;
id: string;
name: string;
hostname: string;
mode: HostMode;
portMin: number;
portMax: number;
bindAddress: string;
publicAddress: string;
minimumFreeBytes: number;
createdAt: string;
lastHeartbeat: string | null;
connected: boolean;
connectionState: 'online' | 'offline';
readiness: HostReadiness;
readinessReasons: string[];
inventory: HostInventory;
enrolled: boolean;
credentialActive: boolean;
decommission: HostDecommission | null;
preparation?: HostPreparation;
enablingSince?: string | null;
}
export interface HostOperation {
id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;
payload: Record<string, unknown>; state: HostOperationState; result: unknown; error: string | null;
createdAt: string; updatedAt: string;
id: string;
hostId: string;
kind: string;
instanceId: string | null;
generation: number | null;
payload: Record<string, unknown>;
state: HostOperationState;
result: unknown;
error: string | null;
createdAt: string;
updatedAt: string;
}
export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'result'> {
hostName: string | null;
@@ -130,11 +254,16 @@ export interface HostOperationSummary extends Omit<HostOperation, 'payload' | 'r
routine: boolean;
}
export type HostOperationFilter = 'important' | 'routine' | 'all';
export interface HostOperationScope { host?: string; server?: string; operation?: string }
export interface HostOperationScope {
host?: string;
server?: string;
operation?: string;
}
export function hostOperationScope(params: URLSearchParams): HostOperationScope {
const scope: HostOperationScope = {};
for (const key of ['host', 'server', 'operation'] as const) {
const value = params.get(key);
// eslint-disable-next-line no-control-regex -- Reject control characters in operation scope filters.
if (value && value.length <= 512 && !/[\x00-\x1f\x7f]/.test(value)) scope[key] = value;
}
return scope;
@@ -143,28 +272,45 @@ export function isHostOperationFilter(value: unknown): value is HostOperationFil
return typeof value === 'string' && ['important', 'routine', 'all'].includes(value);
}
export interface HostOperationHistoryPage {
operations: HostOperationSummary[]; page: number; pageSize: 25; total: number; totalPages: number; filter: HostOperationFilter;
operations: HostOperationSummary[];
page: number;
pageSize: 25;
total: number;
totalPages: number;
filter: HostOperationFilter;
}
export interface HostPreparationItem {
kind: 'runtime' | 'library' | 'plutonium'; label: string;
state: 'waiting' | 'preparing' | 'ready' | 'blocked'; detail: string;
operationId?: string; bytesDownloaded?: number; bytesTotal?: number;
kind: 'runtime' | 'library' | 'plutonium';
label: string;
state: 'waiting' | 'preparing' | 'ready' | 'blocked';
detail: string;
operationId?: string;
bytesDownloaded?: number;
bytesTotal?: number;
cache?: HostLibraryCache;
}
export interface HostPreparation {
state: HostPreparationItem['state']; items: HostPreparationItem[];
state: HostPreparationItem['state'];
items: HostPreparationItem[];
}
export interface NativeDeploymentSelection {
libraryIds: Partial<Record<'t5' | 't6', string>>; image: string;
bindAddress?: string; publicAddress?: string; publicPort?: number;
libraryIds: Partial<Record<'t5' | 't6', string>>;
image: string;
bindAddress?: string;
publicAddress?: string;
publicPort?: number;
}
/** Queued automatic launches have no assigned host or port until provisioning. */
export interface NativePlacement {
hostId?: string; libraryId: string; image?: string;
bindAddress?: string; publicAddress?: string; publicPort?: number;
hostId?: string;
libraryId: string;
image?: string;
bindAddress?: string;
publicAddress?: string;
publicPort?: number;
automatic?: boolean;
network?: { bindAddress?: string; publicAddress?: string; publicPort?: number };
}
src/lib/leaderboard-efficiency.ts
+52 -22
View File
@@ -5,20 +5,40 @@ export const EFFICIENCY_MIN_KILLS = 100;
export const EFFICIENCY_MIN_DOWNS = 5;
export const efficiencyMetrics = [
{ key: 'headshotRate', label: 'Headshots / kills', format: 'percent',
description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.` },
{ key: 'killsPerDown', label: 'Kills per down', format: 'number',
description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.` },
{ key: 'averageKills', label: 'Average kills', format: 'number',
description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` },
{ key: 'averageRounds', label: 'Average rounds', format: 'number',
description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.` },
{ key: 'completionRate', label: 'Completion rate', format: 'percent',
description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.` }
{
key: 'headshotRate',
label: 'Headshots / kills',
format: 'percent',
description: `Total headshots divided by total kills in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_KILLS} kills.`
},
{
key: 'killsPerDown',
label: 'Kills per down',
format: 'number',
description: `Total kills divided by total downs in ended games. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games and ${EFFICIENCY_MIN_DOWNS} downs.`
},
{
key: 'averageKills',
label: 'Average kills',
format: 'number',
description: `Total kills divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.`
},
{
key: 'averageRounds',
label: 'Average rounds',
format: 'number',
description: `Total observed rounds divided by games played, including games the player left before they ended. Ranking requires at least ${EFFICIENCY_MIN_GAMES} ended games.`
},
{
key: 'completionRate',
label: 'Completion rate',
format: 'percent',
description: `Games with the player in the final report divided by games played. Only ended games count, including games the player left early. Ranking requires at least ${EFFICIENCY_MIN_GAMES} games.`
}
] as const;
export type EfficiencyMetric = typeof efficiencyMetrics[number]['key'];
const metrics = new Set<string>(efficiencyMetrics.map(metric => metric.key));
export type EfficiencyMetric = (typeof efficiencyMetrics)[number]['key'];
const metrics = new Set<string>(efficiencyMetrics.map((metric) => metric.key));
export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric {
return metrics.has(metric);
@@ -26,11 +46,16 @@ export function isEfficiencyMetric(metric: string): metric is EfficiencyMetric {
function sample(stats: PlayerStats, metric: EfficiencyMetric): [number, number] {
switch (metric) {
case 'headshotRate': return [stats.headshots, stats.kills];
case 'killsPerDown': return [stats.kills, stats.downs];
case 'averageKills': return [stats.kills, stats.gamesPlayed];
case 'averageRounds': return [stats.roundsSurvived, stats.gamesPlayed];
case 'completionRate': return [stats.gamesCompleted, stats.gamesPlayed];
case 'headshotRate':
return [stats.headshots, stats.kills];
case 'killsPerDown':
return [stats.kills, stats.downs];
case 'averageKills':
return [stats.kills, stats.gamesPlayed];
case 'averageRounds':
return [stats.roundsSurvived, stats.gamesPlayed];
case 'completionRate':
return [stats.gamesCompleted, stats.gamesPlayed];
}
}
@@ -60,11 +85,16 @@ const count = (value: number, singular: string, plural = `${singular}s`) =>
export function efficiencySample(stats: PlayerStats, metric: EfficiencyMetric): string {
switch (metric) {
case 'headshotRate': return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`;
case 'killsPerDown': return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`;
case 'averageKills': return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'averageRounds': return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'completionRate': return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'headshotRate':
return `${count(stats.headshots, 'headshot')} / ${count(stats.kills, 'kill')}`;
case 'killsPerDown':
return `${count(stats.kills, 'kill')} / ${count(stats.downs, 'down')}`;
case 'averageKills':
return `${count(stats.kills, 'kill')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'averageRounds':
return `${count(stats.roundsSurvived, 'observed round')} / ${count(stats.gamesPlayed, 'ended game')}`;
case 'completionRate':
return `${count(stats.gamesCompleted, 'finish', 'finishes')} / ${count(stats.gamesPlayed, 'ended game')}`;
}
}
src/lib/leaderboard-navigation.ts
+6 -3
View File
@@ -12,10 +12,12 @@ export function leaderboardReturnHref(value: string | null | undefined): string
const params = new URLSearchParams();
for (const key of contextKeys) {
const entry = url.searchParams.get(key);
if (entry || key === 'map' && entry !== null) params.set(key, entry!);
if (entry || (key === 'map' && entry !== null)) params.set(key, entry!);
}
return `/leaderboard${params.size ? `?${params}` : ''}`;
} catch { return '/leaderboard'; }
} catch {
return '/leaderboard';
}
}
export function playerContextHref(guid: string, source: string): string {
@@ -31,7 +33,8 @@ export function playerContextHref(guid: string, source: string): string {
if (game === 't5' || game === 't6') params.set('game', game);
if (map !== null && map.length <= 1024) params.set('map', map);
if (players && /^[1-4]$/.test(players)) params.set('players', players);
if (metric && leaderboardMetrics.some(item => item.key === metric)) params.set('metric', metric);
if (metric && leaderboardMetrics.some((item) => item.key === metric))
params.set('metric', metric);
return `/leaderboard/players/${encodeURIComponent(guid)}?${params}`;
}
src/lib/leaderboard.ts
+2 -2
View File
@@ -14,8 +14,8 @@ export const playerStatMetrics = [
] as const;
export const leaderboardMetrics = [...playerStatMetrics, ...efficiencyMetrics] as const;
export type LeaderboardMetric = typeof leaderboardMetrics[number]['key'];
export type PlayerStatMetric = typeof playerStatMetrics[number]['key'];
export type LeaderboardMetric = (typeof leaderboardMetrics)[number]['key'];
export type PlayerStatMetric = (typeof playerStatMetrics)[number]['key'];
export type PlayerStats = Record<PlayerStatMetric, number>;
export type PersonalBestMetric = 'highestRound' | 'bestKills' | 'bestScore';
src/lib/maintenance.ts
+6 -1
View File
@@ -2,7 +2,12 @@ import type { WatchdogOverview } from './watchdog.js';
import type { PlutoniumStatus } from './plutonium.js';
export type MaintenancePolicy = 'health' | 'drift' | 'updates';
export interface MaintenancePolicies { paused: boolean; health: boolean; drift: boolean; updates: boolean }
export interface MaintenancePolicies {
paused: boolean;
health: boolean;
drift: boolean;
updates: boolean;
}
export interface MaintenanceOverview {
policies: MaintenancePolicies;
recovery: WatchdogOverview;
src/lib/map-aliases.ts
+67 -20
View File
@@ -36,14 +36,54 @@ export const T5_ZOMBIE_MAPS: readonly T5MapEntry[] = Object.freeze([
// Rulesets from xerxes-at/T6ServerConfigs, also seeded by Plutainer.
// Several selections share an engine map; slug preserves the selected variant.
export const T6_ZOMBIE_MAPS: readonly ZombieMapEntry[] = Object.freeze([
{ friendly: 'TranZit', slug: 'transit', canonical: 'zm_transit', gameSettings: 'zm_classic_transit.cfg' },
{ friendly: 'Town Survival', slug: 'town', canonical: 'zm_transit', gameSettings: 'zm_standard_town.cfg' },
{ friendly: 'Farm Survival', slug: 'farm', canonical: 'zm_transit', gameSettings: 'zm_standard_farm.cfg' },
{ friendly: 'Bus Depot Survival', slug: 'busdepot', canonical: 'zm_transit', gameSettings: 'zm_standard_transit.cfg' },
{ friendly: 'Nuketown', slug: 'nuked', canonical: 'zm_nuked', gameSettings: 'zm_standard_nuked.cfg' },
{ friendly: 'Die Rise', slug: 'highrise', canonical: 'zm_highrise', gameSettings: 'zm_classic_rooftop.cfg' },
{ friendly: 'Mob of the Dead', slug: 'prison', canonical: 'zm_prison', gameSettings: 'zm_classic_prison.cfg' },
{ friendly: 'Buried', slug: 'buried', canonical: 'zm_buried', gameSettings: 'zm_classic_processing.cfg' },
{
friendly: 'TranZit',
slug: 'transit',
canonical: 'zm_transit',
gameSettings: 'zm_classic_transit.cfg'
},
{
friendly: 'Town Survival',
slug: 'town',
canonical: 'zm_transit',
gameSettings: 'zm_standard_town.cfg'
},
{
friendly: 'Farm Survival',
slug: 'farm',
canonical: 'zm_transit',
gameSettings: 'zm_standard_farm.cfg'
},
{
friendly: 'Bus Depot Survival',
slug: 'busdepot',
canonical: 'zm_transit',
gameSettings: 'zm_standard_transit.cfg'
},
{
friendly: 'Nuketown',
slug: 'nuked',
canonical: 'zm_nuked',
gameSettings: 'zm_standard_nuked.cfg'
},
{
friendly: 'Die Rise',
slug: 'highrise',
canonical: 'zm_highrise',
gameSettings: 'zm_classic_rooftop.cfg'
},
{
friendly: 'Mob of the Dead',
slug: 'prison',
canonical: 'zm_prison',
gameSettings: 'zm_classic_prison.cfg'
},
{
friendly: 'Buried',
slug: 'buried',
canonical: 'zm_buried',
gameSettings: 'zm_classic_processing.cfg'
},
{ friendly: 'Origins', slug: 'tomb', canonical: 'zm_tomb', gameSettings: 'zm_classic_tomb.cfg' }
]);
@@ -55,23 +95,29 @@ export function zombieMapBySlug(game: string, slug: string): ZombieMapEntry | un
return zombieMaps(game).find((entry) => entry.slug === slug.toLowerCase());
}
export function selectedZombieMap(game: string | null, map: string | null, slug?: string | null): ZombieMapEntry | undefined {
export function selectedZombieMap(
game: string | null,
map: string | null,
slug?: string | null
): ZombieMapEntry | undefined {
const maps = zombieMaps(game);
// T5 maps are unambiguous, including legacy friendly slugs. T6 variants
// require the stored selection; never fall back from an unknown variant.
return maps.find((entry) => entry.canonical === map && (
game !== 't6' || !slug || entry.slug === slug
|| entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug
));
return maps.find(
(entry) =>
entry.canonical === map &&
(game !== 't6' ||
!slug ||
entry.slug === slug ||
entry.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === slug)
);
}
export function mapRotation(entry: ZombieMapEntry): string {
return `${entry.gameSettings ? `execgts ${entry.gameSettings} ` : ''}map ${entry.canonical}`;
}
const BY_SLUG: ReadonlyMap<string, T5MapEntry> = new Map(
T5_ZOMBIE_MAPS.map((m) => [m.slug, m])
);
const BY_SLUG: ReadonlyMap<string, T5MapEntry> = new Map(T5_ZOMBIE_MAPS.map((m) => [m.slug, m]));
export function t5ZombieMapBySlug(slug: string): T5MapEntry | undefined {
return BY_SLUG.get(slug.toLowerCase());
@@ -83,7 +129,7 @@ export interface ParsedServerName {
game: string; // 't5'
gameMode: GameMode | null; // 'zombie' — null when the second segment isn't a known mode
mapSlug: string; // 'cosmodrome' — joined without separators for lookup
mapCanonical: string | null;// 'zombie_cosmodrome' — null when slug not in catalog
mapCanonical: string | null; // 'zombie_cosmodrome' — null when slug not in catalog
lobby: number | null;
}
@@ -122,9 +168,10 @@ export function parseServerName(name: string): ParsedServerName | null {
const mapSlug = slugParts.join('').toLowerCase();
const catalog = zombieMaps(game.toLowerCase());
const entry = gameMode === null || gameMode === 'zombie'
? catalog.find((m) => m.slug === mapSlug)
?? catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug)
const entry =
gameMode === null || gameMode === 'zombie'
? (catalog.find((m) => m.slug === mapSlug) ??
catalog.find((m) => m.friendly.toLowerCase().replace(/[^a-z0-9]/g, '') === mapSlug))
: undefined;
const mapCanonical = entry?.canonical ?? null;
return { game, gameMode, mapSlug, mapCanonical, lobby };
src/lib/match-history-query.ts
+47 -11
View File
@@ -1,12 +1,26 @@
export const matchPageSize = 25;
export const matchSorts = ['started', 'round', 'duration', 'players'] as const;
export type MatchSort = typeof matchSorts[number];
export type MatchSort = (typeof matchSorts)[number];
export interface MatchHistoryQuery {
game: string; map: string; q: string; days: number;
game: string;
map: string;
q: string;
days: number;
status: '' | 'active' | 'completed' | 'interrupted';
sort: MatchSort; direction: 'asc' | 'desc'; page: number;
sort: MatchSort;
direction: 'asc' | 'desc';
page: number;
}
export const matchQueryKeys = ['game', 'map', 'q', 'days', 'status', 'sort', 'direction', 'page'] as const;
export const matchQueryKeys = [
'game',
'map',
'q',
'days',
'status',
'sort',
'direction',
'page'
] as const;
export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery {
const game = params.get('game') ?? '';
const map = params.get('map') ?? '';
@@ -16,11 +30,29 @@ export function readMatchQuery(params: URLSearchParams): MatchHistoryQuery {
const sort = params.get('sort') || 'started';
const direction = params.get('direction') || 'desc';
const page = Number(params.get('page') || 1);
if (!['', 't5', 't6'].includes(game) || map.length > 1024 || q.length > 100
|| ![0, 7, 30, 90].includes(days) || !['', 'active', 'completed', 'interrupted'].includes(status)
|| !matchSorts.includes(sort as MatchSort) || !['asc', 'desc'].includes(direction)
|| !Number.isSafeInteger(page) || page < 1 || page > 1000000) throw new Error('Invalid match history filters.');
return { game, map, q, days, status: status as MatchHistoryQuery['status'], sort: sort as MatchSort, direction: direction as 'asc' | 'desc', page };
if (
!['', 't5', 't6'].includes(game) ||
map.length > 1024 ||
q.length > 100 ||
![0, 7, 30, 90].includes(days) ||
!['', 'active', 'completed', 'interrupted'].includes(status) ||
!matchSorts.includes(sort as MatchSort) ||
!['asc', 'desc'].includes(direction) ||
!Number.isSafeInteger(page) ||
page < 1 ||
page > 1000000
)
throw new Error('Invalid match history filters.');
return {
game,
map,
q,
days,
status: status as MatchHistoryQuery['status'],
sort: sort as MatchSort,
direction: direction as 'asc' | 'desc',
page
};
}
/** Namespaced filters also work inside an instance's Matches tab. */
export function matchApiParams(url: URL): URLSearchParams {
@@ -31,13 +63,17 @@ export function matchApiParams(url: URL): URLSearchParams {
}
return params;
}
export function matchHistoryHref(url: URL, changes: Partial<Record<keyof MatchHistoryQuery, string>>): string {
export function matchHistoryHref(
url: URL,
changes: Partial<Record<keyof MatchHistoryQuery, string>>
): string {
const next = new URL(url);
next.searchParams.delete('matches'); // Retire the former growing-list limit.
next.searchParams.delete('matchPage');
for (const [key, value] of Object.entries(changes)) {
const name = `match${key[0].toUpperCase()}${key.slice(1)}`;
if (value) next.searchParams.set(name, value); else next.searchParams.delete(name);
if (value) next.searchParams.set(name, value);
else next.searchParams.delete(name);
}
return next.pathname + next.search;
}
src/lib/metrics-insights.ts
+68 -24
View File
@@ -23,51 +23,95 @@ export interface MetricRecommendation {
/** Apply the same visible rules to all selected maps, including sparse observations. */
export function metricRecommendations(maps: MapMetrics[]): MetricRecommendation[] {
return maps.map(map => {
return maps
.map((map) => {
const utilization = map.observed > 0 ? map.active / map.observed : null;
const pressure = map.capacityObserved > 0 ? map.full / map.capacityObserved : null;
const knownShare = map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0;
const knownShare =
map.observed + map.unknown > 0 ? map.observed / (map.observed + map.unknown) : 0;
const base = {
game: map.game,
map: map.map,
evidence: `${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` +
(pressure === null ? 'Capacity unavailable.' : `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`),
tentative: map.observed < HOSTING_RULES.minimumEvidenceHours || pressure === null || map.capacityObserved < HOSTING_RULES.minimumEvidenceHours || knownShare < 0.8
evidence:
`${metricDuration(map.active)} with players / ${metricDuration(map.observed)} known server time (${metricPercent(map.active, map.observed)}). ` +
(pressure === null
? 'Capacity unavailable.'
: `All lobbies full ${metricDuration(map.full)} / ${metricDuration(map.capacityObserved)} known map time (${metricPercent(map.full, map.capacityObserved)}).`),
tentative:
map.observed < HOSTING_RULES.minimumEvidenceHours ||
pressure === null ||
map.capacityObserved < HOSTING_RULES.minimumEvidenceHours ||
knownShare < 0.8
};
if (utilization === null) return {
...base, kind: 'capacity' as const, title: map.unknown > 0 ? 'Check occupancy reporting' : 'Let observations accumulate',
reason: map.unknown > 0
if (utilization === null)
return {
...base,
kind: 'capacity' as const,
title:
map.unknown > 0
? 'Check occupancy reporting'
: 'Let observations accumulate',
reason:
map.unknown > 0
? 'Captured status is unknown. Check a server’s telemetry or RCON diagnostics before choosing a hosting change.'
: 'This map has no elapsed observations in the selected window. Wait for another status check or try a wider range.',
outcome: 'Watch for known server time in the next report.', tentative: true
outcome: 'Watch for known server time in the next report.',
tentative: true
};
if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion) return {
...base, kind: 'expand' as const, title: 'Trial one additional lobby',
if (pressure !== null && pressure >= HOSTING_RULES.pressureExpansion)
return {
...base,
kind: 'expand' as const,
title: 'Trial one additional lobby',
reason: 'Recorded lobbies were all full for at least 20% of known capacity time. Try one extra lobby during busy hours.',
outcome: 'Compare all-full time and use of the additional lobby over the next comparable period.'
outcome:
'Compare all-full time and use of the additional lobby over the next comparable period.'
};
if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion) return {
...base, kind: 'expand' as const, title: 'Trial an extra lobby during busy hours',
if (pressure === null && utilization >= HOSTING_RULES.occupancyExpansion)
return {
...base,
kind: 'expand' as const,
title: 'Trial an extra lobby during busy hours',
reason: 'Players were present for at least 80% of known server time. Capacity is unknown, so this experiment uses occupancy as a proxy, not evidence of saturation.',
outcome: 'Check whether players use the extra lobby, and measure capacity if the server reports it.', tentative: true
outcome:
'Check whether players use the extra lobby, and measure capacity if the server reports it.',
tentative: true
};
if (utilization <= HOSTING_RULES.idleUtilization && (pressure === null || pressure < HOSTING_RULES.quietPressure)) return {
...base, kind: 'consolidate' as const, title: 'Trial less idle hosting',
if (
utilization <= HOSTING_RULES.idleUtilization &&
(pressure === null || pressure < HOSTING_RULES.quietPressure)
)
return {
...base,
kind: 'consolidate' as const,
title: 'Trial less idle hosting',
reason: 'Players were present for at most 10% of known time, with less than 1% measured pressure or unavailable capacity. Review one idle lobby or a shorter hosting window; preserve access to this map.',
outcome: 'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.'
outcome:
'Watch for increased all-full time or loss of player-hours, and restore the lineup if the trial hurts availability.'
};
if (pressure === null) return {
...base, kind: 'capacity' as const, title: 'Check capacity during busy hours',
if (pressure === null)
return {
...base,
kind: 'capacity' as const,
title: 'Check capacity during busy hours',
reason: 'Occupancy is recorded, but player limits are unavailable. Check capacity reporting while keeping the current lineup.',
outcome: 'Look for known-capacity time and all-full periods before the next hosting trial.'
outcome:
'Look for known-capacity time and all-full periods before the next hosting trial.'
};
return {
...base, kind: 'watch' as const, title: 'Keep the lineup and watch peaks',
...base,
kind: 'watch' as const,
title: 'Keep the lineup and watch peaks',
reason: 'These observations do not trigger the expansion or idle-hosting rules. Keep the current lineup for another comparison period.',
outcome: 'Watch whether occupied time and measured capacity pressure change.'
};
}).sort((a, b) => {
})
.sort((a, b) => {
const priority = { expand: 0, consolidate: 1, capacity: 2, watch: 3 };
return priority[a.kind] - priority[b.kind] || a.map.localeCompare(b.map) || a.game.localeCompare(b.game);
return (
priority[a.kind] - priority[b.kind] ||
a.map.localeCompare(b.map) ||
a.game.localeCompare(b.game)
);
});
}
src/lib/metrics.ts
+15 -4
View File
@@ -22,11 +22,22 @@ export function parseMetricsQuery(params: URLSearchParams): MetricsQuery {
const requestedGame = params.get('game')?.trim() ?? '';
const game = /^[a-z0-9_-]{1,32}$/.test(requestedGame) ? requestedGame : null;
const requestedMap = params.get('map');
const map = game && requestedMap && requestedMap.trim() && requestedMap.length <= 200 && !/[\u0000-\u001f\u007f]/.test(requestedMap)
? requestedMap : null;
const map =
game &&
requestedMap &&
requestedMap.trim() &&
requestedMap.length <= 200 &&
// eslint-disable-next-line no-control-regex -- Reject control characters in requested map names.
!/[\u0000-\u001f\u007f]/.test(requestedMap)
? requestedMap
: null;
const requestedZone = params.get('timeZone');
return { days: parseMetricsDays(params.get('days')), game, map,
timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE };
return {
days: parseMetricsDays(params.get('days')),
game,
map,
timeZone: validTimeZone(requestedZone) ? requestedZone : DEFAULT_TIME_ZONE
};
}
/** All durations are hours. Observed is known server time, not collection coverage. */
src/lib/navigation-context.ts
+38 -6
View File
@@ -1,14 +1,25 @@
const origin = 'https://electric-cherry.invalid';
const pagePath = /^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/;
const pagePath =
/^\/(?:$|activity|operations|autoscaling|account|capacity|metrics|server-keys|settings(?:\/users)?|servers\/[^/]+|hosts(?:\/[^/]+)?|collections(?:\/launches\/[^/]+)?|(?:scripts|configs|plugins|betas|campaigns|maintenance|provisioning|matches)(?:\/[^/]+)?|game-files|leaderboard(?:\/players\/[^/]+(?:\/[^/]+)?)?)$/;
/** Only return to application pages, never an external URL or a mutation/API endpoint. */
export function safeReturnHref(value: string | null | undefined, fallback: string): string {
if (!value || value.length > 12000 || !value.startsWith('/') || value.startsWith('//') || /[\\\x00-\x1f\x7f]/.test(value)) return fallback;
if (
!value ||
value.length > 12000 ||
!value.startsWith('/') ||
value.startsWith('//') ||
// eslint-disable-next-line no-control-regex -- Reject control characters in redirect paths.
/[\\\x00-\x1f\x7f]/.test(value)
)
return fallback;
try {
const url = new URL(value, origin);
if (url.origin !== origin || !pagePath.test(url.pathname)) return fallback;
return url.pathname + url.search + url.hash;
} catch { return fallback; }
} catch {
return fallback;
}
}
export function contextHref(destination: string, source: URL | string): string {
@@ -25,11 +36,32 @@ export function returnLabel(href: string): string {
if (path.startsWith('/servers/')) return 'Back to server';
if (path.startsWith('/leaderboard/players/')) return 'Back to player';
if (path.startsWith('/collections/launches/')) return 'Back to collection launch';
if (path === '/collections' && url.searchParams.get('view') === 'active') return 'Back to active launches';
if (path === '/collections' && url.searchParams.get('view') === 'history') return 'Back to launch history';
if (path === '/collections' && url.searchParams.get('view') === 'active')
return 'Back to active launches';
if (path === '/collections' && url.searchParams.get('view') === 'history')
return 'Back to launch history';
if (path.startsWith('/matches/')) return 'Back to match';
if (path === '/hosts/operations') return 'Back to host operation history';
if (path.startsWith('/hosts/')) return 'Back to host';
const names: Record<string, string> = { '': 'Servers', activity: 'Activity', operations: 'Activity', autoscaling: 'Autoscaling', collections: 'Collections', hosts: 'Hosts', capacity: 'Capacity', scripts: 'Scripts', configs: 'Configs', plugins: 'Plugins', 'game-files': 'Game Files', betas: 'Betas', campaigns: 'Campaigns', maintenance: 'Maintenance', provisioning: 'Server queues', leaderboard: 'Leaderboard', matches: 'Match History', metrics: 'Metrics' };
const names: Record<string, string> = {
'': 'Servers',
activity: 'Activity',
operations: 'Activity',
autoscaling: 'Autoscaling',
collections: 'Collections',
hosts: 'Hosts',
capacity: 'Capacity',
scripts: 'Scripts',
configs: 'Configs',
plugins: 'Plugins',
'game-files': 'Game Files',
betas: 'Betas',
campaigns: 'Campaigns',
maintenance: 'Maintenance',
provisioning: 'Server queues',
leaderboard: 'Leaderboard',
matches: 'Match History',
metrics: 'Metrics'
};
return `Back to ${names[path.split('/')[1]] ?? 'previous page'}`;
}
src/lib/navigation.ts
+85 -21
View File
@@ -1,52 +1,116 @@
import type { UserRole } from './auth.js';
export interface NavigationItem { label: string; href: string; shortcut?: string; matches?: (path: string) => boolean }
export interface NavigationGroup { id: string; label: string; admin?: boolean; items: NavigationItem[] }
export interface NavigationItem {
label: string;
href: string;
shortcut?: string;
matches?: (path: string) => boolean;
}
export interface NavigationGroup {
id: string;
label: string;
admin?: boolean;
items: NavigationItem[];
}
export const navigationGroups: NavigationGroup[] = [
{ id: 'fleet', label: 'Fleet', items: [
{ label: 'Servers', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') },
{ label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) },
{
id: 'fleet',
label: 'Fleet',
items: [
{
label: 'Servers',
href: '/',
shortcut: 'f',
matches: (path) => path === '/' || path.startsWith('/servers/')
},
{
label: 'Hosts',
href: '/hosts',
shortcut: 'o',
matches: (path) =>
path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path)
},
{ label: 'Capacity', href: '/capacity' },
{ label: 'Autoscaling', href: '/autoscaling' },
{ label: 'Collections', href: '/collections' }
] },
{ id: 'releases', label: 'Releases', items: [
]
},
{
id: 'releases',
label: 'Releases',
items: [
{ label: 'Scripts', href: '/scripts', shortcut: 's' },
{ label: 'Configs', href: '/configs', shortcut: 'c' },
{ label: 'Plugins', href: '/plugins', shortcut: 'p' },
{ label: 'Game files', href: '/game-files' },
{ label: 'Betas', href: '/betas', shortcut: 'b' },
{ label: 'Campaigns', href: '/campaigns', shortcut: 'a' }
] },
{ id: 'activity', label: 'Activity', items: [
{ label: 'Activity log', href: '/activity', shortcut: 'd', matches: path => path === '/activity' || path === '/operations' || path === '/hosts/operations' },
{ label: 'Provisioning', href: '/provisioning', matches: path => /^\/(provisioning|deployments)(\/|$)/.test(path) },
{ label: 'Maintenance', href: '/maintenance', shortcut: 'w', matches: path => /^\/(maintenance|watchdog)(\/|$)/.test(path) }
] },
{ id: 'players', label: 'Players', items: [
]
},
{
id: 'activity',
label: 'Activity',
items: [
{
label: 'Activity log',
href: '/activity',
shortcut: 'd',
matches: (path) =>
path === '/activity' || path === '/operations' || path === '/hosts/operations'
},
{
label: 'Provisioning',
href: '/provisioning',
matches: (path) => /^\/(provisioning|deployments)(\/|$)/.test(path)
},
{
label: 'Maintenance',
href: '/maintenance',
shortcut: 'w',
matches: (path) => /^\/(maintenance|watchdog)(\/|$)/.test(path)
}
]
},
{
id: 'players',
label: 'Players',
items: [
{ label: 'Leaderboard', href: '/leaderboard', shortcut: 'l' },
{ label: 'Matches', href: '/matches' },
{ label: 'Usage', href: '/metrics', shortcut: 'm' }
] },
{ id: 'admin', label: 'Admin', admin: true, items: [
]
},
{
id: 'admin',
label: 'Admin',
admin: true,
items: [
{ label: 'Users', href: '/settings/users' },
{ label: 'Keys', href: '/server-keys', shortcut: 'k' },
{ label: 'Settings', href: '/settings', shortcut: 'h', matches: path => path === '/settings' }
] }
{
label: 'Settings',
href: '/settings',
shortcut: 'h',
matches: (path) => path === '/settings'
}
]
}
];
export function visibleNavigation(role: UserRole): NavigationGroup[] {
return navigationGroups.filter(group => !group.admin || role === 'admin');
return navigationGroups.filter((group) => !group.admin || role === 'admin');
}
export function navigationItemActive(item: NavigationItem, path: string): boolean {
return item.matches ? item.matches(path) : path === item.href || path.startsWith(item.href + '/');
return item.matches
? item.matches(path)
: path === item.href || path.startsWith(item.href + '/');
}
export function navigationLocation(path: string, role: UserRole) {
for (const group of visibleNavigation(role)) {
const item = group.items.find(item => navigationItemActive(item, path));
const item = group.items.find((item) => navigationItemActive(item, path));
if (item) return { group, item };
}
return null;
src/lib/operation-flow.ts
+90 -29
View File
@@ -1,19 +1,42 @@
export interface ActivityEvent { id: number; created_at: string; message: string }
export interface OperationStep { label: string; state: 'pending' | 'current' | 'done' | 'failed' }
interface BetaServer {
stage: string; deployed_at: string | null; restarted_at: string | null;
healthy_at: string | null; match_seen_at: string | null;
export interface ActivityEvent {
id: number;
created_at: string;
message: string;
}
export interface OperationStep {
label: string;
state: 'pending' | 'current' | 'done' | 'failed';
}
export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] {
interface BetaServer {
stage: string;
deployed_at: string | null;
restarted_at: string | null;
healthy_at: string | null;
match_seen_at: string | null;
}
export function betaFlow(
beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean },
servers: BetaServer[]
): OperationStep[] {
const stages = ['deploying', 'awaiting-empty', 'restarting'];
const labels = ['Stage file changes', 'Wait for empty servers', 'Apply files and restart'];
if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); }
if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); }
if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); }
stages.push('ready'); labels.push('Review and promote');
const positions = servers.map(s => {
if (beta.wait_for_health) {
stages.push('awaiting-health');
labels.push('Verify health');
}
if (beta.wait_for_match) {
stages.push('awaiting-match');
labels.push('Observe a completed match');
}
if (beta.testPlan) {
stages.push('observing');
labels.push('Evaluate test requirements');
}
stages.push('ready');
labels.push('Review and promote');
const positions = servers.map((s) => {
if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage));
if (!s.deployed_at) return 0;
if (!s.restarted_at) return 2;
@@ -21,29 +44,67 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo
if (beta.wait_for_match && !s.match_seen_at) return stages.indexOf('awaiting-match');
return stages.length - 1;
});
const current = beta.state === 'promoted' ? stages.length
: beta.state === 'ready-promote' ? stages.length - 1
: positions.length ? Math.min(...positions) : 0;
return labels.map((label, i) => ({ label, state: i < current ? 'done' : i === current
? beta.state === 'rolled-back' ? 'pending' : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) ? 'failed' : 'current'
: 'pending' }));
const current =
beta.state === 'promoted'
? stages.length
: beta.state === 'ready-promote'
? stages.length - 1
: positions.length
? Math.min(...positions)
: 0;
return labels.map((label, i) => ({
label,
state:
i < current
? 'done'
: i === current
? beta.state === 'rolled-back'
? 'pending'
: servers.some((s, j) => s.stage === 'failed' && positions[j] === i)
? 'failed'
: 'current'
: 'pending'
}));
}
export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean,
files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] {
const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map(s => {
export function campaignFlow(
servers: {
server?: string;
restart_state: string;
restarted_at: string | null;
files_changed: number;
}[],
finished: boolean,
files: { server: string; snapshot_pending: number; apply_state: string }[] = []
): OperationStep[] {
const active = servers.filter((s) => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map((s) => {
if (s.restart_state === 'done') return 5;
if (s.restarted_at) return 4;
const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled');
if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2;
if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3;
const changes = files.filter((f) => f.server === s.server && f.apply_state !== 'cancelled');
if (changes.some((f) => f.apply_state === 'applying'))
return changes.some((f) => f.snapshot_pending) ? 1 : 2;
if (changes.length && changes.every((f) => f.apply_state === 'applied')) return 3;
return 0;
});
const current = positions.length ? Math.min(...positions) : finished ? 5 : 0;
return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
label, state: i < current ? 'done' : i !== current ? 'pending'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
return [
'Wait for empty servers and restart capacity',
'Capture rollback snapshots',
'Apply file changes',
'Restart servers',
'Verify health'
].map((label, i) => ({
label,
state:
i < current
? 'done'
: i !== current
? 'pending'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i)
? 'failed'
: active.some((s) => ['manual', 'held'].includes(s.restart_state))
? 'pending'
: 'current'
}));
}
src/lib/plutonium.ts
+34 -10
View File
@@ -52,9 +52,16 @@ export const PLUTONIUM_MANIFEST_URL = 'https://cdn.plutoniummod.com/updater/prod
export function parsePlutoniumManifest(value: unknown): number {
const manifest = value as { product?: unknown; revision?: unknown } | null;
if (!manifest || manifest.product !== 'plutonium-core-prod' || typeof manifest.revision !== 'number'
|| !Number.isSafeInteger(manifest.revision) || manifest.revision < 1) {
throw new Error('The production updater manifest did not contain a valid Plutonium revision.');
if (
!manifest ||
manifest.product !== 'plutonium-core-prod' ||
typeof manifest.revision !== 'number' ||
!Number.isSafeInteger(manifest.revision) ||
manifest.revision < 1
) {
throw new Error(
'The production updater manifest did not contain a valid Plutonium revision.'
);
}
return manifest.revision;
}
@@ -66,12 +73,20 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
const [hour, minute] = time.split(':').map(Number);
const target = hour * 60 + minute;
const formatter = new Intl.DateTimeFormat('en-CA', {
timeZone, year: 'numeric', month: '2-digit', day: '2-digit',
hour: '2-digit', minute: '2-digit', hourCycle: 'h23'
timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
hourCycle: 'h23'
});
function local(at: number) {
const parts = Object.fromEntries(formatter.formatToParts(at).map(p => [p.type, p.value]));
return { day: `${parts.year}-${parts.month}-${parts.day}`, minute: Number(parts.hour) * 60 + Number(parts.minute) };
const parts = Object.fromEntries(formatter.formatToParts(at).map((p) => [p.type, p.value]));
return {
day: `${parts.year}-${parts.month}-${parts.day}`,
minute: Number(parts.hour) * 60 + Number(parts.minute)
};
}
const today = local(now.getTime()).day;
const start = Math.floor(now.getTime() / 60_000) * 60_000;
@@ -83,9 +98,13 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
const wall = local(at);
if (wall.day < today || wall.minute < target) continue;
const previous = days.get(wall.day);
if (!previous || wall.minute < previous.minute) days.set(wall.day, { at, minute: wall.minute });
if (!previous || wall.minute < previous.minute)
days.set(wall.day, { at, minute: wall.minute });
}
const next = [...days.values()].map(d => d.at).filter(at => at > now.getTime()).sort((a, b) => a - b)[0];
const next = [...days.values()]
.map((d) => d.at)
.filter((at) => at > now.getTime())
.sort((a, b) => a - b)[0];
if (next === undefined) throw new Error('Could not calculate the next maintenance time.');
return new Date(next).toISOString();
}
@@ -94,5 +113,10 @@ export function nextMaintenanceAt(time: string, now = new Date(), timeZone = 'UT
export function maintenanceTimeInZone(schedule: MaintenanceSchedule, timeZone: string): string {
if (schedule.timeZone === timeZone) return schedule.time;
const at = schedule.nextAt ?? nextMaintenanceAt(schedule.time, new Date(), schedule.timeZone);
return new Intl.DateTimeFormat('en-GB', { timeZone, hour: '2-digit', minute: '2-digit', hourCycle: 'h23' }).format(new Date(at));
return new Intl.DateTimeFormat('en-GB', {
timeZone,
hour: '2-digit',
minute: '2-digit',
hourCycle: 'h23'
}).format(new Date(at));
}
src/lib/provisioning-detail.ts
+40 -8
View File
@@ -1,15 +1,32 @@
import type { ProvisioningAction } from './provisioning.js';
import type { LaunchRecipeSummary } from './deployment-plan.js';
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare',
'container-action', 'cleanup', 'verify-container', 'verify-game',
'verify-map', 'verify-plutonium', 'verify-stopped'] as const;
export type ProvisioningPhase = typeof provisioningPhases[number];
export const provisioningPhases = [
'requested',
'eligibility',
'startup-queue',
'prepare',
'container-action',
'cleanup',
'verify-container',
'verify-game',
'verify-map',
'verify-plutonium',
'verify-stopped'
] as const;
export type ProvisioningPhase = (typeof provisioningPhases)[number];
export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown';
export type EvidenceSource = 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null;
export interface ProvisioningCheck {
key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped';
key:
| 'container'
| 'game'
| 'map'
| 'plutonium-release'
| 'plutonium-version'
| 'occupancy'
| 'stopped';
label: string;
result: EvidenceResult;
source: EvidenceSource;
@@ -46,7 +63,11 @@ export interface ProvisioningProgress {
blocker: ProvisioningBlocker | null;
checks: ProvisioningCheck[];
checkpoints: ProvisioningCheckpoint[];
command: { state: 'attempted' | 'accepted' | 'rejected' | 'uncertain'; at: string; message: string } | null;
command: {
state: 'attempted' | 'accepted' | 'rejected' | 'uncertain';
at: string;
message: string;
} | null;
recovery: RecoveryObservation | null;
recipeStatus: 'queued' | 'attempted' | null;
}
@@ -105,7 +126,14 @@ export interface ProvisioningDetail {
cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
backend?: 'native';
workerActivity?: WorkerActivity[];
agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null;
agentOperation?: {
id: string;
hostId: string;
state: string;
updatedAt: string;
createdAt: string;
error: string | null;
} | null;
deployment: ProvisioningDeployment;
events: ProvisioningEvent[];
activity: { total: number; latestId: number | null; failureExcerpt: ProvisioningEvent | null };
@@ -122,7 +150,11 @@ export interface ProvisioningDetail {
warningMessages: string[];
generatedAt: string;
}
export interface WorkerActivity { key: string; label: string; startedAt: string }
export interface WorkerActivity {
key: string;
label: string;
startedAt: string;
}
export interface ProvisioningEventPage {
events: ProvisioningEvent[];
total: number;
src/lib/provisioning-list.ts
+82 -20
View File
@@ -2,42 +2,104 @@ import type { ProvisioningAction } from './provisioning.js';
import type { ProvisioningDeployment } from './provisioning-detail.js';
import type { ProvisioningSource } from './provisioning-detail.js';
export const provisioningListStatuses = ['pending', 'running', 'awaiting-health', 'done', 'attention', 'failed', 'interrupted', 'warning', 'cancelled'] as const;
export const provisioningListSources = ['manual', 'campaign', 'beta', 'maintenance', 'recovery', 'autoscaling', 'rebalance', 'system'] as const;
export type ProvisioningListSource = typeof provisioningListSources[number];
export const provisioningListStatuses = [
'pending',
'running',
'awaiting-health',
'done',
'attention',
'failed',
'interrupted',
'warning',
'cancelled'
] as const;
export const provisioningListSources = [
'manual',
'campaign',
'beta',
'maintenance',
'recovery',
'autoscaling',
'rebalance',
'system'
] as const;
export type ProvisioningListSource = (typeof provisioningListSources)[number];
export const provisioningSourceLabels: Record<ProvisioningListSource, string> = {
manual: 'Manual', campaign: 'Campaign', beta: 'Beta', maintenance: 'Maintenance',
recovery: 'Recovery', autoscaling: 'Autoscaling', rebalance: 'Rebalance', system: 'System'
manual: 'Manual',
campaign: 'Campaign',
beta: 'Beta',
maintenance: 'Maintenance',
recovery: 'Recovery',
autoscaling: 'Autoscaling',
rebalance: 'Rebalance',
system: 'System'
};
export const provisioningSourceKinds = {
campaign: ['campaign'], beta: ['beta'], maintenance: ['maintenance'],
recovery: ['watchdog'], autoscaling: ['autoscaling'], rebalance: ['rebalance']
} as const satisfies Record<Exclude<ProvisioningListSource, 'manual' | 'system'>, readonly ProvisioningSource['kind'][]>;
campaign: ['campaign'],
beta: ['beta'],
maintenance: ['maintenance'],
recovery: ['watchdog'],
autoscaling: ['autoscaling'],
rebalance: ['rebalance']
} as const satisfies Record<
Exclude<ProvisioningListSource, 'manual' | 'system'>,
readonly ProvisioningSource['kind'][]
>;
/** Record titles and incident summaries belong in expanded details, not the source column. */
export function provisioningSourceLabel(row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>): string {
export function provisioningSourceLabel(
row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>
): string {
const kind = row.sources?.[0]?.kind;
return provisioningSourceLabels[kind === 'watchdog' ? 'recovery' : kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system')];
return provisioningSourceLabels[
kind === 'watchdog'
? 'recovery'
: (kind ?? (row.triggered_by === 'ui' ? 'manual' : 'system'))
];
}
export function provisioningMatchesSource(row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>, source: ProvisioningListSource): boolean {
export function provisioningMatchesSource(
row: Pick<ProvisioningListRow, 'sources' | 'triggered_by'>,
source: ProvisioningListSource
): boolean {
const sources = row.sources ?? [];
if (source === 'manual' || source === 'system') return !sources.length && (source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui');
if (source === 'manual' || source === 'system')
return (
!sources.length &&
(source === 'manual' ? row.triggered_by === 'ui' : row.triggered_by !== 'ui')
);
const kinds: readonly ProvisioningSource['kind'][] = provisioningSourceKinds[source];
return sources.some(item => kinds.includes(item.kind));
return sources.some((item) => kinds.includes(item.kind));
}
export type ProvisioningListRow = Pick<ProvisioningDeployment,
'id' | 'server' | 'server_deleted_at' | 'action' | 'state' | 'stage' | 'started_at' |
'finished_at' | 'triggered_by' | 'requested_by' | 'error' | 'warnings' | 'server_exists'> & {
canCancelRemoval: number; canCancel?: number; sources?: ProvisioningSource[];
phase?: string | null; phaseStartedAt?: string | null; waitCode?: string | null;
};
export type ProvisioningListRow = Pick<
ProvisioningDeployment,
| 'id'
| 'server'
| 'server_deleted_at'
| 'action'
| 'state'
| 'stage'
| 'started_at'
| 'finished_at'
| 'triggered_by'
| 'requested_by'
| 'error'
| 'warnings'
| 'server_exists'
> & {
canCancelRemoval: number;
canCancel?: number;
sources?: ProvisioningSource[];
phase?: string | null;
phaseStartedAt?: string | null;
waitCode?: string | null;
};
export interface ProvisioningListOptions {
view?: 'active' | 'history' | 'all';
action?: ProvisioningAction;
status?: typeof provisioningListStatuses[number];
status?: (typeof provisioningListStatuses)[number];
source?: ProvisioningListSource;
server?: string;
limit?: number;
src/lib/provisioning.ts
+23 -6
View File
@@ -1,17 +1,34 @@
export const provisioningActions = ['create', 'start', 'restart', 'rebuild', 'stop', 'destroy'] as const;
export type ProvisioningAction = typeof provisioningActions[number];
export const provisioningActions = [
'create',
'start',
'restart',
'rebuild',
'stop',
'destroy'
] as const;
export type ProvisioningAction = (typeof provisioningActions)[number];
export type RuntimeAction = Exclude<ProvisioningAction, 'create' | 'destroy'>;
export const actionLabels: Record<ProvisioningAction, string> = {
create: 'Create', start: 'Start', restart: 'Restart', rebuild: 'Rebuild', stop: 'Stop', destroy: 'Destroy'
create: 'Create',
start: 'Start',
restart: 'Restart',
rebuild: 'Rebuild',
stop: 'Stop',
destroy: 'Destroy'
};
export const actionCompleted: Record<ProvisioningAction, string> = {
create: 'Server creation completed', start: 'Server started', restart: 'Server restarted',
rebuild: 'Server rebuilt', stop: 'Server stopped', destroy: 'Server destroyed'
create: 'Server creation completed',
start: 'Server started',
restart: 'Server restarted',
rebuild: 'Server rebuilt',
stop: 'Server stopped',
destroy: 'Server destroyed'
};
export const runtimeActions: RuntimeAction[] = ['start', 'restart', 'rebuild', 'stop'];
export const runtimeDescriptions: Record<RuntimeAction, string> = {
start: 'Start the existing container and verify its health.',
restart: 'Restart the container after the server is empty, then verify its health.',
rebuild: 'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.',
rebuild:
'Remove the instance and delete all of its server files, then create a fresh instance in the same server slot and port with the latest promoted setup. No instance data is retained. Uses the key saved separately in Keys. Waits until empty, then verifies health and map.',
stop: 'Stop the container after the server is empty. Its files, key, and registration are retained so you can start it again.'
};
src/lib/rebalancing.ts
+57 -17
View File
@@ -4,15 +4,26 @@ export interface RebalanceSettings {
maxMoves: number;
}
export interface RebalanceHostOption { id: string; name: string; eligible: boolean; reason: string | null }
export interface RebalanceHostOption {
id: string;
name: string;
eligible: boolean;
reason: string | null;
}
export interface RebalanceHostProjection {
id: string; name: string;
id: string;
name: string;
before: { cpuPercent: number; memoryPercent: number; instances: number };
after: { cpuPercent: number; memoryPercent: number; instances: number };
}
export interface RebalanceCandidate {
server: string; instanceId: string; generation: number;
sourceHostId: string; sourceHostName: string; targetHostId: string; targetHostName: string;
server: string;
instanceId: string;
generation: number;
sourceHostId: string;
sourceHostName: string;
targetHostId: string;
targetHostName: string;
/** Estimated reduction in the source/destination pair's peak utilization, in percentage points. */
improvement: number;
}
@@ -23,26 +34,55 @@ export interface RebalancePlan {
warnings: string[];
}
export interface RebalancePreview extends RebalancePlan {
id: string; settings: RebalanceSettings; createdAt: string; expiresAt: string;
id: string;
settings: RebalanceSettings;
createdAt: string;
expiresAt: string;
}
export interface RebalanceMove extends RebalanceCandidate {
id: string; state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed';
operationId: string | null; message: string; updatedAt: string;
id: string;
state: 'planned' | 'moving' | 'done' | 'skipped' | 'failed';
operationId: string | null;
message: string;
updatedAt: string;
}
export interface RebalanceRun {
id: string; settings: RebalanceSettings;
id: string;
settings: RebalanceSettings;
state: 'running' | 'stopping' | 'done' | 'stopped' | 'failed';
createdAt: string; updatedAt: string; message: string; moves: RebalanceMove[];
createdAt: string;
updatedAt: string;
message: string;
moves: RebalanceMove[];
}
export interface RebalanceReport {
hosts: RebalanceHostOption[];
runs: RebalanceRun[];
}
export interface RebalanceReport { hosts: RebalanceHostOption[]; runs: RebalanceRun[] }
export function rebalanceSettings(input: unknown): RebalanceSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide rebalance settings.');
if (!input || typeof input !== 'object' || Array.isArray(input))
throw new Error('Provide rebalance settings.');
const value = input as Record<string, unknown>;
if (!Array.isArray(value.hostIds) || value.hostIds.length < 2 || value.hostIds.length > 100
|| value.hostIds.some(id => typeof id !== 'string' || !id || id.length > 100)
|| new Set(value.hostIds).size !== value.hostIds.length) throw new Error('Choose between 2 and 100 different hosts.');
if (!['all', 't5', 't6'].includes(value.game as string)) throw new Error('Choose a supported game.');
if (!Number.isInteger(value.maxMoves) || (value.maxMoves as number) < 1 || (value.maxMoves as number) > 20) throw new Error('Choose a move limit between 1 and 20.');
return { hostIds: [...value.hostIds as string[]].sort(), game: value.game as RebalanceSettings['game'], maxMoves: value.maxMoves as number };
if (
!Array.isArray(value.hostIds) ||
value.hostIds.length < 2 ||
value.hostIds.length > 100 ||
value.hostIds.some((id) => typeof id !== 'string' || !id || id.length > 100) ||
new Set(value.hostIds).size !== value.hostIds.length
)
throw new Error('Choose between 2 and 100 different hosts.');
if (!['all', 't5', 't6'].includes(value.game as string))
throw new Error('Choose a supported game.');
if (
!Number.isInteger(value.maxMoves) ||
(value.maxMoves as number) < 1 ||
(value.maxMoves as number) > 20
)
throw new Error('Choose a move limit between 1 and 20.');
return {
hostIds: [...(value.hostIds as string[])].sort(),
game: value.game as RebalanceSettings['game'],
maxMoves: value.maxMoves as number
};
}
src/lib/script-library.ts
+53 -19
View File
@@ -2,20 +2,34 @@ import { T5_ZOMBIE_MAPS, T6_ZOMBIE_MAPS } from './map-aliases.js';
import { gameTitle } from './ui/server-labels.js';
export interface LibraryVersion {
id: number; version: number; sha256: string; size: number;
state: 'untested' | 'tested' | 'test-failed'; uploaded_at: string;
id: number;
version: number;
sha256: string;
size: number;
state: 'untested' | 'tested' | 'test-failed';
uploaded_at: string;
}
export interface LibraryScript {
id: number; name: string; target_rel_path: string;
scope_type: 'game' | 'map' | null; scope_value: string | null;
id: number;
name: string;
target_rel_path: string;
scope_type: 'game' | 'map' | null;
scope_value: string | null;
current_version_id: number | null;
currentVersion: LibraryVersion | null; latestVersion: LibraryVersion | null;
versionCount: number; servers: string[]; runningBetaId: number | null;
currentVersion: LibraryVersion | null;
latestVersion: LibraryVersion | null;
versionCount: number;
servers: string[];
runningBetaId: number | null;
}
export type ScriptStatus = 'untested' | 'testing' | 'tested' | 'failed' | 'promoted' | 'empty';
export const statusLabels: Record<ScriptStatus, string> = {
untested: 'Needs testing', testing: 'Beta active', tested: 'Test passed',
failed: 'Test failed', promoted: 'Promoted', empty: 'No versions'
untested: 'Needs testing',
testing: 'Beta active',
tested: 'Test passed',
failed: 'Test failed',
promoted: 'Promoted',
empty: 'No versions'
};
export function scriptStatus(s: LibraryScript): ScriptStatus {
if (s.runningBetaId !== null) return 'testing';
@@ -24,17 +38,29 @@ export function scriptStatus(s: LibraryScript): ScriptStatus {
return s.latestVersion.state === 'test-failed' ? 'failed' : s.latestVersion.state;
}
export function canTest(s: LibraryScript): boolean {
return s.runningBetaId === null && s.latestVersion !== null && s.latestVersion.id !== s.current_version_id;
return (
s.runningBetaId === null &&
s.latestVersion !== null &&
s.latestVersion.id !== s.current_version_id
);
}
export function mapLabel(canonical: string): string {
const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter(m => m.canonical === canonical);
return matches.length > 1 ? `${matches[0].friendly} + ${matches.length - 1} variants` : matches[0]?.friendly ?? canonical;
const matches = [...T5_ZOMBIE_MAPS, ...T6_ZOMBIE_MAPS].filter((m) => m.canonical === canonical);
return matches.length > 1
? `${matches[0].friendly} + ${matches.length - 1} variants`
: (matches[0]?.friendly ?? canonical);
}
export function scopeLabel(s: Pick<LibraryScript, 'scope_type' | 'scope_value'>): string {
return s.scope_type === 'game' ? gameTitle(s.scope_value)
: s.scope_type === 'map' ? mapLabel(s.scope_value ?? '') : 'Unscoped';
return s.scope_type === 'game'
? gameTitle(s.scope_value)
: s.scope_type === 'map'
? mapLabel(s.scope_value ?? '')
: 'Unscoped';
}
export interface BetaSelection {
scriptId: number;
versionId: number;
}
export interface BetaSelection { scriptId: number; versionId: number; }
export function betaSelectionHref(selections: BetaSelection[]): string {
const query = new URLSearchParams();
for (const s of selections) query.append('artifact', `${s.scriptId}:${s.versionId}`);
@@ -42,13 +68,21 @@ export function betaSelectionHref(selections: BetaSelection[]): string {
}
export function parseBetaSelection(query: URLSearchParams): BetaSelection[] {
const values = query.getAll('artifact');
if (!values.length && query.has('pluginId')) values.push(`${query.get('pluginId')}:${query.get('versionId')}`);
if (!values.length && query.has('pluginId'))
values.push(`${query.get('pluginId')}:${query.get('versionId')}`);
const seen = new Set<number>();
return values.map(value => {
if (!/^[1-9]\d*:[1-9]\d*$/.test(value)) throw new Error('The beta selection link is invalid. Choose artifacts below.');
return values.map((value) => {
if (!/^[1-9]\d*:[1-9]\d*$/.test(value))
throw new Error('The beta selection link is invalid. Choose artifacts below.');
const [scriptId, versionId] = value.split(':').map(Number);
if (!Number.isSafeInteger(scriptId) || !Number.isSafeInteger(versionId) || seen.has(scriptId)) {
throw new Error('The beta selection link contains invalid or repeated artifacts. Choose artifacts below.');
if (
!Number.isSafeInteger(scriptId) ||
!Number.isSafeInteger(versionId) ||
seen.has(scriptId)
) {
throw new Error(
'The beta selection link contains invalid or repeated artifacts. Choose artifacts below.'
);
}
seen.add(scriptId);
return { scriptId, versionId };
src/lib/server-capacity.ts
+13 -4
View File
@@ -6,11 +6,20 @@ export interface PlayerCapacityObservation {
}
/** A newer roster must never renew the capacity reported by an older process. */
export function currentCapacity(server: Partial<PlayerCapacityObservation> & { telemetry_after?: string | null }, now: number): number | null {
export function currentCapacity(
server: Partial<PlayerCapacityObservation> & { telemetry_after?: string | null },
now: number
): number | null {
const value = server.capacity;
const observedAt = server.capacity_observed_at;
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0
|| !freshStatusTimestamp(observedAt ?? null, now)) return null;
if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after))) return null;
if (
typeof value !== 'number' ||
!Number.isSafeInteger(value) ||
value <= 0 ||
!freshStatusTimestamp(observedAt ?? null, now)
)
return null;
if (server.telemetry_after && !(Date.parse(observedAt!) > Date.parse(server.telemetry_after)))
return null;
return value;
}
src/lib/server-observation.ts
+114 -27
View File
@@ -36,51 +36,112 @@ export interface ServerObservation extends GameObservation {
}
/** Keep a first transient monitoring failure in diagnostics without demoting health. */
export function rconHealthError(status: Pick<GameObservation, 'last_error' | 'rcon_failures'>): string | null {
export function rconHealthError(
status: Pick<GameObservation, 'last_error' | 'rcon_failures'>
): string | null {
const reason = status.last_error?.split(':')[0];
return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error') ? null : status.last_error;
return status.rcon_failures === 1 && (reason === 'timeout' || reason === 'send-error')
? null
: status.last_error;
}
export function freshTelemetry(report: GameTelemetry | undefined, now: number, after?: string | null): report is GameTelemetry {
export function freshTelemetry(
report: GameTelemetry | undefined,
now: number,
after?: string | null
): report is GameTelemetry {
if (!report || !freshStatusTimestamp(report.receivedAt, now)) return false;
if (after && (!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))) return false;
try { parseGameEvent(report); return true; } catch { return false; }
if (
after &&
(!Number.isFinite(Date.parse(after)) || Date.parse(report.receivedAt) <= Date.parse(after))
)
return false;
try {
parseGameEvent(report);
return true;
} catch {
return false;
}
}
function playerMatchGuid(guid: string, game: string | null | undefined, source: 'rcon' | 'telemetry'): string {
function playerMatchGuid(
guid: string,
game: string | null | undefined,
source: 'rcon' | 'telemetry'
): string {
if (game !== 't6') return guid;
// T6 status prints hexadecimal account IDs; telemetry sends decimal IDs.
// Keep integer precision for IDs beyond Number.MAX_SAFE_INTEGER.
if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid)) return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString();
if (source === 'rcon' && /^(?:0x)?[0-9a-f]+$/i.test(guid))
return BigInt(`0x${guid.replace(/^0x/i, '')}`).toString();
if (source === 'telemetry' && /^\d+$/.test(guid)) return BigInt(guid).toString();
return guid;
}
/** Display policy: use one current roster/count source, keeping raw RCON for diagnostics/fallback. */
export function selectServerObservation<T extends ServerObservation>(server: T, report: GameTelemetry | undefined, now: number): T {
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
export function selectServerObservation<T extends ServerObservation>(
server: T,
report: GameTelemetry | undefined,
now: number
): T {
if (
server.telemetry &&
(!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))
)
report = server.telemetry;
const raw = server.rcon_status ?? server;
const result: T = { ...server, ...raw, rcon_status: raw, last_error: rconHealthError(raw), telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
const result: T = {
...server,
...raw,
rcon_status: raw,
last_error: rconHealthError(raw),
telemetry: report,
status_source: raw.last_seen_at ? 'rcon' : null
};
const containerUnknown = server.container_state === 'unknown';
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
if (
['provisioning', 'starting'].includes(server.health) ||
(server.container_state && server.container_state !== 'running' && !containerUnknown) ||
(!server.container_state && server.health === 'offline')
) {
return {
...result,
health: server.health,
last_error: null,
last_players: null,
roster: null,
status_source: null
};
}
const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
const containerFresh = server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const containerFresh =
server.container_state === 'running' && freshStatusTimestamp(checked ?? null, now);
const after =
!server.telemetry_after ||
Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !result.last_error;
if (containerFresh && report?.server === server.name && freshTelemetry(report, now, server.telemetry_after)) {
if (
containerFresh &&
report?.server === server.name &&
freshTelemetry(report, now, server.telemetry_after)
) {
const players = report.players.length;
// Telemetry owns roster membership. Supplement missing pings only with
// unambiguous RCON identities from the current map and server lifecycle.
const pings = new Map<string, number | null>();
const guidKey = (guid: string, source: 'rcon' | 'telemetry') => playerMatchGuid(guid, server.game, source);
const guidKey = (guid: string, source: 'rcon' | 'telemetry') =>
playerMatchGuid(guid, server.game, source);
if (fresh && raw.last_map === report.map) {
for (const player of raw.roster ?? []) {
const ping = player.ping;
if (player.guid) {
const guid = guidKey(player.guid, 'rcon');
pings.set(guid, pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0 ? null : ping);
pings.set(
guid,
pings.has(guid) || ping === null || !Number.isSafeInteger(ping) || ping < 0
? null
: ping
);
}
}
}
@@ -89,23 +150,49 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
const timing = report.timing?.server ?? {
activeSince: server.active_since ?? report.receivedAt,
inactiveSince: server.inactive_since ?? report.receivedAt,
gameStartedAt: report.event === 'game_start' ? report.receivedAt : server.game_started_at,
gameEndedAt: report.event === 'game_start' ? null : report.event === 'game_end' ? report.receivedAt : server.game_ended_at
gameStartedAt:
report.event === 'game_start' ? report.receivedAt : server.game_started_at,
gameEndedAt:
report.event === 'game_start'
? null
: report.event === 'game_end'
? report.receivedAt
: server.game_ended_at
};
return { ...result, health: 'online', status_source: 'telemetry', last_players: players,
last_map: report.map, last_seen_at: report.receivedAt, last_error: null,
roster: report.players.map(player => ({ num: null, score: 'score' in player ? player.score : null,
ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null, guid: player.guid, name: player.name })),
return {
...result,
health: 'online',
status_source: 'telemetry',
last_players: players,
last_map: report.map,
last_seen_at: report.receivedAt,
last_error: null,
roster: report.players.map((player) => ({
num: null,
score: 'score' in player ? player.score : null,
ping: player.ping ?? pings.get(guidKey(player.guid, 'telemetry')) ?? null,
guid: player.guid,
name: player.name
})),
active_since: players > 0 ? timing.activeSince : null,
inactive_since: players === 0 ? timing.inactiveSince : null,
game_started_at: players === 0 ? null : timing.gameStartedAt,
game_ended_at: players === 0 ? null : timing.gameEndedAt };
game_ended_at: players === 0 ? null : timing.gameEndedAt
};
}
// A current authenticated RCON reply can establish game liveness when
// inspection is unavailable, without claiming the container was inspected.
const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
const age = now - Date.parse(raw.last_seen_at ?? '');
const health = fresh && !containerExpired ? 'online'
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline';
return { ...result, health, last_error: health === 'online' ? result.last_error : raw.last_error };
const health =
fresh && !containerExpired
? 'online'
: after && age >= 0 && age <= 120_000
? 'stale'
: 'offline';
return {
...result,
health,
last_error: health === 'online' ? result.last_error : raw.last_error
};
}
src/lib/server/activity.ts
+87 -23
View File
@@ -1,6 +1,12 @@
import { hostOperationTitles } from '../host-operation-labels.js';
import type Database from 'better-sqlite3';
import { normalizeActivityQuery, type ActivityPage, type ActivityQuery, type ActivityRow, type ActivitySource } from '../activity.js';
import {
normalizeActivityQuery,
type ActivityPage,
type ActivityQuery,
type ActivityRow,
type ActivitySource
} from '../activity.js';
import { serverTitle } from '../ui/server-labels.js';
import { db } from './db/index.js';
import { ensureHostSchema } from './hosts/schema.js';
@@ -9,16 +15,22 @@ import { importantOperationSql } from './hosts/observations.js';
const initialized = new WeakSet<Database.Database>();
const quote = (value: string) => `'${value.replaceAll("'", "''")}'`;
const labelCase = (column: string, labels: Record<string, string>, fallback: string) =>
`CASE ${column} ${Object.entries(labels).map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`).join(' ')} ELSE ${quote(fallback)} END`;
`CASE ${column} ${Object.entries(labels)
.map(([key, value]) => `WHEN ${quote(key)} THEN ${quote(value)}`)
.join(' ')} ELSE ${quote(fallback)} END`;
// Old attempts may no longer be the native_provisioning pointer. Their stable
// idempotency keys still identify the same retained controller operation.
const lifecycleKey = `CASE
WHEN substr(o.idempotency_key,-6)=':stage' THEN substr(o.idempotency_key,1,length(o.idempotency_key)-6)
${[':migration-create:', ':migrate-remove:'].map(marker => `WHEN instr(o.idempotency_key,${quote(marker)})>0
${[':migration-create:', ':migrate-remove:']
.map(
(marker) => `WHEN instr(o.idempotency_key,${quote(marker)})>0
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length})<>''
AND substr(o.idempotency_key,instr(o.idempotency_key,${quote(marker)})+${marker.length}) NOT GLOB '*[^0-9]*'
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`).join('\n')}
THEN substr(o.idempotency_key,1,instr(o.idempotency_key,${quote(marker)})-1)`
)
.join('\n')}
ELSE o.idempotency_key END`;
/** One row per user-level operation or finding; no command payloads or observations are read. */
@@ -128,19 +140,50 @@ normalized AS (
)
`;
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & { sources: string; route: string; recordId: string; secondaryHostId: string | null };
const sourceOrder: ActivitySource[] = ['maintenance', 'recovery', 'campaign', 'beta', 'autoscaling', 'provisioning', 'host'];
type SqlRow = Omit<ActivityRow, 'source' | 'sources' | 'href'> & {
sources: string;
route: string;
recordId: string;
secondaryHostId: string | null;
};
const sourceOrder: ActivitySource[] = [
'maintenance',
'recovery',
'campaign',
'beta',
'autoscaling',
'provisioning',
'host'
];
// Use the same membership rule for Maintenance history and its exclusion from the main log.
const maintenanceMembershipSql = "(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
const maintenanceMembershipSql =
"(instr(sources,'|maintenance|')>0 OR instr(sources,'|recovery|')>0)";
function present(row: SqlRow): ActivityRow {
const sources = sourceOrder.filter(source => row.sources.includes(`|${source}|`));
const href = row.route === 'maintenance-job' ? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery' ? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host' ? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
const sources = sourceOrder.filter((source) => row.sources.includes(`|${source}|`));
const href =
row.route === 'maintenance-job'
? `/maintenance?view=overview&job=${encodeURIComponent(row.recordId)}#job-${row.recordId}`
: row.route === 'recovery'
? `/maintenance/${encodeURIComponent(row.recordId)}`
: row.route === 'host'
? `/hosts/operations?operation=${encodeURIComponent(row.recordId)}&filter=all`
: `/${row.route === 'campaign' ? 'campaigns' : row.route === 'beta' ? 'betas' : 'provisioning'}/${encodeURIComponent(row.recordId)}`;
return { id: row.id, at: row.at, type: row.type, source: sources[0] ?? 'provisioning', sources,
state: row.state, rawState: row.rawState, title: row.title, detail: row.detail,
server: row.server, serverDeletedAt: row.serverDeletedAt, hostId: row.hostId, hostName: row.hostName, href };
return {
id: row.id,
at: row.at,
type: row.type,
source: sources[0] ?? 'provisioning',
sources,
state: row.state,
rawState: row.rawState,
title: row.title,
detail: row.detail,
server: row.server,
serverDeletedAt: row.serverDeletedAt,
hostId: row.hostId,
hostName: row.hostName,
href
};
}
/** All predicates, totals and pagination operate on the same retained SQL union. */
@@ -149,7 +192,9 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
const handle = db();
ensureHostSchema(handle);
if (!initialized.has(handle)) {
handle.function('activity_server_title', { deterministic: true }, (name: unknown) => typeof name === 'string' ? serverTitle(name) : '');
handle.function('activity_server_title', { deterministic: true }, (name: unknown) =>
typeof name === 'string' ? serverTitle(name) : ''
);
initialized.add(handle);
}
const conditions: string[] = [];
@@ -158,25 +203,44 @@ export function activityHistory(input: ActivityQuery = {}, now = Date.now()): Ac
if (query.excludeProvisioning) conditions.push("route <> 'provisioning' AND type <> 'restart'");
if (query.excludeMaintenance) conditions.push(`NOT ${maintenanceMembershipSql}`);
const bindings: Record<string, string | number> = {};
for (const key of ['server', 'type', 'state'] as const) if (query[key]) { conditions.push(`${key}=@${key}`); bindings[key] = query[key]; }
if (query.host) { conditions.push('(hostId=@host OR secondaryHostId=@host)'); bindings.host = query.host; }
for (const key of ['server', 'type', 'state'] as const)
if (query[key]) {
conditions.push(`${key}=@${key}`);
bindings[key] = query[key];
}
if (query.host) {
conditions.push('(hostId=@host OR secondaryHostId=@host)');
bindings.host = query.host;
}
if (query.source) {
// Maintenance is the user-facing umbrella; Recovery remains available as a narrower filter.
conditions.push(query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0');
conditions.push(
query.source === 'maintenance' ? maintenanceMembershipSql : 'instr(sources,@source)>0'
);
if (query.source !== 'maintenance') bindings.source = `|${query.source}|`;
}
if (query.days) { conditions.push('julianday(at)>=julianday(@since)'); bindings.since = new Date(now - query.days * 86_400_000).toISOString(); }
if (query.days) {
conditions.push('julianday(at)>=julianday(@since)');
bindings.since = new Date(now - query.days * 86_400_000).toISOString();
}
if (query.q) {
conditions.push("instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0");
conditions.push(
"instr(lower(COALESCE(server,'') || ' ' || activity_server_title(server) || ' ' || COALESCE(hostName,'') || ' ' || COALESCE(hostId,'') || ' ' || title || ' ' || COALESCE(detail,'') || ' ' || id),@q)>0"
);
bindings.q = query.q.toLocaleLowerCase();
}
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = query.limit!;
return handle.transaction(() => {
const { total } = handle.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`).get(bindings) as { total: number };
const { total } = handle
.prepare(`${activitySql} SELECT COUNT(*) AS total FROM normalized ${where}`)
.get(bindings) as { total: number };
const offset = Math.min(query.offset!, Math.max(0, Math.ceil(total / limit) - 1) * limit);
const rows = handle.prepare(`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`)
const rows = handle
.prepare(
`${activitySql} SELECT id,at,type,sources,state,rawState,title,detail,server,serverDeletedAt,hostId,hostName,secondaryHostId,route,recordId
FROM normalized ${where} ORDER BY julianday(at) DESC,id DESC LIMIT @limit OFFSET @offset`
)
.all({ ...bindings, limit, offset }) as SqlRow[];
return { rows: rows.map(present), total, limit, offset, query: { ...query, offset } };
})();
src/lib/server/auth/access.ts
+50 -16
View File
@@ -1,8 +1,10 @@
import type { UserRole } from '../../auth.js';
export type RoutePermission = 'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
export type RoutePermission =
'public' | 'machine' | 'authenticated' | 'viewer' | 'operator' | 'admin';
const within = (route: string, base: string): boolean => route === base || route.startsWith(base + '/');
const within = (route: string, base: string): boolean =>
route === base || route.startsWith(base + '/');
const readMethod = (method: string): boolean => method === 'GET' || method === 'HEAD';
/** Classify SvelteKit's resolved route ID, never a user-supplied URL pathname.
@@ -15,32 +17,63 @@ export function routePermission(routeId: string | null, method: string): RoutePe
if (routeId === null) return reading ? 'viewer' : 'operator';
// These cannot occur in this application's resolved route IDs. Fail closed
// if a future caller accidentally supplies a raw or noncanonical pathname.
if (!routeId.startsWith('/') || /[%\\?#\x00-\x1f\x7f]/.test(routeId)
|| routeId !== '/' && routeId.split('/').slice(1).some(part => !part || part === '.' || part === '..')) return 'admin';
if (
!routeId.startsWith('/') ||
// eslint-disable-next-line no-control-regex -- Reject control characters in resolved route IDs.
/[%\\?#\x00-\x1f\x7f]/.test(routeId) ||
(routeId !== '/' &&
routeId
.split('/')
.slice(1)
.some((part) => !part || part === '.' || part === '..'))
)
return 'admin';
if (within(routeId, '/api/agent')) return 'machine';
if (routeId === '/api/telemetry' && method === 'POST') return 'machine';
if (routeId === '/api/leaderboard' && reading) return 'public';
if ((routeId === '/login' || routeId === '/setup') && reading) return 'public';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST') return 'public';
if (routeId === '/account' || ['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)) return 'authenticated';
if ((routeId === '/api/auth/login' || routeId === '/api/auth/setup') && method === 'POST')
return 'public';
if (
routeId === '/account' ||
['/api/auth/me', '/api/auth/logout', '/api/auth/password'].includes(routeId)
)
return 'authenticated';
if (within(routeId, '/api/users') || within(routeId, '/settings') || within(routeId, '/api/settings') || within(routeId, '/api/auth')
|| within(routeId, '/api/server-keys') || within(routeId, '/server-keys') || within(routeId, '/keys')) return 'admin';
if (
within(routeId, '/api/users') ||
within(routeId, '/settings') ||
within(routeId, '/api/settings') ||
within(routeId, '/api/auth') ||
within(routeId, '/api/server-keys') ||
within(routeId, '/server-keys') ||
within(routeId, '/keys')
)
return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some(base => within(routeId, base))) return 'admin';
if (!reading && ['/api/hosts', '/api/game-files'].some((base) => within(routeId, base)))
return 'admin';
if (routeId === '/api/provisioning/[id]/override') return 'admin';
// These endpoints mix operational actions with global policy and schedule
// changes. Per-incident/job actions remain available to operators.
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId)) return 'admin';
if (!reading && ['/api/maintenance', '/api/plutonium', '/api/watchdog'].includes(routeId))
return 'admin';
// Summaries remain visible to viewers; raw configuration, historical file
// bytes, and unfiltered diagnostics may contain runtime credentials.
if (routeId === '/configs/[id]' || routeId === '/api/configs/[id]' || routeId === '/api/configs/inspect'
|| routeId === '/api/campaigns/[id]/file/[fileId]' || routeId === '/api/deployments/[id]/file/[fileId]'
|| routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]'
|| routeId === '/api/servers/[name]/logs/[file]' || routeId === '/api/servers/[name]/telemetry-requests'
|| routeId === '/api/servers/[name]/rcon-diagnostics') return 'operator';
if (
routeId === '/configs/[id]' ||
routeId === '/api/configs/[id]' ||
routeId === '/api/configs/inspect' ||
routeId === '/api/campaigns/[id]/file/[fileId]' ||
routeId === '/api/deployments/[id]/file/[fileId]' ||
routeId === '/api/betas/[id]/revisions/[revision]/files/[scriptId]' ||
routeId === '/api/servers/[name]/logs/[file]' ||
routeId === '/api/servers/[name]/telemetry-requests' ||
routeId === '/api/servers/[name]/rcon-diagnostics'
)
return 'operator';
return reading ? 'viewer' : 'operator';
}
@@ -49,7 +82,8 @@ export function routePermission(routeId: string | null, method: string): RoutePe
export function roleAllows(role: UserRole, permission: RoutePermission): boolean {
if (!['viewer', 'operator', 'admin'].includes(role)) return false;
if (permission === 'machine') return false;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer') return true;
if (permission === 'public' || permission === 'authenticated' || permission === 'viewer')
return true;
if (permission === 'operator') return role === 'operator' || role === 'admin';
return permission === 'admin' && role === 'admin';
}
src/lib/server/auth/guard.ts
+174 -52
View File
@@ -1,112 +1,234 @@
import { error, json, redirect, type Handle } from '@sveltejs/kit';
import { readSession, readSessionHash, recordAudit, setupRequired } from './index.js';
import { authBody, authError, requireSameOrigin, requireUserTransport, sessionCookie } from './http.js';
import {
authBody,
authError,
requireSameOrigin,
requireUserTransport,
sessionCookie
} from './http.js';
import { roleAllows, routePermission, type RoutePermission } from './access.js';
import { withRequestUsername } from './request-context.js';
function publicAsset(routeId: string | null, path: string, method: string) {
return routeId === null && ['GET','HEAD'].includes(method) &&
(path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/ec-agent-update.py','/agent/install.sh','/agent/manifest.json'].includes(path));
return (
routeId === null &&
['GET', 'HEAD'].includes(method) &&
(path.startsWith('/_app/') ||
path === '/favicon.svg' ||
[
'/agent/ec-agent.py',
'/agent/ec-agent-update.py',
'/agent/install.sh',
'/agent/manifest.json'
].includes(path))
);
}
/** Streaming responses must lose access when the underlying session does. */
export function protectEventStream(response: Response, tokenHash: string, permission: RoutePermission, signal: AbortSignal): Response {
export function protectEventStream(
response: Response,
tokenHash: string,
permission: RoutePermission,
signal: AbortSignal
): Response {
if (!response.body) return response;
const reader = response.body.getReader();
let timer: ReturnType<typeof setInterval> | undefined;
let closed = false;
let closeStream: () => void = () => {};
const cleanup = () => { if (timer) clearInterval(timer); signal.removeEventListener('abort', closeStream); };
const cleanup = () => {
if (timer) clearInterval(timer);
signal.removeEventListener('abort', closeStream);
};
const stream = new ReadableStream<Uint8Array>({
start(controller) {
closeStream = () => {
if (closed) return; closed = true; cleanup();
void reader.cancel().catch(() => {}); controller.close();
if (closed) return;
closed = true;
cleanup();
void reader.cancel().catch(() => {});
controller.close();
};
timer = setInterval(() => {
try {
const current = readSessionHash(tokenHash);
if (!current || current.user.mustChangePassword || !roleAllows(current.user.role,permission)) closeStream();
} catch { closeStream(); }
},15_000);
signal.addEventListener('abort',closeStream,{once:true});
if (signal.aborted) { closeStream(); return; }
if (
!current ||
current.user.mustChangePassword ||
!roleAllows(current.user.role, permission)
)
closeStream();
} catch {
closeStream();
}
}, 15_000);
signal.addEventListener('abort', closeStream, { once: true });
if (signal.aborted) {
closeStream();
return;
}
},
async pull(controller) {
try {
const next = await reader.read();
if (closed) return;
if (next.done) { closed=true; cleanup(); controller.close(); }
else controller.enqueue(next.value);
} catch (cause) { if (!closed) { closed=true; cleanup(); controller.error(cause); } }
if (next.done) {
closed = true;
cleanup();
controller.close();
} else controller.enqueue(next.value);
} catch (cause) {
if (!closed) {
closed = true;
cleanup();
controller.error(cause);
}
}
},
cancel() { closed=true; cleanup(); return reader.cancel(); }
cancel() {
closed = true;
cleanup();
return reader.cancel();
}
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: response.headers
});
return new Response(stream,{status:response.status,statusText:response.statusText,headers:response.headers});
}
export const authenticationHandle: Handle = async ({ event, resolve }) => {
event.locals.user = null; event.locals.sessionHash = null;
event.locals.user = null;
event.locals.sessionHash = null;
const routeId = event.route.id;
const method = event.request.method.toUpperCase();
if (publicAsset(routeId,event.url.pathname,method)) return resolve(event);
const permission = routePermission(routeId,method);
if (!routeId?.startsWith('/api/agent/') && /^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization')??'')) {
return json({error:'Host credentials only authorize the host agent API.'},{status:403});
if (publicAsset(routeId, event.url.pathname, method)) return resolve(event);
const permission = routePermission(routeId, method);
if (
!routeId?.startsWith('/api/agent/') &&
/^Bearer ec_(?:host|enroll)_/i.test(event.request.headers.get('authorization') ?? '')
) {
return json(
{ error: 'Host credentials only authorize the host agent API.' },
{ status: 403 }
);
}
if (permission === 'machine') return resolve(event);
const api = routeId?.startsWith('/api/') || event.url.pathname.startsWith('/api/');
try {
requireUserTransport(event.url);
if (!['GET','HEAD','OPTIONS'].includes(method)) requireSameOrigin(event.request,event.url);
} catch (cause) { if (api) return authError(cause); throw cause; }
if (!['GET', 'HEAD', 'OPTIONS'].includes(method))
requireSameOrigin(event.request, event.url);
} catch (cause) {
if (api) return authError(cause);
throw cause;
}
const session = readSession(event.cookies.get(sessionCookie(event.url)));
event.locals.user = session?.user??null;
event.locals.sessionHash = session?.tokenHash??null;
event.locals.user = session?.user ?? null;
event.locals.sessionHash = session?.tokenHash ?? null;
const setup = setupRequired();
if (permission !== 'public') {
if (!session) {
if (api) return json({error:setup?'Complete administrator setup before using the management API.':'Sign in to continue.',setupRequired:setup},{status:401,headers:{'Cache-Control':'no-store'}});
redirect(303,setup?'/setup':`/login?next=${encodeURIComponent(event.url.pathname+event.url.search)}`);
if (api)
return json(
{
error: setup
? 'Complete administrator setup before using the management API.'
: 'Sign in to continue.',
setupRequired: setup
},
{ status: 401, headers: { 'Cache-Control': 'no-store' } }
);
redirect(
303,
setup
? '/setup'
: `/login?next=${encodeURIComponent(event.url.pathname + event.url.search)}`
);
}
if (session.user.mustChangePassword && permission !== 'authenticated') {
if (api) return json({error:'Change your temporary password from Account before continuing.',mustChangePassword:true},{status:403,headers:{'Cache-Control':'no-store'}});
redirect(303,'/account');
if (api)
return json(
{
error: 'Change your temporary password from Account before continuing.',
mustChangePassword: true
},
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
redirect(303, '/account');
}
if (!roleAllows(session.user.role,permission)) {
const message = permission==='admin'?'Administrator access is required.':'Operator access is required for this action.';
if (api) return json({error:message},{status:403,headers:{'Cache-Control':'no-store'}});
error(403,message);
if (!roleAllows(session.user.role, permission)) {
const message =
permission === 'admin'
? 'Administrator access is required.'
: 'Operator access is required for this action.';
if (api)
return json(
{ error: message },
{ status: 403, headers: { 'Cache-Control': 'no-store' } }
);
error(403, message);
}
// Launch APIs can also save credentials. Operators may launch with saved
// keys, but only administrators may introduce or replace server keys.
if (session.user.role === 'operator' && method === 'POST' && ['/api/servers','/api/deployment-collections'].includes(routeId??'')) {
if (
session.user.role === 'operator' &&
method === 'POST' &&
['/api/servers', '/api/deployment-collections'].includes(routeId ?? '')
) {
try {
const body = await authBody(event.request.clone(),1024*1024);
if (body.plutoKey || (body.keys && typeof body.keys==='object' && Object.values(body.keys).some(Boolean))) {
return json({error:'An administrator must save server keys before an operator launches these servers.'},{status:403});
const body = await authBody(event.request.clone(), 1024 * 1024);
if (
body.plutoKey ||
(body.keys &&
typeof body.keys === 'object' &&
Object.values(body.keys).some(Boolean))
) {
return json(
{
error: 'An administrator must save server keys before an operator launches these servers.'
},
{ status: 403 }
);
}
} catch (cause) { return authError(cause); }
} catch (cause) {
return authError(cause);
}
} else if (['/login','/setup'].includes(routeId??'')) {
if (session) redirect(303,session.user.mustChangePassword?'/account':'/');
if (routeId==='/setup'&&!setup) redirect(303,'/login');
if (routeId==='/login'&&setup) redirect(303,'/setup');
}
} else if (['/login', '/setup'].includes(routeId ?? '')) {
if (session) redirect(303, session.user.mustChangePassword ? '/account' : '/');
if (routeId === '/setup' && !setup) redirect(303, '/login');
if (routeId === '/login' && setup) redirect(303, '/setup');
}
let response = await withRequestUsername(session?.user.username ?? null, () => resolve(event));
if (session && !['GET','HEAD','OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id,'request',routeId,{method,status:response.status,path:event.url.pathname});
if (session && !['GET', 'HEAD', 'OPTIONS'].includes(method) && response.status < 400) {
recordAudit(session.user.id, 'request', routeId, {
method,
status: response.status,
path: event.url.pathname
});
}
if (session && response.headers.get('content-type')?.startsWith('text/event-stream')) {
response = protectEventStream(response,session.tokenHash,permission,event.request.signal);
response = protectEventStream(
response,
session.tokenHash,
permission,
event.request.signal
);
}
const headers = new Headers(response.headers);
headers.set('Cache-Control','no-store');
headers.set('Vary',[headers.get('Vary'),'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy','same-origin');
headers.set('X-Content-Type-Options','nosniff');
headers.set('X-Frame-Options','DENY');
return new Response(response.body,{status:response.status,statusText:response.statusText,headers});
headers.set('Cache-Control', 'no-store');
headers.set('Vary', [headers.get('Vary'), 'Cookie'].filter(Boolean).join(', '));
headers.set('Referrer-Policy', 'same-origin');
headers.set('X-Content-Type-Options', 'nosniff');
headers.set('X-Frame-Options', 'DENY');
return new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers
});
};
src/lib/server/auth/http.ts
+76 -22
View File
@@ -2,48 +2,102 @@ import { error, isHttpError, json, type Cookies } from '@sveltejs/kit';
import { AuthError, SESSION_COOKIE, type AuthResult } from './index.js';
import type { AuthUser } from '../../auth.js';
export function sessionCookie(url: URL): string { return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev'; }
export function sessionCookie(url: URL): string {
return url.protocol === 'https:' ? SESSION_COOKIE : 'ec_session_dev';
}
export function requireUserTransport(url: URL): void {
if (url.protocol === 'https:') return;
if (process.env.EC_AUTH_ALLOW_HTTP === 'true' && ['localhost','127.0.0.1','[::1]','::1'].includes(url.hostname)) return;
error(426, 'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.');
if (
process.env.EC_AUTH_ALLOW_HTTP === 'true' &&
['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)
)
return;
error(
426,
'Operator login requires HTTPS. For local development only, set EC_AUTH_ALLOW_HTTP=true and use localhost.'
);
}
export function requireSameOrigin(request: Request, url: URL): void {
if (request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') {
if (
request.headers.get('origin') !== url.origin ||
request.headers.get('sec-fetch-site') === 'cross-site'
) {
error(403, 'This action requires a same-origin request.');
}
}
export function sessionResponse(result: AuthResult, cookies: Cookies, url: URL) {
cookies.set(sessionCookie(url), result.session.token, { path: '/', httpOnly: true, secure: url.protocol === 'https:',
sameSite: 'lax', expires: new Date(result.session.expiresAt) });
return json({ user: result.user }, { headers: { 'Cache-Control':'no-store' } });
cookies.set(sessionCookie(url), result.session.token, {
path: '/',
httpOnly: true,
secure: url.protocol === 'https:',
sameSite: 'lax',
expires: new Date(result.session.expiresAt)
});
return json({ user: result.user }, { headers: { 'Cache-Control': 'no-store' } });
}
export function requireUser(user: AuthUser | null): AuthUser {
if (!user) throw new AuthError('Sign in to continue.',401);
if (!user) throw new AuthError('Sign in to continue.', 401);
return user;
}
export async function authBody(request: Request, maxBytes = 8192): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json')) throw new AuthError('Send a JSON request.',415);
if (Number(request.headers.get('content-length')) > maxBytes) throw new AuthError('Request is too large.',413);
const reader = request.body?.getReader(); if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = []; let length = 0;
export async function authBody(
request: Request,
maxBytes = 8192
): Promise<Record<string, unknown>> {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('application/json'))
throw new AuthError('Send a JSON request.', 415);
if (Number(request.headers.get('content-length')) > maxBytes)
throw new AuthError('Request is too large.', 413);
const reader = request.body?.getReader();
if (!reader) throw new AuthError('A JSON object is required.');
const chunks: Uint8Array[] = [];
let length = 0;
try {
for (;;) {
const { done, value } = await reader.read(); if (done) break;
const { done, value } = await reader.read();
if (done) break;
length += value.byteLength;
if (length > maxBytes) { void reader.cancel().catch(() => {}); throw new AuthError('Request is too large.',413); }
if (length > maxBytes) {
void reader.cancel().catch(() => {});
throw new AuthError('Request is too large.', 413);
}
chunks.push(value);
}
} finally { reader.releaseLock(); }
} finally {
reader.releaseLock();
}
let body: unknown;
try { body = JSON.parse(Buffer.concat(chunks,length).toString('utf8')); } catch { throw new AuthError('Invalid JSON.'); }
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new AuthError('A JSON object is required.');
try {
body = JSON.parse(Buffer.concat(chunks, length).toString('utf8'));
} catch {
throw new AuthError('Invalid JSON.');
}
if (!body || typeof body !== 'object' || Array.isArray(body))
throw new AuthError('A JSON object is required.');
return body as Record<string, unknown>;
}
export function authError(cause: unknown): Response {
if (isHttpError(cause)) return json({ error: cause.body.message }, { status:cause.status, headers:{'Cache-Control':'no-store'} });
if (isHttpError(cause))
return json(
{ error: cause.body.message },
{ status: cause.status, headers: { 'Cache-Control': 'no-store' } }
);
const message = cause instanceof Error ? cause.message : 'Authentication request failed.';
const status = cause instanceof AuthError ? cause.status : message.includes('busy') ? 429 : message.startsWith('Use a password') ? 400 : 500;
return json({ error: status === 500 ? 'The authentication request could not be completed.' : message },
{ status, headers:{'Cache-Control':'no-store',...(status===429?{'Retry-After':'900'}:{})} });
const status =
cause instanceof AuthError
? cause.status
: message.includes('busy')
? 429
: message.startsWith('Use a password')
? 400
: 500;
return json(
{ error: status === 500 ? 'The authentication request could not be completed.' : message },
{
status,
headers: {
'Cache-Control': 'no-store',
...(status === 429 ? { 'Retry-After': '900' } : {})
}
}
);
}
src/lib/server/auth/index.ts
+326 -90
View File
@@ -8,179 +8,415 @@ export const SESSION_COOKIE = '__Host-ec_session';
export const SESSION_LIFETIME_MS = 12 * 60 * 60_000;
export const SESSION_IDLE_MS = 30 * 60_000;
const digest = (value: string) => createHash('sha256').update(value).digest('hex');
function database() { const handle = db(); ensureAuthSchema(handle); return handle; }
type UserRow = { id: string; username: string; display_name: string; password_hash: string; role: UserRole; disabled: number;
must_change_password: number; created_at: number; updated_at: number; last_login_at: number | null };
function database() {
const handle = db();
ensureAuthSchema(handle);
return handle;
}
type UserRow = {
id: string;
username: string;
display_name: string;
password_hash: string;
role: UserRole;
disabled: number;
must_change_password: number;
created_at: number;
updated_at: number;
last_login_at: number | null;
};
export class AuthError extends Error {
constructor(message: string, public status = 400) { super(message); }
constructor(
message: string,
public status = 400
) {
super(message);
}
}
function present(row: UserRow): AuthUser {
return { id: row.id, username: row.username, displayName: row.display_name, role: row.role, disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password, createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(), lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString() };
return {
id: row.id,
username: row.username,
displayName: row.display_name,
role: row.role,
disabled: !!row.disabled,
mustChangePassword: !!row.must_change_password,
createdAt: new Date(row.created_at).toISOString(),
updatedAt: new Date(row.updated_at).toISOString(),
lastLoginAt: row.last_login_at === null ? null : new Date(row.last_login_at).toISOString()
};
}
function userRow(id: string) {
return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined;
}
export function getUser(id: string): AuthUser | null {
const row = userRow(id);
return row ? present(row) : null;
}
export function listUsers(): AuthUser[] {
return (
database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]
).map(present);
}
export function setupRequired(): boolean {
return !database().prepare('SELECT 1 FROM auth_users LIMIT 1').get();
}
function userRow(id: string) { return database().prepare('SELECT * FROM auth_users WHERE id=?').get(id) as UserRow | undefined; }
export function getUser(id: string): AuthUser | null { const row = userRow(id); return row ? present(row) : null; }
export function listUsers(): AuthUser[] { return (database().prepare('SELECT * FROM auth_users ORDER BY username').all() as UserRow[]).map(present); }
export function setupRequired(): boolean { return !(database().prepare('SELECT 1 FROM auth_users LIMIT 1').get()); }
function username(value: unknown): string {
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value)) throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$/.test(value))
throw new AuthError('Usernames need 3–64 letters, numbers, dots, underscores or hyphens.');
const normalized = value.toLowerCase();
if (normalized === systemUsername.toLowerCase()) throw new AuthError(`${systemUsername} is reserved for system actions. Choose another username.`);
if (normalized === systemUsername.toLowerCase())
throw new AuthError(
`${systemUsername} is reserved for system actions. Choose another username.`
);
return normalized;
}
function displayName(value: unknown, fallback: string): string {
const name = value === undefined ? fallback : value;
if (typeof name !== 'string' || !name.trim() || name.length > 100 || /[\x00-\x1f\x7f]/.test(name)) throw new AuthError('Display name must contain 1–100 printable characters.');
if (
typeof name !== 'string' ||
!name.trim() ||
name.length > 100 ||
// eslint-disable-next-line no-control-regex -- Reject control characters in display names.
/[\x00-\x1f\x7f]/.test(name)
)
throw new AuthError('Display name must contain 1–100 printable characters.');
return name.trim();
}
function role(value: unknown): UserRole {
if (!['admin','operator','viewer'].includes(value as string)) throw new AuthError('Choose administrator, operator or viewer.');
if (!['admin', 'operator', 'viewer'].includes(value as string))
throw new AuthError('Choose administrator, operator or viewer.');
return value as UserRole;
}
export function recordAudit(actorId: string | null, action: string, targetId: string | null = null, details: Record<string, unknown> = {}): void {
database().prepare('INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)')
export function recordAudit(
actorId: string | null,
action: string,
targetId: string | null = null,
details: Record<string, unknown> = {}
): void {
database()
.prepare(
'INSERT INTO auth_audit(actor_id,action,target_id,details,created_at) VALUES(?,?,?,?,?)'
)
.run(actorId, action, targetId, JSON.stringify(details), Date.now());
}
export function listAuthAudit() {
return database().prepare(`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`).all();
return database()
.prepare(
`SELECT a.id,a.actor_id AS actorId,u.username AS actor,a.action,a.target_id AS targetId,a.details,a.created_at AS createdAt
FROM auth_audit a LEFT JOIN auth_users u ON u.id=a.actor_id ORDER BY a.id DESC LIMIT 200`
)
.all();
}
/** Persistent fixed windows are consumed before hashing, including failed setup. */
export function consumeAuthAttempt(clientAddress: string, account: string, kind = 'login'): void {
const now = Date.now(); const windowMs = 15 * 60_000;
const now = Date.now();
const windowMs = 15 * 60_000;
const ipKey = digest(`${kind}:ip:${clientAddress}`);
const accountKey = digest(`${kind}:account:${account.toLowerCase()}`);
database().transaction(() => {
database().prepare('DELETE FROM auth_rate_limits WHERE window_start < ?').run(now - 24 * 60 * 60_000);
for (const [key, limit] of [[ipKey, 30], [accountKey, 10]] as const) {
const previous = database().prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?').get(key) as { window_start: number; attempts: number } | undefined;
if (previous && previous.window_start > now - windowMs && previous.attempts >= limit) throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
database()
.transaction(() => {
database()
.prepare('DELETE FROM auth_rate_limits WHERE window_start < ?')
.run(now - 24 * 60 * 60_000);
for (const [key, limit] of [
[ipKey, 30],
[accountKey, 10]
] as const) {
const previous = database()
.prepare('SELECT window_start,attempts FROM auth_rate_limits WHERE key=?')
.get(key) as { window_start: number; attempts: number } | undefined;
if (
previous &&
previous.window_start > now - windowMs &&
previous.attempts >= limit
)
throw new AuthError('Too many attempts. Try again in 15 minutes.', 429);
}
for (const key of [ipKey, accountKey]) database().prepare(`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
for (const key of [ipKey, accountKey])
database()
.prepare(
`INSERT INTO auth_rate_limits(key,window_start,attempts) VALUES(?,?,1)
ON CONFLICT(key) DO UPDATE SET attempts=CASE WHEN window_start <= ? THEN 1 ELSE attempts+1 END,
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`).run(key, now, now - windowMs, now - windowMs);
}).immediate();
window_start=CASE WHEN window_start <= ? THEN excluded.window_start ELSE window_start END`
)
.run(key, now, now - windowMs, now - windowMs);
})
.immediate();
}
function insertUser(input: { username: string; displayName: string; role: UserRole; passwordHash: string; mustChangePassword: boolean }): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username)) throw new AuthError('That username is already in use.', 409);
const id = randomUUID(); const now = Date.now();
database().prepare(`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`)
.run(id,input.username,input.displayName,input.passwordHash,input.role,Number(input.mustChangePassword),now,now);
function insertUser(input: {
username: string;
displayName: string;
role: UserRole;
passwordHash: string;
mustChangePassword: boolean;
}): AuthUser {
if (database().prepare('SELECT 1 FROM auth_users WHERE username=?').get(input.username))
throw new AuthError('That username is already in use.', 409);
const id = randomUUID();
const now = Date.now();
database()
.prepare(
`INSERT INTO auth_users(id,username,display_name,password_hash,role,must_change_password,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`
)
.run(
id,
input.username,
input.displayName,
input.passwordHash,
input.role,
Number(input.mustChangePassword),
now,
now
);
return getUser(id)!;
}
export interface AuthResult { user: AuthUser; session: { token: string; expiresAt: number } }
export async function bootstrapAdmin(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
export interface AuthResult {
user: AuthUser;
session: { token: string; expiresAt: number };
}
export async function bootstrapAdmin(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
consumeAuthAttempt(clientAddress, 'bootstrap', 'setup');
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const expected = process.env.EC_SETUP_TOKEN;
if (!expected || expected.length < 32) throw new AuthError('The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.', 503);
if (typeof input.token !== 'string' || !timingSafeEqual(Buffer.from(digest(input.token), 'hex'), Buffer.from(digest(expected), 'hex'))) throw new AuthError('Invalid setup token.', 401);
const name = username(input.username); const label = displayName(input.displayName, name);
if (!expected || expected.length < 32)
throw new AuthError(
'The controller owner must configure EC_SETUP_TOKEN with at least 32 characters before setup.',
503
);
if (
typeof input.token !== 'string' ||
!timingSafeEqual(
Buffer.from(digest(input.token), 'hex'),
Buffer.from(digest(expected), 'hex')
)
)
throw new AuthError('Invalid setup token.', 401);
const name = username(input.username);
const label = displayName(input.displayName, name);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
return database()
.transaction(() => {
if (!setupRequired()) throw new AuthError('Initial setup is already complete.', 409);
const user = insertUser({ username: name, displayName: label, passwordHash, role: 'admin', mustChangePassword: false });
const user = insertUser({
username: name,
displayName: label,
passwordHash,
role: 'admin',
mustChangePassword: false
});
recordAudit(user.id, 'setup', user.id);
return { user, session: createSession(user.id) };
}).immediate();
})
.immediate();
}
export async function authenticatePassword(input: Record<string, unknown>, clientAddress: string): Promise<AuthResult> {
const name = typeof input.username === 'string' ? input.username.slice(0,64).toLowerCase() : '';
export async function authenticatePassword(
input: Record<string, unknown>,
clientAddress: string
): Promise<AuthResult> {
const name =
typeof input.username === 'string' ? input.username.slice(0, 64).toLowerCase() : '';
consumeAuthAttempt(clientAddress, name);
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as UserRow | undefined;
const row = database().prepare('SELECT * FROM auth_users WHERE username=?').get(name) as
UserRow | undefined;
const valid = await verifyPassword(input.password, row?.password_hash ?? null);
const result = database().transaction(() => {
const result = database()
.transaction(() => {
const current = row ? userRow(row.id) : undefined;
if (!valid || !current || current.disabled || current.password_hash !== row?.password_hash) {
recordAudit(null, 'login.failed'); return null;
if (
!valid ||
!current ||
current.disabled ||
current.password_hash !== row?.password_hash
) {
recordAudit(null, 'login.failed');
return null;
}
database().prepare('UPDATE auth_users SET last_login_at=? WHERE id=?').run(Date.now(), current.id);
database()
.prepare('UPDATE auth_users SET last_login_at=? WHERE id=?')
.run(Date.now(), current.id);
recordAudit(current.id, 'login', current.id);
return { user: getUser(current.id)!, session: createSession(current.id) };
}).immediate();
})
.immediate();
if (!result) throw new AuthError('Invalid username or password.', 401);
return result;
}
export function createSession(userId: string): { token: string; expiresAt: number } {
const now = Date.now(); const user = userRow(userId);
const now = Date.now();
const user = userRow(userId);
if (!user || user.disabled) throw new AuthError('Account is unavailable.', 401);
const token = randomBytes(32).toString('base64url'); const expiresAt = now + SESSION_LIFETIME_MS;
database().prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?').run(now,now-SESSION_IDLE_MS);
database().prepare('INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)').run(digest(token),userId,now,expiresAt,now);
const token = randomBytes(32).toString('base64url');
const expiresAt = now + SESSION_LIFETIME_MS;
database()
.prepare('DELETE FROM auth_sessions WHERE expires_at <= ? OR last_seen_at <= ?')
.run(now, now - SESSION_IDLE_MS);
database()
.prepare(
'INSERT INTO auth_sessions(token_hash,user_id,created_at,expires_at,last_seen_at) VALUES(?,?,?,?,?)'
)
.run(digest(token), userId, now, expiresAt, now);
return { token, expiresAt };
}
export function readSession(token: string | undefined, touch = true): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSession(
token: string | undefined,
touch = true
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null;
return readSessionHash(digest(token), touch);
}
export function readSessionHash(tokenHash: string, touch = false): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
export function readSessionHash(
tokenHash: string,
touch = false
): { user: AuthUser; tokenHash: string; expiresAt: number } | null {
const now = Date.now();
const row = database().prepare(`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`).get(tokenHash,now,now-SESSION_IDLE_MS) as (UserRow & { expires_at: number; last_seen_at: number }) | undefined;
const row = database()
.prepare(
`SELECT u.*,s.expires_at,s.last_seen_at FROM auth_sessions s JOIN auth_users u ON u.id=s.user_id
WHERE s.token_hash=? AND s.expires_at>? AND s.last_seen_at>? AND u.disabled=0`
)
.get(tokenHash, now, now - SESSION_IDLE_MS) as
(UserRow & { expires_at: number; last_seen_at: number }) | undefined;
if (!row) return null;
if (touch && row.last_seen_at < now-60_000) database().prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?').run(now,tokenHash);
if (touch && row.last_seen_at < now - 60_000)
database()
.prepare('UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?')
.run(now, tokenHash);
return { user: present(row), tokenHash, expiresAt: row.expires_at };
}
export function revokeSession(token: string | undefined): void { if (token) database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token)); }
export function revokeSession(token: string | undefined): void {
if (token)
database().prepare('DELETE FROM auth_sessions WHERE token_hash=?').run(digest(token));
}
export function revokeUserSessions(actorId: string, userId: string): void {
database().transaction(() => {
requireAdministrator(actorId);
if (!userRow(userId)) throw new AuthError('User not found.',404);
if (!userRow(userId)) throw new AuthError('User not found.', 404);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(actorId,'sessions.revoked',userId);
recordAudit(actorId, 'sessions.revoked', userId);
})();
}
function requireAdministrator(actorId: string) {
const actor = userRow(actorId);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password) throw new AuthError('Administrator access is required.',403);
if (!actor || actor.disabled || actor.role !== 'admin' || actor.must_change_password)
throw new AuthError('Administrator access is required.', 403);
}
export async function createUser(actorId: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function createUser(
actorId: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
const name = username(input.username), label = displayName(input.displayName,name), userRole = role(input.role);
const name = username(input.username),
label = displayName(input.displayName, name),
userRole = role(input.role);
const passwordHash = await hashPassword(input.password);
return database().transaction(() => {
return database()
.transaction(() => {
requireAdministrator(actorId);
const user = insertUser({ username:name,displayName:label,role:userRole,passwordHash,mustChangePassword:true });
recordAudit(actorId,'user.created',user.id,{role:userRole}); return user;
}).immediate();
const user = insertUser({
username: name,
displayName: label,
role: userRole,
passwordHash,
mustChangePassword: true
});
recordAudit(actorId, 'user.created', user.id, { role: userRole });
return user;
})
.immediate();
}
export async function updateUser(actorId: string, id: string, input: Record<string, unknown>): Promise<AuthUser> {
export async function updateUser(
actorId: string,
id: string,
input: Record<string, unknown>
): Promise<AuthUser> {
requireAdministrator(actorId);
if (input.disabled !== undefined && typeof input.disabled !== 'boolean') throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id) throw new AuthError('Change your own password from Account using your current password.');
if (input.disabled !== undefined && typeof input.disabled !== 'boolean')
throw new AuthError('Disabled must be true or false.');
if (input.password !== undefined && actorId === id)
throw new AuthError('Change your own password from Account using your current password.');
const passwordHash = input.password === undefined ? null : await hashPassword(input.password);
return database().transaction(() => {
return database()
.transaction(() => {
requireAdministrator(actorId);
const current = userRow(id); if (!current) throw new AuthError('User not found.',404);
const current = userRow(id);
if (!current) throw new AuthError('User not found.', 404);
const nextRole = input.role === undefined ? current.role : role(input.role);
const disabled = input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName,current.display_name);
if (current.role === 'admin' && !current.disabled && (nextRole !== 'admin' || disabled)) {
const other = database().prepare("SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1").get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.',409);
const disabled =
input.disabled === undefined ? current.disabled : Number(input.disabled);
const label = displayName(input.displayName, current.display_name);
if (
current.role === 'admin' &&
!current.disabled &&
(nextRole !== 'admin' || disabled)
) {
const other = database()
.prepare(
"SELECT 1 FROM auth_users WHERE role='admin' AND disabled=0 AND id!=? LIMIT 1"
)
.get(id);
if (!other) throw new AuthError('Keep at least one enabled administrator.', 409);
}
database().prepare(`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`)
.run(label,nextRole,disabled,passwordHash??current.password_hash,passwordHash?1:current.must_change_password,Date.now(),id);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role) database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId,'user.updated',id,{role:nextRole,disabled:!!disabled,passwordReset:!!passwordHash});
database()
.prepare(
`UPDATE auth_users SET display_name=?,role=?,disabled=?,password_hash=?,must_change_password=?,updated_at=? WHERE id=?`
)
.run(
label,
nextRole,
disabled,
passwordHash ?? current.password_hash,
passwordHash ? 1 : current.must_change_password,
Date.now(),
id
);
if (passwordHash || disabled !== current.disabled || nextRole !== current.role)
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(id);
recordAudit(actorId, 'user.updated', id, {
role: nextRole,
disabled: !!disabled,
passwordReset: !!passwordHash
});
return getUser(id)!;
}).immediate();
})
.immediate();
}
export async function changePassword(userId: string, currentPassword: unknown, newPassword: unknown): Promise<AuthResult> {
export async function changePassword(
userId: string,
currentPassword: unknown,
newPassword: unknown
): Promise<AuthResult> {
const original = userRow(userId);
if (!original || original.disabled || !await verifyPassword(currentPassword,original.password_hash)) throw new AuthError('Current password is incorrect.',400);
if (
!original ||
original.disabled ||
!(await verifyPassword(currentPassword, original.password_hash))
)
throw new AuthError('Current password is incorrect.', 400);
if (currentPassword === newPassword) throw new AuthError('Choose a different password.');
const passwordHash = await hashPassword(newPassword);
return database().transaction(() => {
return database()
.transaction(() => {
const current = userRow(userId);
if (!current || current.disabled || current.password_hash !== original.password_hash) throw new AuthError('Account changed during password verification. Sign in again.',409);
database().prepare('UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?').run(passwordHash,Date.now(),userId);
if (!current || current.disabled || current.password_hash !== original.password_hash)
throw new AuthError(
'Account changed during password verification. Sign in again.',
409
);
database()
.prepare(
'UPDATE auth_users SET password_hash=?,must_change_password=0,updated_at=? WHERE id=?'
)
.run(passwordHash, Date.now(), userId);
database().prepare('DELETE FROM auth_sessions WHERE user_id=?').run(userId);
recordAudit(userId,'password.changed',userId);
recordAudit(userId, 'password.changed', userId);
return { user: getUser(userId)!, session: createSession(userId) };
}).immediate();
})
.immediate();
}
src/lib/server/auth/passwords.js
+15 -5
View File
@@ -15,12 +15,14 @@ export function validatePassword(password) {
/** @param {string} password @param {string} salt @returns {Promise<Buffer>} */
function derive(password, salt) {
if (active >= 2) return Promise.reject(new Error('Password service is busy. Try again shortly.'));
if (active >= 2)
return Promise.reject(new Error('Password service is busy. Try again shortly.'));
active++;
return new Promise((resolve, reject) => {
scrypt(password, salt, 64, parameters, (error, result) => {
active--;
if (error) reject(error); else resolve(result);
if (error) reject(error);
else resolve(result);
});
});
}
@@ -37,9 +39,17 @@ export async function hashPassword(password) {
export async function verifyPassword(password, encoded) {
if (typeof password !== 'string' || [...password].length > 128) return false;
const parts = encoded?.split('$');
const valid = parts?.length === 6 && parts[0] === 'scrypt' && parts[1] === '131072' && parts[2] === '8' && parts[3] === '1'
&& /^[a-f0-9]{64}$/.test(parts[4]) && /^[a-f0-9]{128}$/.test(parts[5]);
const valid =
parts?.length === 6 &&
parts[0] === 'scrypt' &&
parts[1] === '131072' &&
parts[2] === '8' &&
parts[3] === '1' &&
/^[a-f0-9]{64}$/.test(parts[4]) &&
/^[a-f0-9]{128}$/.test(parts[5]);
// Missing accounts perform the same expensive operation as real accounts.
const result = await derive(password, valid ? parts[4] : '0'.repeat(64));
return timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid;
return (
timingSafeEqual(result, Buffer.from(valid ? parts[5] : '0'.repeat(128), 'hex')) && !!valid
);
}
src/lib/server/auth/request-context.ts
+9 -3
View File
@@ -3,11 +3,17 @@ import { AsyncLocalStorage } from 'node:async_hooks';
const requester = new AsyncLocalStorage<{ username: string | null; active: boolean }>();
/** Use only the authenticated session, and stop attribution when the request finishes. */
export async function withRequestUsername<T>(username: string | null, work: () => T | Promise<T>): Promise<T> {
export async function withRequestUsername<T>(
username: string | null,
work: () => T | Promise<T>
): Promise<T> {
const context = { username, active: true };
return requester.run(context, async () => {
try { return await work(); }
finally { context.active = false; }
try {
return await work();
} finally {
context.active = false;
}
});
}
src/lib/server/auth/telemetry.ts
+17 -8
View File
@@ -1,24 +1,32 @@
import { createHash, timingSafeEqual } from 'node:crypto';
export type TelemetryAuthentication =
| { ok: true }
| { ok: false; status: 401 | 503; error: string };
{ ok: true } | { ok: false; status: 401 | 503; error: string };
const tokenPattern = /^[A-Za-z0-9._~+\/-]+=*$/;
const validToken = (value: string): boolean => value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const tokenPattern = /^[A-Za-z0-9._~+/-]+=*$/;
const validToken = (value: string): boolean =>
value.length >= 32 && value.length <= 512 && tokenPattern.test(value);
const hash = (value: string): Buffer => createHash('sha256').update(value, 'utf8').digest();
/** Machine telemetry has its own credential; browser sessions never authorize it. */
export function authenticateTelemetry(request: Request): TelemetryAuthentication {
const expected = process.env.EC_TELEMETRY_TOKEN ?? '';
if (!validToken(expected)) {
return { ok: false, status: 503, error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.' };
return {
ok: false,
status: 503,
error: 'Game telemetry authentication is not configured. Set EC_TELEMETRY_TOKEN to a secret of at least 32 characters.'
};
}
const authorization = request.headers.get('authorization');
const custom = request.headers.get('x-ec-telemetry-token');
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+\/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({ ok: false, status: 401, error: 'A valid game telemetry token is required.' });
const bearer = authorization?.match(/^Bearer ([A-Za-z0-9._~+/-]+=*)$/i)?.[1];
const denied = (): TelemetryAuthentication => ({
ok: false,
status: 401,
error: 'A valid game telemetry token is required.'
});
// Reject malformed/ambiguous credentials rather than falling back to a
// second header. A shared secret must never come from a URL, cookie or body.
@@ -28,7 +36,8 @@ export function authenticateTelemetry(request: Request): TelemetryAuthentication
const supplied = bearer ?? custom;
if (!supplied) return denied();
const suppliedHash = hash(supplied);
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom))) return denied();
if (bearer !== undefined && custom !== null && !timingSafeEqual(suppliedHash, hash(custom)))
return denied();
if (!timingSafeEqual(suppliedHash, hash(expected))) return denied();
return { ok: true };
}
src/lib/server/autoscaling/admission.ts
+55 -10
View File
@@ -4,29 +4,74 @@ import { scalingSnapshot } from './snapshot.js';
import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
import { deploymentStage } from '../provisioning/history.js';
export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
export interface ScalingIntent {
policy_id: string;
revision: number;
decision: string;
operation_id: string;
}
export function scalingIntent(operationId: string): ScalingIntent | undefined {
return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
return db()
.prepare(
'SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0'
)
.get(operationId) as ScalingIntent | undefined;
}
/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
export function scalingAdmission(operationId: string): string | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
const row = db()
.prepare('SELECT * FROM autoscaling_policies WHERE id=?')
.get(intent.policy_id) as
| {
id: string;
settings: string;
mode: ScalingPolicy['mode'];
revision: number;
paused_reason: string | null;
owns_map: number;
}
| undefined;
let reason: string | null = null;
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
const paused =
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
if (
!row ||
!liveScaling(row.mode) ||
!row.owns_map ||
row.revision !== intent.revision ||
row.paused_reason ||
paused
)
reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = {
...JSON.parse(row.settings),
id: row.id,
mode: row.mode,
revision: row.revision,
pausedReason: null,
updatedAt: ''
};
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
snapshot.reservedNames = snapshot.reservedNames.filter((n) => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(
(b) => b !== `Unresolved native instance ${requested.server}.`
);
}
const current = planScaling(policy, snapshot, Date.now());
if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
if (current.action !== requested.action || current.server !== requested.server)
reason = `Autoscaling intent no longer eligible: ${current.reason}`;
}
if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
if (reason)
deploymentStage(operationId, reason, {
blocker: { code: 'autoscaling', message: reason, automatic: true }
});
return reason;
}
src/lib/server/autoscaling/authorization.ts
+77 -17
View File
@@ -7,46 +7,106 @@ export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
export function removalAuthorization(operationId: string, hostId: string): object | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability))
throw new Error('Update the host agent before automatic removal.');
const reason = scalingAdmission(operationId);
if (reason) throw new Error(reason);
const expiresAt = Date.now() + 60_000;
db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
db()
.prepare(
'INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)'
)
.run(operationId, intent.policy_id, intent.revision, expiresAt);
return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
}
/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
export function claimRemovalAuthorization(
hostId: string,
commandId: string,
input: Record<string, unknown>
): { authorized: boolean; validForMs?: number; reason?: string } {
return db().transaction(() => {
const now = Date.now();
const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
const row = db()
.prepare(
`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
{ operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
|| row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
|| row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`
)
.get(commandId, hostId) as
| {
operation_id: string;
policy_id: string;
revision: number;
expires_at: number;
claimed_at: number | null;
revoked_at: number | null;
deployment_state: string;
command_state: string;
}
| undefined;
if (
!row ||
input.policyId !== row.policy_id ||
input.revision !== row.revision ||
input.expiresAt !== row.expires_at ||
row.claimed_at !== null ||
row.revoked_at !== null ||
row.expires_at <= now ||
row.deployment_state !== 'running' ||
!['pending', 'accepted', 'running'].includes(row.command_state)
)
return {
authorized: false,
reason: 'Removal authorization expired, changed, or was already claimed.'
};
const reason = scalingAdmission(row.operation_id);
if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
if (reason) {
db()
.prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?')
.run(now, row.operation_id);
return { authorized: false, reason };
}
db()
.prepare(
'UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL'
)
.run(now, row.operation_id);
// Only a short final occupancy check may occur after this boundary.
return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
})();
}
export function revokePolicyRemovals(policyId: string): void {
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId))
return;
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL'
)
.run(Date.now(), policyId);
const at = new Date().toISOString();
const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
const commands = db()
.prepare(
`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`
)
.all(policyId) as { id: string }[];
for (const command of commands) {
const result = JSON.stringify({ autoscalingInvalidated: true });
db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
db()
.prepare(
"UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?"
)
.run(result, at, command.id);
db()
.prepare(
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')"
)
.run(command.id, at);
}
}
src/lib/server/autoscaling/index.ts
+427 -86
View File
@@ -1,6 +1,13 @@
import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingEvaluation, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import {
liveScaling,
scalingSettings,
type ScalingPolicy,
type ScalingEvaluation,
type ScalingReport,
type ScalingView
} from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
@@ -14,33 +21,98 @@ import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
export class ScalingError extends Error {
constructor(
message: string,
public status = 400
) {
super(message);
}
}
interface PolicyRow {
id: string;
revision: number;
settings: string;
mode: ScalingPolicy['mode'];
paused_reason: string | null;
updated_at: string;
}
const present = (row: PolicyRow): ScalingPolicy => ({
...JSON.parse(row.settings),
id: row.id,
revision: row.revision,
mode: row.mode,
pausedReason: row.paused_reason,
updatedAt: row.updated_at
});
export const listScalingPolicies = (): ScalingPolicy[] =>
(
db()
.prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug')
.all() as PolicyRow[]
).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined {
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as
PolicyRow | undefined;
return row && present(row);
}
export const scalingGloballyPaused = (): boolean =>
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
export function evaluateScaling(input: unknown, id?: string, revision?: number): ScalingEvaluation {
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
const settings = scalingSettings(input),
existing = id ? getScalingPolicy(id) : undefined;
if (id && !existing) throw new ScalingError('Policy not found.', 404);
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'new-policy', revision: existing?.revision ?? 0,
pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
if (existing && existing.revision !== revision)
throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug))
throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = {
...settings,
id: existing?.id ?? 'new-policy',
revision: existing?.revision ?? 0,
pausedReason: null,
updatedAt: existing?.updatedAt ?? ''
};
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
if (owner && owner !== policy.id)
blockers.push(
`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`
);
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
for (const member of snapshot.members)
if (!['occupied', 'empty'].includes(member.state))
blockers.push(
`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`
);
if (policy.mode === 'create-destroy')
for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
if (
!hostId ||
!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)
)
blockers.push(
`Update the host agent for ${member.name} before enabling automatic removal.`
);
}
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
)
blockers.push('Settle the current autoscaling operation before editing a live policy.');
try {
nativeLaunchSelection({ automatic: true }, policy.game);
} catch (error) {
blockers.push((error as Error).message);
}
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
}
const decision = planScaling(policy, snapshot, Date.now());
return { policy, snapshot, decision, blockers };
@@ -50,16 +122,48 @@ export function saveScaling(input: unknown, id?: string, revision?: number): Sca
return db().transaction(() => {
const evaluation = evaluateScaling(input, id, revision);
if (evaluation.blockers.length) throw new ScalingError(evaluation.blockers[0]);
const policy = evaluation.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
const policy = evaluation.policy,
settings = scalingSettings(policy),
nextId = id ?? randomUUID(),
at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
if (
!id &&
db()
.prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?')
.get(policy.game, policy.mapSlug)
)
throw new ScalingError('A policy already exists for this scope.', 409);
db()
.prepare(
`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`
)
.run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode)) for (const member of evaluation.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
.run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
db()
.prepare(
'UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?'
)
.run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode))
for (const member of evaluation.snapshot.members)
db()
.prepare(
'INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)'
)
.run(
nextId,
member.name,
member.identity,
JSON.stringify({
identity: member.identity,
observedAt: member.observedAt,
emptySince: null,
evaluatedAt: Date.now()
})
);
reconcileScaling(getScalingPolicy(nextId)!);
return getScalingPolicy(nextId)!;
})();
@@ -69,10 +173,18 @@ export function deleteScaling(id: string, revision: number): void {
db().transaction(() => {
const policy = getScalingPolicy(id);
if (!policy) throw new ScalingError('Policy not found.', 404);
if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409);
if (policy.revision !== revision)
throw new ScalingError('Policy changed. Refresh before deleting.', 409);
// Keep admission evidence and ownership while a host command may still act.
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) {
throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409);
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(id)
) {
throw new ScalingError(
'Pause this policy and settle its outstanding Provisioning work before deleting it.',
409
);
}
revokePolicyRemovals(id);
db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id);
@@ -85,39 +197,105 @@ export function deleteScaling(id: string, revision: number): void {
export function setScalingGlobalPause(paused: boolean): void {
db().transaction(() => {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
db()
.prepare(
"INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value"
)
.run(paused ? '1' : '0');
if (paused)
for (const policy of listScalingPolicies()) {
revokePolicyRemovals(policy.id);
reconcileScaling(policy);
}
})();
}
function reconcileScaling(policy: ScalingPolicy): void {
const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
const intents = db()
.prepare(
`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
{ id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as {
id: string;
revision: number;
operation_id: string;
action: string;
state: string;
submitted_at: string | null;
command_id: string | null;
}[];
for (const intent of intents) {
if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
&& (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (
intent.action === 'create' &&
['queued', 'waiting', 'cancelled'].includes(intent.state) &&
!intent.command_id &&
!intent.submitted_at &&
(scalingAdmission(intent.operation_id) || intent.state === 'cancelled')
)
cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
revokePolicyRemovals(policy.id);
if (!intent.command_id) cancelRemoval(intent.operation_id, false);
}
const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done','warning','failed','cancelled'].includes(current.state);
const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
&& !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
const current = db()
.prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?')
.get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done', 'warning', 'failed', 'cancelled'].includes(current.state);
const command = intent.command_id
? (db()
.prepare('SELECT state,result FROM ec_host_operations WHERE id=?')
.get(intent.command_id) as { state: string; result: string | null } | undefined)
: undefined;
const invalidated =
command?.state === 'failed' &&
command.result &&
JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending =
current.state === 'cancelled' &&
!current.server_deleted_at &&
command?.state === 'completed' &&
!!db()
.prepare(
'SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL'
)
.get(intent.operation_id);
const settled =
(!intent.command_id ||
(!!command && ['completed', 'failed'].includes(command.state))) &&
!receiptPending;
if (
!invalidated &&
(current.state === 'failed' ||
current.state === 'interrupted' ||
command?.state === 'unknown') &&
liveScaling(policy.mode)
) {
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?')
.run(
`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`,
new Date().toISOString(),
policy.id
);
}
if (terminal && settled) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled'))
db()
.prepare(
"UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?"
)
.run(policy.id);
}
}
if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
if (
!liveScaling(policy.mode) &&
!db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
) {
db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
}
}
@@ -131,48 +309,152 @@ export async function runAutoscaling(): Promise<boolean> {
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
if (scalingGloballyPaused())
snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db()
.prepare(
`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.get(policy.id) as { id: string; stage: string } | undefined;
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
// Keep the observation interval that justified a pending removal;
// admission reads fresh evidence and revokes it on any new blocker.
if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
if (
member.state === 'removing' &&
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0'
)
.get(policy.id, member.operationId)
)
continue;
const observation = emptyObservation(
member,
{
identity: member.identity,
observedAt: member.observedAt,
emptySince: member.emptySince,
evaluatedAt: now
},
now
);
db()
.prepare(
`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`
)
.run(policy.id, member.name, member.identity, JSON.stringify(observation));
}
// Evidence includes provenance and timestamps but never keys or player rosters.
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy');
const evidence = JSON.stringify({
members: snapshot.members,
reservations: snapshot.reservedNames,
keyPool: snapshot.keyPool,
blockers: snapshot.blockers
});
const fingerprint = createHash('sha256')
.update(
JSON.stringify([
policy.revision,
decision,
snapshot.members.map((m) => [m.name, m.identity, m.state])
])
)
.digest('hex');
const last = db()
.prepare(
'SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1'
)
.get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute =
liveScaling(policy.mode) &&
(decision.action === 'create' || decision.action === 'destroy');
const decisionId = randomUUID();
if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
if (!execute && last?.fingerprint === fingerprint)
db()
.prepare(
'UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?'
)
.run(now, evidence, last.id);
else
db()
.prepare(
'INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)'
)
.run(
decisionId,
policy.id,
policy.revision,
now,
now,
fingerprint,
JSON.stringify(decision),
evidence
);
if (execute) {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?')
.run(decisionId);
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
const result =
decision.action === 'create'
? queueNativeCreationIntent({
server: decision.server!,
game: policy.game,
mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection(
{ automatic: true },
policy.game
),
createdBy: `autoscaling:${policy.id}`,
autoscalingDecisionId: decisionId
})
: destroyServer(
decision.server!,
`autoscaling:${policy.id}`,
decisionId
);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
if (!result.deploymentId)
throw new Error('Provisioning did not return an operation identity.');
db()
.prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?')
.run(result.deploymentId, decisionId);
db()
.prepare(
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)"
)
.run(result.deploymentId, policy.id, decision.server);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?')
.run(decisionId);
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?')
.run(
`Scaling action could not be queued: ${(error as Error).message}`,
policy.id
);
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
db()
.prepare(
`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`
)
.run(
policy.id,
Math.floor(now / 15_000) * 15_000,
policy.mode,
decision.occupied,
decision.empty,
decision.unknown,
decision.deficit
);
})();
}
return submitted;
@@ -180,37 +462,96 @@ export async function runAutoscaling(): Promise<boolean> {
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
db()
.prepare(
`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`
)
.run(cutoff);
db()
.prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1')
.run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
export function scalingReport(): ScalingReport {
const policies: ScalingView[] = listScalingPolicies().map(policy => {
const policies: ScalingView[] = listScalingPolicies().map((policy) => {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
const history = (
db()
.prepare(
`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`
)
.all(policy.id) as {
id: string;
evaluated_at: number;
last_seen_at: number;
decision: string;
operation_id: string | null;
operation_state: string | null;
}[]
).map((row) => ({
id: row.id,
at: row.evaluated_at,
lastSeenAt: row.last_seen_at,
decision: JSON.parse(row.decision),
operationId: row.operation_id,
operationState: row.operation_state
}));
const unsettled = db()
.prepare(
`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(
db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as {
owns_map: number;
}
).owns_map;
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
const totals = db()
.prepare(
`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`
)
.get(policy.id) as ScalingView['totals'];
const operations = db()
.prepare(
`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
return { policy, snapshot, decision, blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`
)
.get(
policy.id,
new Date(Date.now() - 31 * 86400_000).toISOString()
) as ScalingView['operations'];
return {
policy,
snapshot,
decision,
blockers: [],
history,
lastEvaluatedAt: history[0]?.lastSeenAt ?? null,
totals,
operations,
unsettled,
ownsMap
};
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
return {
policies,
globalPaused: scalingGloballyPaused(),
liveAvailable: true,
removalAvailable: true
};
}
src/lib/server/autoscaling/ownership.ts
+36 -8
View File
@@ -4,18 +4,29 @@ import { liveScaling, type ScalingPolicy } from '../../autoscaling.js';
import { mapIdentity } from './snapshot.js';
function owningPolicy(game: string, mapSlug: string) {
return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1")
.get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
return db()
.prepare(
"SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1"
)
.get(game, mapSlug, mapSlug) as
{ id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
}
/** Durable ownership prevents overlapping policies even while scaling is paused. */
export function scalingOwner(game: string, mapSlug: string): string | null {
return owningPolicy(game, mapSlug)?.id ?? null;
}
const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
const ownershipMessage = (owner: string) =>
`Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
function manualLineupBlocker(game: string, mapSlug: string): string | null {
const owner = owningPolicy(game, mapSlug);
if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null;
if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null;
if (
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1'
)
return null;
return ownershipMessage(owner.id);
}
export function scalingOwnershipBlocker(server: string): string | null {
@@ -30,13 +41,30 @@ export function assertScalingOwnership(game: string, mapSlug: string, decisionId
}
const owner = scalingOwner(game, mapSlug);
if (!owner) throw new Error('Autoscaling no longer owns this map.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
if (
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL'
)
.get(decisionId, owner)
)
return;
throw new Error(ownershipMessage(owner));
}
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
export function pauseScalingForOperation(operationId: string): void {
const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
const row = db()
.prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?')
.get(operationId) as { policy_id: string } | undefined;
if (!row) return;
db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
db()
.prepare(
"UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?"
)
.run(new Date().toISOString(), row.policy_id);
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL'
)
.run(Date.now(), row.policy_id);
}
src/lib/server/autoscaling/planner.ts
+141 -43
View File
@@ -1,72 +1,170 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import type {
ScalingDecision,
ScalingMember,
ScalingPolicy,
ScalingSnapshot
} from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
export interface EmptyObservation {
identity: string;
observedAt: number | null;
emptySince: number | null;
evaluatedAt: number;
}
/** Only distinct, continuous evidence can advance an empty interval. */
export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
export function emptyObservation(
member: ScalingMember,
previous: EmptyObservation | undefined,
now: number
): EmptyObservation {
const stamp = member.observedAt;
const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
&& (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
&& previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
&& now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
const empty =
member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous =
empty &&
previous?.identity === member.identity &&
previous.emptySince !== null &&
(!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) &&
previous.observedAt !== null &&
stamp >= previous.observedAt &&
stamp - previous.observedAt <= ONLINE_MAX_MS &&
now >= previous.evaluatedAt &&
now - previous.evaluatedAt <= ONLINE_MAX_MS;
return {
identity: member.identity,
observedAt: stamp,
emptySince: empty ? (continuous ? previous!.emptySince : stamp) : null,
evaluatedAt: now
};
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
export function planScaling(
policy: ScalingPolicy,
snapshot: ScalingSnapshot,
now: number
): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
const decisions = maps.map((map) => {
const matches = (name: string) =>
(snapshot.members.find((m) => m.name === name)?.mapSlug ??
parseServerName(name)?.mapSlug) === map.slug;
return planScaling(
{ ...policy, mapSlug: map.slug },
{
...snapshot,
members: snapshot.members.filter((m) => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
}, now);
},
now
);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const pending = snapshot.members.find(
(m) => m.state === 'creating' || m.state === 'removing'
);
const selected =
decisions.find((d) => d.action === 'create') ??
decisions.find((d) => d.action === 'destroy') ??
decisions.find((d) => d.deficit > 0 || d.unknown > 0) ??
decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
return {
...selected,
action: pending ? 'wait' : selected.action,
server: pending ? null : selected.server,
reason: pending
? (pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.`)
: `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0),
deficit: decisions.reduce((n, d) => n + d.deficit, 0),
occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0),
pending: decisions.reduce((n, d) => n + d.pending, 0),
unknown: decisions.reduce((n, d) => n + d.unknown, 0)
};
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const occupied = members.filter((m) => m.state === 'occupied').length,
empty = members.filter((m) => m.state === 'empty').length;
const pending = members.filter((m) => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter((m) => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
const result = (
action: ScalingDecision['action'],
reason: string,
server: string | null = null
): ScalingDecision => ({
action,
reason,
server,
desired,
deficit: Math.max(0, policy.spares - empty),
occupied,
empty,
pending,
unknown
});
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
const blocked = members.find((m) => !['occupied', 'empty'].includes(m.state));
if (blocked)
return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (
members.some(
(m) => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS
)
)
return result('wait', 'Waiting for fresh occupancy evidence.');
if (occupied + empty < desired) {
if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
if (snapshot.reservedNames.length >= policy.maximum)
return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available)
return result(
'wait',
snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`);
const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`)
.find(name => !snapshot.reservedNames.includes(name));
return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.');
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`
);
const available = Array.from(
{ length: 20 },
(_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`
).find((name) => !snapshot.reservedNames.includes(name));
return available
? result('create', 'Replenish empty lobby capacity.', available)
: result('wait', 'No unused lobby identity is available.');
}
if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
// Protect the lowest-numbered spares before checking removal timers.
// Lobby 1 must survive even when a higher lobby has only just emptied.
const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER)
|| a.name.localeCompare(b.name, 'en', { numeric: true }));
const orderedEmpty = members
.filter((m) => m.state === 'empty')
.sort(
(a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) -
(parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) ||
a.name.localeCompare(b.name, 'en', { numeric: true })
);
const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied);
const candidates = orderedEmpty.slice(requiredEmpty).reverse()
.filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000);
if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
const candidates = orderedEmpty
.slice(requiredEmpty)
.reverse()
.filter(
(m) =>
m.emptySince !== null &&
m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000
);
if (candidates.length)
return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
return result('wait', 'Waiting for a continuous empty interval.');
}
return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
return result(
'wait',
empty < policy.spares
? 'Maximum reached; the empty spare target cannot be met.'
: 'Empty spare target is satisfied.'
);
}
src/lib/server/autoscaling/schema.ts
+13 -5
View File
@@ -36,20 +36,28 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
`);
handle.transaction(() => {
const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[];
if (columns.some(column => column.name === 'next_action_at')) {
if (columns.some((column) => column.name === 'next_action_at')) {
handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at');
}
handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes')
WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=?
handle
.prepare(
`UPDATE autoscaling_authorizations SET revoked_at=?
WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now());
handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`
)
.run(Date.now());
handle
.prepare(
`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
revision=revision+1, paused_reason='This autoscaling mode was removed. Review and turn autoscaling On to resume.',
owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0)
THEN owns_map ELSE 0 END, updated_at=?
WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString());
WHERE mode IN ('dry-run','create-only')`
)
.run(new Date().toISOString());
})();
}
src/lib/server/autoscaling/snapshot.ts
+186 -42
View File
@@ -7,74 +7,218 @@ import { freshStatusTimestamp } from '../../server-freshness.js';
import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { emptyObservation, type EmptyObservation } from './planner.js';
export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
export function mapIdentity(server: {
name: string;
game?: string | null;
map_slug?: string | null;
map?: string | null;
}): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name),
game = server.game ?? parsed?.game;
if (!game) return null;
if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
if (server.map_slug && zombieMaps(game).some((m) => m.slug === server.map_slug))
return { game, mapSlug: server.map_slug };
if (server.map_slug) return null;
if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter(m => m.canonical === server.map);
if (
parsed?.game === game &&
parsed.mapCanonical &&
(!server.map || parsed.mapCanonical === server.map)
)
return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter((m) => m.canonical === server.map);
return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
}
export function protectedWork(server: string, exceptOperation = ''): string | null {
const handle = db();
if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
if (
handle
.prepare(
`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
)
.get(server)
)
return 'Active beta';
if (
handle
.prepare(
`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.get(server)
)
return 'Campaign work';
if (
handle
.prepare(
`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`
)
.get(server)
)
return 'Maintenance work';
if (
handle
.prepare(
`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`
)
.get(server, exceptOperation)
)
return 'Unresolved recovery';
return null;
}
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
export function scalingSnapshot(
policy: ScalingPolicy,
now = Date.now(),
exceptOperation = ''
): ScalingSnapshot {
const handle = db(),
observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const matches = (value: Parameters<typeof mapIdentity>[0]) => {
const identity = mapIdentity(value);
return (
identity?.game === policy.game &&
(policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug)
);
};
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
for (const row of rows)
if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game)
blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle
.prepare(
`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`
)
.all(exceptOperation) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
}[];
const reservations = handle
.prepare(
`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`
)
.all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
const observation = observe(row), operation = operations.find(o => o.server === row.name);
const observation = observe(row),
operation = operations.find((o) => o.server === row.name);
const protection = protectedWork(row.name, exceptOperation);
const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
const host = row.host_id
? (handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as
{ mode: string } | undefined)
: undefined;
const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
let state: ScalingMember['state'] = 'unknown',
blocker: string | null = null;
let players = observation.health === 'online' ? observation.last_players : null;
let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
let observedAt =
players !== null && observation.last_seen_at
? Date.parse(observation.last_seen_at)
: null;
let source = observation.status_source;
// Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
if (
(row.last_players ?? 0) > 0 &&
freshStatusTimestamp(row.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))
) {
players = row.last_players;
observedAt = Date.parse(row.last_seen_at!);
source = 'rcon';
}
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
if (protection || !row.native_instance_id || host?.mode !== 'active') {
state = 'protected';
blocker = protection ?? 'Native instance on an active host required';
} else if (operation) {
state =
operation.action === 'create'
? 'creating'
: operation.action === 'destroy'
? 'removing'
: 'protected';
blocker = `Provisioning ${operation.id}: ${operation.stage}`;
} else if (
row.desired_state === 'stopped' ||
['stopped', 'exited', 'created', 'dead'].includes(row.container_state ?? '')
)
state = 'stopped';
else if (
observation.health === 'online' &&
players !== null &&
observedAt !== null &&
observedAt <= now &&
freshStatusTimestamp(new Date(observedAt).toISOString(), now)
)
state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since]
.map((at) => Date.parse(at ?? ''))
.filter((at) => Number.isFinite(at) && at <= now);
return {
name: row.name,
mapSlug: mapIdentity(row)!.mapSlug,
identity,
state,
players,
observedAt,
source: source ?? null,
emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null,
blocker,
operationId: operation?.id ?? null
};
});
for (const row of reservations.filter(matches))
if (!members.some((m) => m.name === row.name)) {
const operation = operations.find((o) => o.server === row.name);
members.push({
name: row.name,
mapSlug: row.map_slug,
identity: 'reserved',
state: 'creating',
players: null,
observedAt: null,
source: null,
emptySince: null,
blocker: operation?.stage ?? 'Creation is pending.',
operationId: operation?.id ?? null
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
const retained = handle
.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?')
.all(policy.id) as { server: string; observation: string }[];
for (const member of members) {
const saved = retained.find(r => r.server === member.name);
const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
const saved = retained.find((r) => r.server === member.name);
const previous = saved ? (JSON.parse(saved.observation) as EmptyObservation) : undefined;
member.emptySince = emptyObservation(member, previous, now).emptySince;
}
const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
const reservedNames = new Set([
...rows.filter(matches).map((r) => r.name),
...reservations.filter(matches).map((r) => r.name)
]);
for (const row of handle
.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL')
.all() as { name: string }[])
if (matches(row)) {
reservedNames.add(row.name);
if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
if (!members.some((m) => m.name === row.name))
blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined;
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as {
name: string;
}[])
if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation
? (
handle
.prepare('SELECT server FROM server_deployments WHERE id=?')
.get(exceptOperation) as { server: string } | undefined
)?.server
: undefined;
const { total, available } = serverKeyPool(policy.game, ownServer);
return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers };
}
src/lib/server/backends/files.ts
+107 -29
View File
@@ -16,28 +16,65 @@ function remote(path: string) {
}
return null;
}
function missing(path: string): never { throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' }); }
function missing(path: string): never {
throw Object.assign(new Error(`File does not exist: ${path}`), { code: 'ENOENT' });
}
export interface FileFingerprint { size: number; sha256: string }
export interface FileFingerprint {
size: number;
sha256: string;
}
/** One scoped agent scan; null preserves compatibility with older agents and local files. */
export async function nativeFileManifest(root: string, directories: string[], ignoreNames: string[], managedPaths: string[]): Promise<Map<string, FileFingerprint> | null> {
export async function nativeFileManifest(
root: string,
directories: string[],
ignoreNames: string[],
managedPaths: string[]
): Promise<Map<string, FileFingerprint> | null> {
const target = remote(root);
if (!target || target.path !== '.' || !getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')) return null;
const roots = [...new Set(directories.map(directory => relative(root, directory).split(sep).join('/')))];
if (
!target ||
target.path !== '.' ||
!getHost(target.instance.host_id)?.inventory.capabilities?.includes('files.manifest')
)
return null;
const roots = [
...new Set(directories.map((directory) => relative(root, directory).split(sep).join('/')))
];
// Preserve legacy configurations that scan the entire instance root.
if (roots.some(directory => !directory)) return null;
const paths = [...new Set(managedPaths.map(path => relative(root, path).split(sep).join('/')))];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(target.instance, 'files.manifest', {
roots, paths, ignoreNames: ignoreNames.filter(name => !paths.some(path => path.split('/').includes(name)))
});
if (!result || !Array.isArray(result.files) || result.files.length > 10_000) throw new Error('Agent returned an invalid file manifest.');
if (roots.some((directory) => !directory)) return null;
const paths = [
...new Set(managedPaths.map((path) => relative(root, path).split(sep).join('/')))
];
const result = await nativeRequest<{ files: Array<FileFingerprint & { path: string }> }>(
target.instance,
'files.manifest',
{
roots,
paths,
ignoreNames: ignoreNames.filter(
(name) => !paths.some((path) => path.split('/').includes(name))
)
}
);
if (!result || !Array.isArray(result.files) || result.files.length > 10_000)
throw new Error('Agent returned an invalid file manifest.');
const manifest = new Map<string, FileFingerprint>();
for (const entry of result.files) {
if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.includes('\0')
|| posix.normalize(entry.path) !== entry.path || entry.path.split('/').some(part => part === '..' || part === '.')
|| !roots.some(directory => entry.path.startsWith(`${directory}/`))
|| !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) {
if (
!entry ||
typeof entry.path !== 'string' ||
!entry.path ||
entry.path.includes('\\') ||
entry.path.includes('\0') ||
posix.normalize(entry.path) !== entry.path ||
entry.path.split('/').some((part) => part === '..' || part === '.') ||
!roots.some((directory) => entry.path.startsWith(`${directory}/`)) ||
!Number.isSafeInteger(entry.size) ||
entry.size < 0 ||
!/^[a-f0-9]{64}$/.test(entry.sha256)
) {
throw new Error('Agent returned an invalid file manifest entry.');
}
const path = join(root, entry.path);
@@ -50,44 +87,85 @@ export async function nativeFileManifest(root: string, directories: string[], ig
export async function readFile(path: string): Promise<Buffer> {
const target = remote(path);
if (!target) return fs.readFile(path);
const result = await nativeRequest<{ contentSha256?: string; contentBase64?: string | null } | null>(target.instance, 'files.read', { path: target.path });
const result = await nativeRequest<{
contentSha256?: string;
contentBase64?: string | null;
} | null>(target.instance, 'files.read', { path: target.path });
// Stored receipts reference shared bytes; read them directly without a base64 round trip.
if (typeof result?.contentSha256 === 'string') return getFileBlob(result.contentSha256);
if (result?.contentBase64 === null) return missing(path);
if (typeof result?.contentBase64 === 'string') return Buffer.from(result.contentBase64, 'base64');
if (typeof result?.contentBase64 === 'string')
return Buffer.from(result.contentBase64, 'base64');
throw new Error(`Agent returned an invalid response for files.read: ${target.path}`);
}
export async function exists(path: string): Promise<boolean> {
try { await stat(path); return true; }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; }
try {
await stat(path);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
throw error;
}
}
export async function stat(path: string): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
export async function stat(
path: string
): Promise<{ size: number; mtime: Date; isDirectory(): boolean; isFile(): boolean }> {
const target = remote(path);
if (!target) return fs.stat(path);
const result = await nativeRequest<{ exists?: boolean; size: number; mtime: string | number; isDirectory: boolean; isFile: boolean } | null>(target.instance, 'files.stat', { path: target.path });
const result = await nativeRequest<{
exists?: boolean;
size: number;
mtime: string | number;
isDirectory: boolean;
isFile: boolean;
} | null>(target.instance, 'files.stat', { path: target.path });
if (result === null || result.exists === false) return missing(path);
return { size: result.size, mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime), isDirectory: () => result.isDirectory, isFile: () => result.isFile };
return {
size: result.size,
mtime: new Date(typeof result.mtime === 'number' ? result.mtime * 1000 : result.mtime),
isDirectory: () => result.isDirectory,
isFile: () => result.isFile
};
}
export async function readdir(path: string, options: { withFileTypes: true }): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
export async function readdir(
path: string,
options: { withFileTypes: true }
): Promise<{ name: string; isDirectory(): boolean; isFile(): boolean }[]> {
const target = remote(path);
if (!target) return fs.readdir(path, options);
const result = await nativeRequest<{ entries: { name: string; isDirectory: boolean; isFile?: boolean }[] }>(target.instance, 'files.list', { path: target.path });
return result.entries.map(entry => ({ name: entry.name, isDirectory: () => entry.isDirectory, isFile: () => entry.isFile ?? !entry.isDirectory }));
const result = await nativeRequest<{
entries: { name: string; isDirectory: boolean; isFile?: boolean }[];
}>(target.instance, 'files.list', { path: target.path });
return result.entries.map((entry) => ({
name: entry.name,
isDirectory: () => entry.isDirectory,
isFile: () => entry.isFile ?? !entry.isDirectory
}));
}
export async function nativeTail(path: string, lines: number) {
const target = remote(path);
if (!target) return undefined;
return nativeRequest<import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read', { path: target.path, lines });
return nativeRequest<import('../fleet/logs.js').TailResult | null>(
target.instance,
'logs.read',
{ path: target.path, lines }
);
}
export async function nativeLogChunk(path: string, cursor?: import('../fleet/logs.js').LogCursor) {
const target = remote(path);
if (!target) return undefined;
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes('logs.read-offset');
return nativeRequest<import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null>(target.instance, 'logs.read',
{ path: target.path, ...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 }) });
const incremental = getHost(target.instance.host_id)?.inventory.capabilities?.includes(
'logs.read-offset'
);
return nativeRequest<
import('../fleet/logs.js').LogChunk | import('../fleet/logs.js').TailResult | null
>(target.instance, 'logs.read', {
path: target.path,
...(incremental ? { mode: 'incremental', cursor } : { lines: 2000 })
});
}
src/lib/server/backends/native.ts
+105 -28
View File
@@ -6,13 +6,26 @@ import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
export interface NativeInstance {
id: string; server: string; host_id: string; generation: number; library_id: string;
image: string; bind_address: string; port: number; public_address: string | null;
public_port: number | null; created_at: string; retired_at: string | null;
id: string;
server: string;
host_id: string;
generation: number;
library_id: string;
image: string;
bind_address: string;
port: number;
public_address: string | null;
public_port: number | null;
created_at: string;
retired_at: string | null;
}
export function nativeInstance(server: string): NativeInstance | null {
return db().prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL').get(server) as NativeInstance | undefined ?? null;
return (
(db()
.prepare('SELECT * FROM native_instances WHERE server = ? AND retired_at IS NULL')
.get(server) as NativeInstance | undefined) ?? null
);
}
/** Credentials come from the launch, never from the possibly modified config or
@@ -21,72 +34,136 @@ export function nativeInstance(server: string): NativeInstance | null {
export function runtimePasswordForServer(server: string): string | null {
const instance = nativeInstance(server);
if (!instance) return null;
const row = db().prepare(`SELECT
const row = db()
.prepare(
`SELECT
json_extract(COALESCE(p.payload,o.payload),'$.rconPassword') AS password,
json_extract(COALESCE(p.payload,o.payload),'$.env.PLUTAINER_RCON_PASSWORD') AS env_password
FROM ec_host_operations o JOIN native_instances i ON ${runtimeLaunchSql}
LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id
WHERE i.id=? AND o.state='completed' AND o.history_expired_at IS NULL
ORDER BY o.rowid DESC LIMIT 1`).get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row) throw new Error(`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`);
ORDER BY o.rowid DESC LIMIT 1`
)
.get(instance.id) as { password: unknown; env_password: unknown } | undefined;
if (!row)
throw new Error(
`Cannot verify runtime config for ${server}: the current instance's launch credentials are unavailable.`
);
const password = row.password || row.env_password;
return typeof password === 'string' && password ? password : null;
}
export function nativeDeployment(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
return !!db()
.prepare(
`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_intents WHERE deployment_id = ?
UNION ALL SELECT 1 FROM native_creation_cancellations c
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`).get(id, id, id);
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`
)
.get(id, id, id);
}
export function unplacedNativeCreation(server: string): string | null {
const row = db().prepare(`SELECT d.id FROM server_deployments d
const row = db()
.prepare(
`SELECT d.id FROM server_deployments d
JOIN native_creation_intents i ON i.deployment_id=d.id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state='queued'
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(server) as { id: string } | undefined;
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`
)
.get(server) as { id: string } | undefined;
return row?.id ?? null;
}
const observations = new Map<string, Promise<unknown>>();
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'files.manifest', 'logs.read']);
const readKinds = new Set([
'instance.inspect',
'files.read',
'files.stat',
'files.list',
'files.manifest',
'logs.read'
]);
/** Share overlapping observations only, never completed results or mutations. */
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
export async function nativeRequest<T = Record<string, unknown>>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown> = {},
key?: string
): Promise<T> {
const host = getHost(instance.host_id);
if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command)));
if (!host || host.connectionState !== 'online')
throw new Error('Host agent is offline; instance state is unknown.');
const readOnly =
readKinds.has(kind) ||
(kind === 'rcon.execute' &&
['status', 'getstatus', 'version'].includes(String(payload.command)));
if (key || !readOnly) return executeRequest<T>(instance, kind, payload, key);
// A probe after a lifecycle submission must not join an old-process read.
const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server,
latestGameSubmission(instance.server), kind, payload]);
const identity = JSON.stringify([
instance.host_id,
instance.id,
instance.generation,
instance.server,
latestGameSubmission(instance.server),
kind,
payload
]);
const pending = observations.get(identity);
if (pending) return pending as Promise<T>;
const request = executeRequest<T>(instance, kind, payload).finally(() => { observations.delete(identity); });
const request = executeRequest<T>(instance, kind, payload).finally(() => {
observations.delete(identity);
});
observations.set(identity, request);
return request;
}
/** Only the agent receives host paths, Docker access, or game credentials. */
async function executeRequest<T>(instance: NativeInstance, kind: string, payload: Record<string, unknown>, key?: string): Promise<T> {
async function executeRequest<T>(
instance: NativeInstance,
kind: string,
payload: Record<string, unknown>,
key?: string
): Promise<T> {
const operation = queueHostOperation(instance.host_id, kind, payload, {
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
instanceId: instance.id,
generation: instance.generation,
idempotencyKey: key ?? randomUUID()
});
const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000));
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
const result = await trackActivity(
`agent:${operation.id}`,
`Waiting for ${kind} on ${instance.server}`,
() => waitHostOperationStatus(operation.id, 25_000)
);
if (result.state !== 'completed')
throw new Error(
result.error ||
`Agent operation ${result.id} is ${result.state}; its original ID is retained.`
);
// Do not apply replies from a retired generation to a replacement server.
const current = nativeInstance(instance.server);
if (current?.id !== instance.id || current.generation !== instance.generation) throw new Error('Instance was replaced during the operation.');
if (current?.id !== instance.id || current.generation !== instance.generation)
throw new Error('Instance was replaced during the operation.');
return result.result as T;
}
export async function nativeHealth(server: string): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
export async function nativeHealth(
server: string
): Promise<{ ok: true; state: string } | { ok: false; reason: string; state?: string }> {
const instance = nativeInstance(server);
if (!instance) return { ok: false, reason: 'Native instance is not registered' };
try {
const result = await nativeRequest<{ state: string; healthy: boolean }>(instance, 'instance.inspect');
return result.healthy ? { ok: true, state: result.state } : { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) { return { ok: false, state: 'unknown', reason: (error as Error).message }; }
const result = await nativeRequest<{ state: string; healthy: boolean }>(
instance,
'instance.inspect'
);
return result.healthy
? { ok: true, state: result.state }
: { ok: false, state: result.state, reason: `Container ${result.state}` };
} catch (error) {
return { ok: false, state: 'unknown', reason: (error as Error).message };
}
}
src/lib/server/backends/schema.ts
+36 -12
View File
@@ -43,11 +43,21 @@ export function migrateNativeInstances(handle: Database.Database): void {
);
CREATE INDEX IF NOT EXISTS idx_ec_host_migrations_instance ON ec_host_migrations(instance_id);
`);
if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) {
if (
!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(
(column) => column.name === 'runtime_password'
)
) {
handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT');
}
if (!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(column => column.name === 'recovery_step')) {
handle.exec("ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))");
if (
!(handle.pragma('table_info(native_provisioning)') as { name: string }[]).some(
(column) => column.name === 'recovery_step'
)
) {
handle.exec(
"ALTER TABLE native_provisioning ADD COLUMN recovery_step TEXT CHECK (recovery_step IN ('restart','rebuild'))"
);
}
if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) {
handle.transaction(() => {
@@ -56,29 +66,43 @@ export function migrateNativeInstances(handle: Database.Database): void {
INSERT INTO settings(key,value) VALUES('native_file_plans_v1','1');`);
})();
}
}
/** Accepted native creations receive their host port only when provisioning places them. */
function migrateUnplacedCreationPort(handle: Database.Database): void {
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { name: string; notnull: number }[];
if (!columns.find(column => column.name === 'port')?.notnull) return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'").get() as { sql: string };
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as {
name: string;
notnull: number;
}[];
if (!columns.find((column) => column.name === 'port')?.notnull) return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'")
.get() as { sql: string };
const replacement = sql.replace(/\bport\s+INTEGER\s+NOT\s+NULL\b/i, 'port INTEGER');
if (replacement === sql) throw new Error('Could not migrate unplaced creation ports.');
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const foreignKeys = handle.pragma('foreign_keys', { simple: true });
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced'));
const names = columns.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced')
);
const names = columns
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_creation_jobs_unplaced(rowid,${names}) SELECT rowid,${names} FROM server_creation_jobs;
DROP TABLE server_creation_jobs;
ALTER TABLE server_creation_jobs_unplaced RENAME TO server_creation_jobs;`);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Unplaced creation migration found broken references.');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Unplaced creation migration found broken references.');
})();
} finally { handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); }
} finally {
handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`);
}
}
src/lib/server/config.ts
+17 -9
View File
@@ -17,13 +17,15 @@ function num(name: string, fallback: number): number {
export const config = {
serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'),
t6ServersRoot: env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6ServersRoot:
env.T6_SERVERS_ROOT ?? join(dirname(req('SERVERS_ROOT', '/root/zombies/t5')), 't6'),
t6RelDestScript: env.T6_REL_DEST_SCRIPT ?? 'runtime/plutonium/storage/t6/scripts/zm',
t6ConfigTargetFile: env.T6_CONFIG_TARGET_FILE ?? 'dedicated_zm.cfg',
relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'),
// Per-kind destination sub-paths under each server's directory. `relDest`
// above stays as the script fallback so existing deploys keep working.
relDestScript: env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestScript:
env.REL_DEST_SCRIPT ?? env.REL_DEST ?? 'runtime/plutonium/storage/t5/raw/scripts/sp',
relDestConfig: env.REL_DEST_CONFIG ?? 'configs',
relDestPlugin: env.REL_DEST_PLUGIN ?? 'runtime/plutonium/plugins',
// Legacy T5 config filename. T6 has its own target above.
@@ -54,19 +56,25 @@ export type ArtifactKind = 'script' | 'config' | 'plugin' | 'baseline';
export function relDestForKind(kind: ArtifactKind, game = 't5'): string {
switch (kind) {
case 'script': return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config': return config.relDestConfig;
case 'baseline': return config.relDestConfig;
case 'plugin': return config.relDestPlugin;
case 'script':
return game === 't6' ? config.t6RelDestScript : config.relDestScript;
case 'config':
return config.relDestConfig;
case 'baseline':
return config.relDestConfig;
case 'plugin':
return config.relDestPlugin;
}
}
// Server IDs keep the game prefix even while creation is queued and no registry
// row exists yet. Legacy IDs retain the original T5 filesystem defaults.
export const gameOfServer = (server: string) => server.startsWith('t6_') ? 't6' : 't5';
export const rootOfServer = (server: string) => gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const gameOfServer = (server: string) => (server.startsWith('t6_') ? 't6' : 't5');
export const rootOfServer = (server: string) =>
gameOfServer(server) === 't6' ? config.t6ServersRoot : config.serversRoot;
export const serverDir = (server: string) => `${rootOfServer(server)}/${server}`;
export const configFileForGame = (game: string | null) => game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileForGame = (game: string | null) =>
game === 't6' ? config.t6ConfigTargetFile : config.configTargetFile;
export const configFileOf = (server: string) => configFileForGame(gameOfServer(server));
export const destOfKind = (server: string, kind: ArtifactKind) =>
`${serverDir(server)}/${relDestForKind(kind, gameOfServer(server))}`;
src/lib/server/db/connection.ts
+4 -2
View File
@@ -26,8 +26,10 @@ export function migrationStep<T>(name: string, work: () => T): T {
console.info(`[database] migration starting: ${name}`);
try {
const result = work();
console.info(`[database] migration finished: ${name}`, { elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
console.info(`[database] migration finished: ${name}`, {
elapsedMs: Date.now() - started,
rssMiB: Math.round(process.memoryUsage().rss / 1048576)
});
return result;
} catch (error) {
console.error(`[database] migration failed: ${name}`, error);
src/lib/server/db/file-blobs.ts
+93 -31
View File
@@ -10,67 +10,103 @@ export function ensureFileBlobSchema(handle: Database.Database): void {
CREATE INDEX IF NOT EXISTS idx_file_blob_references_hash ON file_blob_references(sha256);`);
}
export function putFileBlob(content: Uint8Array | null, handle: Database.Database = db()): string | null {
export function putFileBlob(
content: Uint8Array | null,
handle: Database.Database = db()
): string | null {
if (content === null) return null;
const sha256 = createHash('sha256').update(content).digest('hex');
handle.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)').run(sha256, content.byteLength, content);
handle
.prepare('INSERT OR IGNORE INTO file_blobs(sha256,size,content) VALUES(?,?,?)')
.run(sha256, content.byteLength, content);
return sha256;
}
export function getFileBlob(sha256: string, handle: Database.Database = db()): Buffer {
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as { content: Buffer } | undefined;
const row = handle.prepare('SELECT content FROM file_blobs WHERE sha256=?').get(sha256) as
{ content: Buffer } | undefined;
if (!row) throw new Error('File content is unavailable or has expired.');
return row.content;
}
/** SQL fragments are for fixed schema identifiers, never user input. */
export const blobContent = (column: string) => `(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobContent = (column: string) =>
`(SELECT content FROM file_blobs WHERE sha256=${column})`;
export const blobSize = (column: string) => `(SELECT size FROM file_blobs WHERE sha256=${column})`;
export function referenceBlobs(ownerType: string, ownerId: string, hashes: Iterable<string>, handle: Database.Database = db()): void {
handle.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?').run(ownerType, ownerId);
const insert = handle.prepare('INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)');
export function referenceBlobs(
ownerType: string,
ownerId: string,
hashes: Iterable<string>,
handle: Database.Database = db()
): void {
handle
.prepare('DELETE FROM file_blob_references WHERE owner_type=? AND owner_id=?')
.run(ownerType, ownerId);
const insert = handle.prepare(
'INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256) VALUES(?,?,?)'
);
for (const hash of hashes) insert.run(ownerType, ownerId, hash);
}
/** Immutable file bytes in JSON are replaced with hash references, including before images. */
export function packFileJson(value: unknown, handle: Database.Database = db(), hashes = new Set<string>()): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value)) return { value: value.map(item => packFileJson(item, handle, hashes).value), hashes };
export function packFileJson(
value: unknown,
handle: Database.Database = db(),
hashes = new Set<string>()
): { value: unknown; hashes: Set<string> } {
if (Array.isArray(value))
return { value: value.map((item) => packFileJson(item, handle, hashes).value), hashes };
if (!value || typeof value !== 'object') return { value, hashes };
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentBase64' || key === 'content' && 'path' in value && 'sha256' in value) && typeof item === 'string') {
if (
(key === 'contentBase64' ||
(key === 'content' && 'path' in value && 'sha256' in value)) &&
typeof item === 'string'
) {
const hash = putFileBlob(Buffer.from(item, 'base64'), handle)!;
result[key === 'content' ? 'contentHash' : 'contentSha256'] = hash;
hashes.add(hash);
} else {
result[key] = packFileJson(item, handle, hashes).value;
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') hashes.add(item);
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string')
hashes.add(item);
}
}
return { value: result, hashes };
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Unpacked JSON retains the caller-specific stored object shape.
export function unpackFileJson(value: unknown, handle: Database.Database = db()): any {
if (Array.isArray(value)) return value.map(item => unpackFileJson(item, handle));
if (Array.isArray(value)) return value.map((item) => unpackFileJson(item, handle));
if (!value || typeof value !== 'object') return value;
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
if ((key === 'contentHash' || key === 'contentSha256') && typeof item === 'string') {
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(item, handle).toString('base64');
result[key === 'contentHash' ? 'content' : 'contentBase64'] = getFileBlob(
item,
handle
).toString('base64');
} else result[key] = unpackFileJson(item, handle);
}
return result;
}
export const blobColumns = [
['script_versions', 'content'], ['staged_file_changes', 'content'],
['deployment_files', 'content_before'], ['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'], ['beta_revision_files', 'content']
['script_versions', 'content'],
['staged_file_changes', 'content'],
['deployment_files', 'content_before'],
['deployment_files', 'content_after'],
['beta_file_snapshots', 'prior_content'],
['beta_revision_files', 'content']
] as const;
/** Finish each SELECT before writing; better-sqlite3 iterators hold an active cursor. */
export function* migrationRows<T>(handle: Database.Database, sql: string): Generator<T & { rid: number }> {
export function* migrationRows<T>(
handle: Database.Database,
sql: string
): Generator<T & { rid: number }> {
const select = handle.prepare(sql);
let cursor = -Number.MAX_SAFE_INTEGER;
for (;;) {
@@ -84,31 +120,53 @@ export function* migrationRows<T>(handle: Database.Database, sql: string): Gener
/** Backfill one row at a time so large libraries do not exhaust startup memory. */
export function migrateFileBlobs(handle: Database.Database): void {
for (const [table, column] of blobColumns) {
if (!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(item => item.name === column)) continue;
migrationStep(`shared files: ${table}.${column}`, () => handle.transaction(() => {
handle.exec(`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`);
if (
!(handle.pragma(`table_info(${table})`) as { name: string }[]).some(
(item) => item.name === column
)
)
continue;
migrationStep(`shared files: ${table}.${column}`, () =>
handle.transaction(() => {
handle.exec(
`ALTER TABLE ${table} ADD COLUMN ${column}_hash TEXT REFERENCES file_blobs(sha256)`
);
const update = handle.prepare(`UPDATE ${table} SET ${column}_hash=? WHERE rowid=?`);
for (const row of migrationRows<{ content: Buffer | null }>(handle, `SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`)) {
for (const row of migrationRows<{ content: Buffer | null }>(
handle,
`SELECT rowid AS rid,${column} AS content FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT 1`
)) {
update.run(putFileBlob(row.content, handle), row.rid);
}
handle.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`);
})());
})()
);
}
for (const [table, column] of blobColumns) handle.exec(`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`);
for (const [table, column] of blobColumns)
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_${table}_${column}_hash ON ${table}(${column}_hash)`
);
if (!handle.prepare("SELECT 1 FROM settings WHERE key='recipe_blobs_v1'").get()) {
migrationStep('shared files: launch recipes', () => handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1')) {
migrationStep('shared files: launch recipes', () =>
handle.transaction(() => {
for (const row of migrationRows<{ deployment_id: string; recipe: string }>(
handle,
'SELECT rowid AS rid,deployment_id,recipe FROM server_launch_recipes WHERE rowid>? ORDER BY rowid LIMIT 1'
)) {
const packed = packFileJson(JSON.parse(row.recipe), handle);
handle.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(packed.value), row.deployment_id);
handle
.prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?')
.run(JSON.stringify(packed.value), row.deployment_id);
referenceBlobs('recipe', row.deployment_id, packed.hashes, handle);
}
handle.exec("INSERT INTO settings(key,value) VALUES('recipe_blobs_v1','1')");
})());
})()
);
}
handle.exec(`CREATE TRIGGER IF NOT EXISTS release_recipe_blobs AFTER DELETE ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=OLD.deployment_id; END;`);
for (const action of ['INSERT', 'UPDATE OF recipe']) handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
for (const action of ['INSERT', 'UPDATE OF recipe'])
handle.exec(`CREATE TRIGGER IF NOT EXISTS reference_recipe_blobs_${action.startsWith('UPDATE') ? 'update' : 'insert'}
AFTER ${action} ON server_launch_recipes BEGIN
DELETE FROM file_blob_references WHERE owner_type='recipe' AND owner_id=NEW.deployment_id;
INSERT OR IGNORE INTO file_blob_references(owner_type,owner_id,sha256)
@@ -117,8 +175,12 @@ export function migrateFileBlobs(handle: Database.Database): void {
}
export function pruneFileBlobs(limit = 100): number {
return db().prepare(`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
return db()
.prepare(
`DELETE FROM file_blobs WHERE sha256 IN (SELECT b.sha256 FROM file_blobs b
WHERE NOT EXISTS (SELECT 1 FROM file_blob_references r WHERE r.sha256=b.sha256)
${blobColumns.map(([table, column]) => `AND NOT EXISTS (SELECT 1 FROM ${table} WHERE ${column}_hash=b.sha256)`).join('\n')}
LIMIT ?)`).run(limit).changes;
LIMIT ?)`
)
.run(limit).changes;
}
src/lib/server/db/fleet-settings.ts
+15 -5
View File
@@ -1,26 +1,36 @@
import type Database from 'better-sqlite3';
import { defaultFleetSettings, validateFleetSettings, type FleetSettings } from '../../fleet-settings.js';
import {
defaultFleetSettings,
validateFleetSettings,
type FleetSettings
} from '../../fleet-settings.js';
export const FLEET_SETTINGS_KEY = 'fleet_settings';
/** Import legacy environment values once; persisted settings win on every later boot. */
export function initializeFleetSettings(handle: Database.Database): void {
const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
const saved = handle
.prepare('SELECT value FROM settings WHERE key = ?')
.get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
if (saved) {
const settings = JSON.parse(saved.value);
const upgraded = validateFleetSettings({ ...defaultFleetSettings, ...settings });
handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
handle
.prepare('UPDATE settings SET value = ? WHERE key = ?')
.run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
return;
}
const values = { ...defaultFleetSettings };
const legacy: Partial<Record<keyof FleetSettings, string>> = {
deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
deployKeepDays: 'DEPLOY_KEEP_DAYS',
restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY'
};
for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) {
const raw = process.env[legacy[key]!];
if (raw) values[key] = Number(raw);
}
handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
handle
.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')
.run(FLEET_SETTINGS_KEY, JSON.stringify(validateFleetSettings(values)));
}
src/lib/server/db/index.ts
+273 -79
View File
@@ -28,9 +28,15 @@ let optimizeTimer: NodeJS.Timeout | undefined;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_checked_at', ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT' },
{
name: 'container_checked_at',
ddl: 'ALTER TABLE servers ADD COLUMN container_checked_at TEXT'
},
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
{
name: 'desired_state',
ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))"
},
// Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present.
{ name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' },
@@ -43,7 +49,10 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'last_roster', ddl: 'ALTER TABLE servers ADD COLUMN last_roster TEXT' },
{ name: 'last_seen_at', ddl: 'ALTER TABLE servers ADD COLUMN last_seen_at TEXT' },
{ name: 'last_error', ddl: 'ALTER TABLE servers ADD COLUMN last_error TEXT' },
{ name: 'rcon_failures', ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0' },
{
name: 'rcon_failures',
ddl: 'ALTER TABLE servers ADD COLUMN rcon_failures INTEGER NOT NULL DEFAULT 0'
},
{ name: 'last_reply_snippet', ddl: 'ALTER TABLE servers ADD COLUMN last_reply_snippet TEXT' },
// Script-scope columns — static per server, populated via server-edit UI.
{ name: 'game', ddl: 'ALTER TABLE servers ADD COLUMN game TEXT' },
@@ -78,7 +87,10 @@ function migrateServers(handle: Database.Database): void {
const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']);
handle.transaction(() => {
for (const column of rows) {
if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`);
if (retired.has(column.name))
handle.exec(
`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`
);
}
})();
}
@@ -98,8 +110,14 @@ const SCRIPT_VERSIONS_COLUMNS: Array<{ name: string; ddl: string }> = [
// the authoritative "promoted" signal), and any version already gated
// through a ready-promote beta will be re-marked next time that state
// machine fires.
{ name: 'state', ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'" },
{ name: 'kind', ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'state',
ddl: "ALTER TABLE script_versions ADD COLUMN state TEXT NOT NULL DEFAULT 'untested'"
},
{
name: 'kind',
ddl: "ALTER TABLE script_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
@@ -107,18 +125,40 @@ const BETA_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
];
const DEPLOYMENT_SCRIPTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'kind', ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'" }
{
name: 'kind',
ddl: "ALTER TABLE deployment_scripts ADD COLUMN kind TEXT NOT NULL DEFAULT 'script'"
}
];
const DEPLOYMENTS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'script_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)' },
{ name: 'script_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'prior_version_id', ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)' },
{ name: 'beta_id', ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)' },
{ name: 'rolled_back_by', ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)' }
{
name: 'script_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_id INTEGER REFERENCES scripts(id)'
},
{
name: 'script_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN script_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'prior_version_id',
ddl: 'ALTER TABLE deployments ADD COLUMN prior_version_id INTEGER REFERENCES script_versions(id)'
},
{
name: 'beta_id',
ddl: 'ALTER TABLE deployments ADD COLUMN beta_id INTEGER REFERENCES betas(id)'
},
{
name: 'rolled_back_by',
ddl: 'ALTER TABLE deployments ADD COLUMN rolled_back_by TEXT REFERENCES deployments(id)'
}
];
function migrateTable(handle: Database.Database, table: string, cols: Array<{ name: string; ddl: string }>): void {
function migrateTable(
handle: Database.Database,
table: string,
cols: Array<{ name: string; ddl: string }>
): void {
const rows = handle.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
const existing = new Set(rows.map((r) => r.name));
for (const col of cols) {
@@ -169,14 +209,16 @@ function backfillMultiScriptJoins(handle: Database.Database): void {
// operator can promote/rollback in the old app before upgrading.
function dropLegacyBetaTables(handle: Database.Database): void {
const legacy = handle
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')")
.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('betas','beta_scripts','beta_deploys')"
)
.all() as Array<{ name: string }>;
if (legacy.length === 0) return;
const legacyNames = legacy.map((r) => r.name);
// The old `betas` table had a `slug` column; the new one has `id` + `script_id`.
// If we see `slug`, we know it's the old shape.
const betasCols = handle.prepare("PRAGMA table_info(betas)").all() as Array<{ name: string }>;
const betasCols = handle.prepare('PRAGMA table_info(betas)').all() as Array<{ name: string }>;
const isLegacyShape = betasCols.some((c) => c.name === 'slug');
if (!isLegacyShape) return;
@@ -205,8 +247,7 @@ function dropLegacyBetaTables(handle: Database.Database): void {
// active betas aren't blocked.
function assertSnapshotSchemeSafe(handle: Database.Database): void {
const done = handle.prepare("SELECT value FROM settings WHERE key = 'snapshots_v1'").get() as
| { value: string }
| undefined;
{ value: string } | undefined;
if (done) return;
const active = handle
.prepare("SELECT id FROM betas WHERE state IN ('running','ready-promote') LIMIT 1")
@@ -227,19 +268,36 @@ function migrateGameTelemetry(handle: Database.Database): void {
handle.transaction(() => {
const convert = (raw: string): string => {
const plan = JSON.parse(raw) as BetaPlan;
plan.requirements = plan.requirements.map(r => {
plan.requirements = plan.requirements.map((r) => {
const { event: _event, ...rest } = r as BetaRequirement & { event?: string };
return ['feature', 'restart-budget', 'clean-logs'].includes(r.kind)
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } : rest;
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) }
: rest;
});
return JSON.stringify(plan);
};
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ beta_id: number; plan: string }>;
for (const row of plans) handle.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?').run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ name: string; plan: string }>;
for (const row of presets) handle.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?').run(convert(row.plan), row.name);
handle.prepare(`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`).run(JSON.stringify(emptyMetrics()));
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{
beta_id: number;
plan: string;
}>;
for (const row of plans)
handle
.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?')
.run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{
name: string;
plan: string;
}>;
for (const row of presets)
handle
.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?')
.run(convert(row.plan), row.name);
handle
.prepare(
`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`
)
.run(JSON.stringify(emptyMetrics()));
handle.exec(`UPDATE beta_test_plans SET outcome = 'active', reason = 'Gameplay telemetry updated; collecting fresh script events'
WHERE outcome IN ('active','passed') AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'));
UPDATE script_versions SET state = 'untested' WHERE state = 'tested'
@@ -254,54 +312,96 @@ function migrateGameTelemetry(handle: Database.Database): void {
}
function migrateProvisioningActions(handle: Database.Database): void {
const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string };
const table = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'"
)
.get() as { sql: string };
if (table.sql.includes("'rebuild'")) return;
// Rebuild the CHECK constraint without cascading into events, recipes, or
// collection members. Preserve rowids: they order same-server operations.
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
.replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[])
.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
handle.exec(
table.sql
.replace('server_deployments', 'server_deployments_new')
.replace(
"'create', 'destroy'",
"'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"
)
);
const columns = (
handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[]
)
.map((column) => `"${column.name.replaceAll('"', '""')}"`)
.join(', ');
handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns})
SELECT rowid, ${columns} FROM server_deployments;
DROP TABLE server_deployments;
ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Provisioning migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
/** Rebuild constrained tables while preserving links, indexes, triggers, and row order. */
function migrateWatchdogConstraints(handle: Database.Database): void {
for (const name of ['provisioning_sources', 'deployment_servers']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = name === 'provisioning_sources'
? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'"))
: sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement =
name === 'provisioning_sources'
? sql.includes("'autoscaling'")
? sql
: sql.replace(
/'campaign','beta','maintenance'(?:,'watchdog')?/,
"'campaign','beta','maintenance','watchdog','autoscaling'"
)
: sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (sql === replacement) continue;
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependent = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace(name, name + '_watchdog_new'));
handle.exec(`INSERT INTO ${name}_watchdog_new (rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
}) SELECT rowid, ${
(handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ')
} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`);
handle.exec(
`INSERT INTO ${name}_watchdog_new (rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(', ')}) SELECT rowid, ${(
handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]
)
.map((col) => col.name)
.join(
', '
)} FROM ${name}; DROP TABLE ${name}; ALTER TABLE ${name}_watchdog_new RENAME TO ${name};`
);
for (const object of dependent) handle.exec(object.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Watchdog migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Watchdog migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
function backfillDesiredState(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get())
return;
handle.transaction(() => {
// A failed stop may have reached the host agent. Preserve that explicit intent.
handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
@@ -331,35 +431,80 @@ export function db(): Database.Database {
migrateProvisioningActions(handle);
migrateWatchdogConstraints(handle);
migrateTable(handle, 'deployment_files', [
{ name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' }
{
name: 'snapshot_pending',
ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0'
}
]);
migrateTable(handle, 'restart_rollout', [
{
name: 'operation_id',
ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT'
}
]);
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
migrateTable(handle, 'server_deployments', [
{ name: 'requested_by', ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT' },
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
{ name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
{
name: 'requested_by',
ddl: 'ALTER TABLE server_deployments ADD COLUMN requested_by TEXT'
},
{
name: 'submitted_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT'
},
{
name: 'checked_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT'
},
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' },
{ name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' },
{ name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' },
{ name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' }
{
name: 'phase_started_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT'
},
{
name: 'observed_at',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT'
},
{
name: 'observation_json',
ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT'
}
]);
migrateTable(handle, 'server_deployment_events', [
{ name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' },
{ name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' },
{ name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' }
{
name: 'phase_key',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT'
},
{
name: 'details_json',
ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT'
}
]);
for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
migrateTable(handle, table, [
{
name: 'provisioning_id',
ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT`
}
]);
}
migrateTable(handle, 'maintenance_jobs', [
{ name: 'kind', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'" },
{ name: 'origin', ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'" }
{
name: 'kind',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN kind TEXT NOT NULL DEFAULT 'legacy'"
},
{
name: 'origin',
ddl: "ALTER TABLE maintenance_jobs ADD COLUMN origin TEXT NOT NULL DEFAULT 'manual'"
}
]);
migrateServers(handle);
// Old snippets may contain player addresses, including hex-encoded bytes.
// Discard them once; newly retained diagnostics are sanitized failure replies only.
if (!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()) {
if (
!handle.prepare("SELECT 1 FROM settings WHERE key='rcon_private_diagnostics_v1'").get()
) {
handle.transaction(() => {
handle.exec(`UPDATE servers SET last_reply_snippet=NULL;
UPDATE servers SET last_error='bad-reply:unparseable print reply'
@@ -374,25 +519,52 @@ export function db(): Database.Database {
migrateTable(handle, 'scripts', SCRIPTS_COLUMNS);
migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS);
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
migrateTable(handle, 'beta_test_runs', [
{
name: 'revision',
ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1'
}
]);
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
// Older databases gain rolled_back_by above, after schema.sql runs.
handle.exec('CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)');
handle.exec(
'CREATE INDEX IF NOT EXISTS idx_deployments_rolled_back_by ON deployments(rolled_back_by)'
);
migrateTable(handle, 'server_launch_collections', [
{ name: 'cancelled_at', ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT' },
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
{
name: 'cancelled_at',
ddl: 'ALTER TABLE server_launch_collections ADD COLUMN cancelled_at TEXT'
},
{
name: 'mode',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'retained_servers',
ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'"
}
]);
migrateTable(handle, 'server_launch_recipes', [
{ name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' }
{
name: 'recipe_summary',
ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT'
}
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
{
name: 'mode',
ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'"
},
{
name: 'fleet_snapshot',
ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT'
}
]);
migrateTable(handle, 'server_telemetry', [
{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }
]);
migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]);
// Kind-referencing indexes: safe here because migrateTable above just
// ALTER'd the column into existence if it was missing.
handle.exec(
@@ -403,14 +575,27 @@ export function db(): Database.Database {
backfillMultiScriptJoins(handle);
assertSnapshotSchemeSafe(handle);
migrateGameTelemetry(handle);
migrateTable(handle, 'leaderboard_matches', [{ name: 'start_players', ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER' }]);
migrateTable(handle, 'leaderboard_matches', [
{
name: 'start_players',
ddl: 'ALTER TABLE leaderboard_matches ADD COLUMN start_players INTEGER'
}
]);
migrateTable(handle, 'leaderboard_match_players', [
{ name: 'first_observed_round', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER' },
{ name: 'present_at_start', ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER' }
{
name: 'first_observed_round',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN first_observed_round INTEGER'
},
{
name: 'present_at_start',
ddl: 'ALTER TABLE leaderboard_match_players ADD COLUMN present_at_start INTEGER'
}
]);
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
migrationStep('native instance schema and file plans', () =>
migrateNativeInstances(handle)
);
migrationStep('per-game server key pools', () => migrateServerKeys(handle));
migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
migrationStep('rebalancing schema', () => ensureRebalancingSchema(handle));
@@ -436,12 +621,18 @@ export function db(): Database.Database {
ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`);
installPerformanceIndexes(handle);
installReportRevisions(handle);
migrationStep('retention clocks and indexes', () => retentionClocks(handle, {
deployments: ['started_at', 'finished_at'], server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'], watchdog_incidents: ['first_seen_at', 'finished_at'],
migrationStep('retention clocks and indexes', () =>
retentionClocks(handle, {
deployments: ['started_at', 'finished_at'],
server_deployments: ['started_at', 'finished_at'],
maintenance_jobs: ['requested_at', 'finished_at'],
watchdog_incidents: ['first_seen_at', 'finished_at'],
leaderboard_matches: ['started_at', 'last_received_at', 'ended_at'],
betas: ['opened_at', 'closed_at'], script_versions: ['uploaded_at'], retention_controls: ['resolved_at']
}));
betas: ['opened_at', 'closed_at'],
script_versions: ['uploaded_at'],
retention_controls: ['resolved_at']
})
);
// Existing databases require one rebuild to gain the incremental-vacuum page map.
enableIncrementalVacuum(handle);
handle.pragma('optimize = 0x10002');
@@ -472,8 +663,11 @@ export function db(): Database.Database {
_db = handle;
optimizeTimer = setInterval(() => {
try { handle.pragma('optimize'); }
catch (error) { console.error('[database] optimize failed:', error); }
try {
handle.pragma('optimize');
} catch (error) {
console.error('[database] optimize failed:', error);
}
}, 60 * 60_000);
optimizeTimer.unref();
return _db;
src/lib/server/db/retention-clocks.ts
+52 -14
View File
@@ -3,21 +3,46 @@ import { migrationStep, tuneMigrationConnection } from './connection.js';
/** Normalize age-policy clocks once, then compare indexed ISO UTC strings directly. */
export function retentionClocks(handle: Database.Database, clocks: Record<string, string[]>): void {
handle.exec('CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)');
handle.exec(
'CREATE TABLE IF NOT EXISTS database_maintenance_migrations(name TEXT PRIMARY KEY)'
);
for (const [table, columns] of Object.entries(clocks)) {
const marker = `iso-clocks:${table}:v1`;
if (!handle.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?').get(marker)) {
if (
!handle
.prepare('SELECT 1 FROM database_maintenance_migrations WHERE name=?')
.get(marker)
) {
handle.transaction(() => {
for (const column of columns) handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
for (const column of columns)
handle.exec(`UPDATE ${table} SET ${column}=strftime('%Y-%m-%dT%H:%M:%fZ',${column})
WHERE julianday(${column}) IS NOT NULL AND ${column}!=strftime('%Y-%m-%dT%H:%M:%fZ',${column});`);
handle.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)').run(marker);
handle
.prepare('INSERT INTO database_maintenance_migrations(name) VALUES(?)')
.run(marker);
})();
}
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as { name: string; partial: number }[];
const indexed = new Set(indexes.filter(index => !index.partial).map(index =>
(handle.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`).get() as { name: string }).name));
for (const column of columns) if (!indexed.has(column)) {
handle.exec(`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`);
const indexes = handle.prepare(`PRAGMA index_list(${table})`).all() as {
name: string;
partial: number;
}[];
const indexed = new Set(
indexes
.filter((index) => !index.partial)
.map(
(index) =>
(
handle
.prepare(`PRAGMA index_info("${index.name.replaceAll('"', '""')}")`)
.get() as { name: string }
).name
)
);
for (const column of columns)
if (!indexed.has(column)) {
handle.exec(
`CREATE INDEX IF NOT EXISTS idx_retention_${table}_${column} ON ${table}(${column})`
);
}
}
}
@@ -25,8 +50,12 @@ export function retentionClocks(handle: Database.Database, clocks: Record<string
/** VACUUM can renumber implicit rowids. Keep host enable cutoffs tied to command IDs across crashes. */
export function enableIncrementalVacuum(handle: Database.Database): void {
if (handle.pragma('auto_vacuum', { simple: true }) !== 2) {
handle.exec(`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`);
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) {
handle.exec(
`CREATE TABLE IF NOT EXISTS database_vacuum_boundaries(host_id TEXT PRIMARY KEY, operation_id TEXT)`
);
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()
) {
handle.exec(`INSERT OR IGNORE INTO database_vacuum_boundaries
SELECT e.host_id,(SELECT o.id FROM ec_host_operations o WHERE o.host_id=e.host_id
AND o.rowid<=e.operation_floor ORDER BY o.rowid DESC LIMIT 1) FROM ec_host_enable_requests e`);
@@ -42,7 +71,9 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
};
try {
tuneMigrationConnection(handle);
migrationStep('incremental vacuum page map (disk-backed rebuild)', () => handle.exec('VACUUM'));
migrationStep('incremental vacuum page map (disk-backed rebuild)', () =>
handle.exec('VACUUM')
);
} finally {
handle.pragma('shrink_memory');
handle.pragma(`temp_store = ${previous.temp}`);
@@ -51,9 +82,16 @@ export function enableIncrementalVacuum(handle: Database.Database): void {
handle.pragma(`journal_size_limit = ${previous.journal}`);
}
}
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()) {
if (
handle.prepare("SELECT 1 FROM sqlite_master WHERE name='database_vacuum_boundaries'").get()
) {
handle.transaction(() => {
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'").get()) handle.exec(`
if (
handle
.prepare("SELECT 1 FROM sqlite_master WHERE name='ec_host_enable_requests'")
.get()
)
handle.exec(`
UPDATE ec_host_enable_requests AS e SET operation_floor=COALESCE((
SELECT o.rowid FROM database_vacuum_boundaries b JOIN ec_host_operations o ON o.id=b.operation_id
WHERE b.host_id=e.host_id),0) WHERE e.host_id IN (SELECT host_id FROM database_vacuum_boundaries)`);
src/lib/server/db/server-history.ts
+43 -14
View File
@@ -1,17 +1,34 @@
import type Database from 'better-sqlite3';
const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs'];
const historyTables = [
'beta_servers',
'deployment_servers',
'server_deployments',
'leaderboard_matches',
'watchdog_incidents',
'maintenance_jobs'
];
const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */
function detachBetaHistory(handle: Database.Database): void {
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(name) as { sql: string };
const replacement = sql.replace(
/server\s+TEXT NOT NULL REFERENCES servers\(name\)/,
'server TEXT NOT NULL'
);
if (replacement === sql) continue;
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all(name) as { sql: string }[];
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ');
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[])
.map((col) => col.name)
.join(', ');
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
@@ -19,9 +36,12 @@ function detachBetaHistory(handle: Database.Database): void {
handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name};
DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Server history migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
}
@@ -40,12 +60,19 @@ export function migrateServerHistory(handle: Database.Database): void {
detachBetaHistory(handle);
handle.transaction(() => {
for (const table of historyTables) {
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as {
name: string;
}[];
if (!columns.some((column) => column.name === 'server_deleted_at'))
handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
}
const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[];
if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) {
handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'");
const collectionColumns = handle
.prepare('PRAGMA table_info(server_launch_collections)')
.all() as { name: string }[];
if (!collectionColumns.some((column) => column.name === 'retained_server_deletions')) {
handle.exec(
"ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'"
);
}
// Historical unfinished work cannot claim the next server with this name.
handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending;
@@ -96,7 +123,8 @@ export function migrateServerHistory(handle: Database.Database): void {
OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`;
for (const table of historyTables) {
const exception = table === 'server_deployments'
const exception =
table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server
AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning')))
@@ -104,7 +132,8 @@ export function migrateServerHistory(handle: Database.Database): void {
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : '';
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))`
: '';
handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`);
}
src/lib/server/db/server-keys.ts
+78 -26
View File
@@ -22,25 +22,41 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_server_key_leases_current ON server_key_le
/** Preserve every occupied legacy assignment, even duplicated credentials. */
export function migrateServerKeys(handle: Database.Database): void {
handle.transaction(() => {
const legacy = (handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[])
.some(column => column.name === 'server_id');
handle
.transaction(() => {
const legacy = (
handle.prepare('PRAGMA table_info(server_keys)').all() as { name: string }[]
).some((column) => column.name === 'server_id');
if (legacy) handle.exec('ALTER TABLE server_keys RENAME TO legacy_server_keys');
handle.exec(keySchema + leaseSchema);
if (!legacy) return;
const rows = handle.prepare(`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`).all() as
{ server_id: string; pluto_key: string; updated_at: string }[];
const rows = handle
.prepare(
`SELECT server_id, pluto_key, updated_at FROM legacy_server_keys ORDER BY server_id`
)
.all() as { server_id: string; pluto_key: string; updated_at: string }[];
const occupied = handle.prepare(`SELECT 1 WHERE
EXISTS (SELECT 1 FROM servers WHERE name=@server AND destroyed_at IS NULL)
OR EXISTS (SELECT 1 FROM native_instances WHERE server=@server AND retired_at IS NULL)
OR EXISTS (SELECT 1 FROM server_deployments WHERE server=@server AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state NOT IN ('done','warning','cancelled'))`);
const find = handle.prepare('SELECT id FROM server_keys WHERE game=? AND pluto_key=?');
const serverGame = handle.prepare("SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')");
const insert = handle.prepare('INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)');
const lease = handle.prepare('INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)');
const serverGame = handle.prepare(
"SELECT game FROM servers WHERE name=? AND game IN ('t5','t6')"
);
const insert = handle.prepare(
'INSERT INTO server_keys(id,game,label,pluto_key,updated_at) VALUES(?,?,?,?,?)'
);
const lease = handle.prepare(
'INSERT INTO server_key_leases(server_id,key_id,is_current) VALUES(?,?,?)'
);
const assignments = new Map<string, string[]>();
function save(game: string, credential: string, label: string, updatedAt: string): string {
function save(
game: string,
credential: string,
label: string,
updatedAt: string
): string {
const existing = (find.get(game, credential) as { id: string } | undefined)?.id;
if (existing) return existing;
const id = randomUUID();
@@ -48,9 +64,13 @@ export function migrateServerKeys(handle: Database.Database): void {
return id;
}
for (const row of rows) {
const game = /^(t5|t6)_/.exec(row.server_id)?.[1]
?? (serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for saved server key ${row.server_id}`);
const game =
/^(t5|t6)_/.exec(row.server_id)?.[1] ??
(serverGame.get(row.server_id) as { game: string } | undefined)?.game;
if (!game)
throw new Error(
`Cannot determine the game for saved server key ${row.server_id}`
);
const id = save(game, row.pluto_key, row.server_id, row.updated_at);
if (occupied.get({ server: row.server_id })) assignments.set(row.server_id, [id]);
}
@@ -58,29 +78,57 @@ export function migrateServerKeys(handle: Database.Database): void {
// container. Durable create/rebuild payloads are stronger evidence of
// its actual credential. Keep ambiguous delivered attempts as additional
// holds until the instance is destroyed; never free a possibly live key.
const hasTable = (name: string) => !!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name);
const hasTable = (name: string) =>
!!handle
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?")
.get(name);
if (hasTable('ec_host_operations')) {
const payload = hasTable('ec_host_operation_payloads')
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)' : 'o.payload';
? 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'
: 'o.payload';
const command = `CASE WHEN json_valid(${payload}) THEN ${payload} ELSE '{}' END`;
const launched = handle.prepare(`SELECT n.server,o.state,o.updated_at,
const launched = handle
.prepare(
`SELECT n.server,o.state,o.updated_at,
json_extract(${command},'$.env.PLUTO_SERVER_KEY') AS credential,
json_extract(${command},'$.game') AS game,
json_extract(${command},'$.env.PLUTAINER_GAME') AS game_id
FROM native_instances n JOIN ec_host_operations o ON o.instance_id=n.id AND o.host_id=n.host_id
WHERE n.retired_at IS NULL AND o.kind IN ('instance.create','instance.rebuild') AND o.state!='failed'
AND (o.generation=n.generation OR json_extract(${command},'$.nextGeneration')=n.generation)
ORDER BY o.rowid DESC`).all() as
{ server: string; state: string; updated_at: string; credential: unknown; game: unknown; game_id: unknown }[];
ORDER BY o.rowid DESC`
)
.all() as {
server: string;
state: string;
updated_at: string;
credential: unknown;
game: unknown;
game_id: unknown;
}[];
const recovered = new Map<string, string[]>();
const complete = new Set<string>();
for (const row of launched) {
if (complete.has(row.server) || typeof row.credential !== 'string' || !row.credential) continue;
const game = row.game === 't5' || row.game === 't6' ? row.game
: row.game_id === 't5sp' ? 't5' : row.game_id === 't6zm' ? 't6'
: /^(t5|t6)_/.exec(row.server)?.[1]
?? (serverGame.get(row.server) as { game: string } | undefined)?.game;
if (!game) throw new Error(`Cannot determine the game for launched server ${row.server}`);
if (
complete.has(row.server) ||
typeof row.credential !== 'string' ||
!row.credential
)
continue;
const game =
row.game === 't5' || row.game === 't6'
? row.game
: row.game_id === 't5sp'
? 't5'
: row.game_id === 't6zm'
? 't6'
: (/^(t5|t6)_/.exec(row.server)?.[1] ??
(serverGame.get(row.server) as { game: string } | undefined)
?.game);
if (!game)
throw new Error(
`Cannot determine the game for launched server ${row.server}`
);
const ids = recovered.get(row.server) ?? [];
const id = save(game, row.credential, row.server, row.updated_at);
if (!ids.includes(id)) ids.push(id);
@@ -90,11 +138,15 @@ export function migrateServerKeys(handle: Database.Database): void {
for (const [server, ids] of recovered) {
// If no completed launch receipt survives, its saved assignment
// remains a possible prior credential while the attempt resolves.
if (!complete.has(server)) for (const id of assignments.get(server) ?? []) if (!ids.includes(id)) ids.push(id);
if (!complete.has(server))
for (const id of assignments.get(server) ?? [])
if (!ids.includes(id)) ids.push(id);
assignments.set(server, ids);
}
}
for (const [server, ids] of assignments) ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
for (const [server, ids] of assignments)
ids.forEach((id, index) => lease.run(server, id, index === 0 ? 1 : 0));
handle.exec('DROP TABLE legacy_server_keys');
}).immediate();
})
.immediate();
}
src/lib/server/db/slow-statements.ts
+19 -7
View File
@@ -1,19 +1,31 @@
import type Database from 'better-sqlite3';
/** Log query shapes, never bound values or SQL string literals. Disabled by default. */
export function timeStatement(statement: Database.Statement, sql: string, thresholdMs: number): Database.Statement {
export function timeStatement(
statement: Database.Statement,
sql: string,
thresholdMs: number
): Database.Statement {
if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) return statement;
const shape = sql.replace(/'(?:''|[^'])*'/g, '?').replace(/\s+/g, ' ').trim().slice(0, 200);
const shape = sql
.replace(/'(?:''|[^'])*'/g, '?')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
for (const method of ['run', 'get', 'all'] as const) {
const original = statement[method].bind(statement) as (...args: unknown[]) => unknown;
Object.defineProperty(statement, method, { value: (...args: unknown[]) => {
Object.defineProperty(statement, method, {
value: (...args: unknown[]) => {
const started = performance.now();
try { return original(...args); }
finally {
try {
return original(...args);
} finally {
const ms = performance.now() - started;
if (ms >= thresholdMs) console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
if (ms >= thresholdMs)
console.warn('[sql]', { ms: Math.round(ms), method, sql: shape });
}
} });
}
});
}
return statement;
}
src/lib/server/db/version-layout.ts
+27 -9
View File
@@ -3,25 +3,43 @@ import type Database from 'better-sqlite3';
/** Keep metadata ahead of overflow pages, preserving IDs and all references. */
export function migrateVersionLayout(handle: Database.Database): void {
const columns = handle.pragma('table_info(script_versions)') as { name: string }[];
if (!columns.some(column => column.name === 'content') || columns.at(-1)?.name === 'content') return;
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'").get() as { sql: string };
if (!columns.some((column) => column.name === 'content') || columns.at(-1)?.name === 'content')
return;
const { sql } = handle
.prepare("SELECT sql FROM sqlite_master WHERE name='script_versions'")
.get() as { sql: string };
const content = /\bcontent\s+BLOB\s+NOT\s+NULL\s*,/i;
if (!content.test(sql)) throw new Error('Unrecognized script version content layout');
const replacement = sql.replace(content, '').replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
const replacement = sql
.replace(content, '')
.replace(/,\s*UNIQUE\s*\(/i, ', content BLOB NOT NULL, UNIQUE(')
.replace('script_versions', 'script_versions_metadata');
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL")
const dependents = handle
.prepare(
"SELECT sql FROM sqlite_master WHERE tbl_name='script_versions' AND type IN ('index','trigger') AND sql IS NOT NULL"
)
.all() as { sql: string }[];
const sequence = handle.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'").get() as { seq: number } | undefined;
const sequence = handle
.prepare("SELECT seq FROM sqlite_sequence WHERE name='script_versions'")
.get() as { seq: number } | undefined;
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement);
const names = columns.map(column => `"${column.name}"`).join(',');
const names = columns.map((column) => `"${column.name}"`).join(',');
handle.exec(`INSERT INTO script_versions_metadata (${names}) SELECT ${names} FROM script_versions;
DROP TABLE script_versions; ALTER TABLE script_versions_metadata RENAME TO script_versions;`);
if (sequence) handle.prepare("UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'").run(sequence.seq);
if (sequence)
handle
.prepare(
"UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name='script_versions'"
)
.run(sequence.seq);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Version layout migration found broken references');
if ((handle.pragma('foreign_key_check') as unknown[]).length)
throw new Error('Version layout migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
} finally {
handle.pragma('foreign_keys = ON');
}
}
src/lib/server/fleet/beta-evaluation.ts
+81 -26
View File
@@ -19,14 +19,30 @@ export interface TestMetrics {
outages: number;
streaks: Record<string, number>;
}
export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} });
export const emptyMetrics = (): TestMetrics => ({
values: {},
manual: {},
outages: 0,
streaks: {}
});
/** Gameplay comes only from the script. No reporter IDs or beta handshake. */
export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void {
export function applyGameEvent(
plan: BetaPlan,
metrics: TestMetrics,
event: GameEvent,
now: number
): void {
const players = event.players.length;
if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) {
metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined,
lastEvent: now, players, ended: false, rounds: {} };
metrics.match = {
map: event.map,
started: event.event === 'game_start' ? now : undefined,
lastEvent: now,
players,
ended: false,
rounds: {}
};
metrics.streaks = {};
}
const match = metrics.match;
@@ -34,10 +50,12 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
const minutes = Math.max(0, now - match.lastEvent) / 60_000;
for (const r of plan.requirements) {
const current = metrics.values[r.id] ?? 0;
const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
const populated =
players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players);
if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes;
if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60;
if (r.kind === 'player-hours' && populated)
metrics.values[r.id] = current + (minutes * Math.min(players, match.players)) / 60;
if (r.kind === 'sustained-population') {
metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0;
metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]);
@@ -46,10 +64,14 @@ export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: Game
match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round);
}
if (event.event === 'game_end' && r.kind === 'matches'
&& event.round >= MIN_RECORDED_ROUND
&& (match.rounds[r.id] ?? -1) >= r.minRound
&& (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) {
if (
event.event === 'game_end' &&
r.kind === 'matches' &&
event.round >= MIN_RECORDED_ROUND &&
(match.rounds[r.id] ?? -1) >= r.minRound &&
(r.minMinutes === 0 ||
(match.started !== undefined && now - match.started >= r.minMinutes * 60_000))
) {
metrics.values[r.id] = current + 1;
}
}
@@ -66,30 +88,63 @@ export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy:
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
if (r.kind === 'healthy-minutes')
m.values[r.id] =
healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
}
m.lastSample = at; m.wasHealthy = healthy;
m.lastSample = at;
m.wasHealthy = healthy;
}
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
return plan.requirements.map(r => {
export function evaluateRequirements(
plan: BetaPlan,
metrics: TestMetrics,
phase: number
): RequirementProgress[] {
return plan.requirements.map((r) => {
const budget = r.kind === 'failure-budget';
const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id])
: r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target,
const current = budget
? metrics.outages
: r.kind === 'manual'
? Number(!!metrics.manual[r.id])
: r.kind === 'recovery'
? phase
: (metrics.values[r.id] ?? 0);
return {
id: r.id,
label: r.label,
current,
target: r.target,
passed: budget ? current <= r.target : current >= r.target,
failed: budget && current > r.target,
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
: budget ? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}` };
detail:
r.kind === 'manual'
? (metrics.manual[r.id] ?? 'Awaiting tester sign-off')
: budget
? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}`
};
});
}
export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean {
export function coveragePassed(
plan: BetaPlan,
servers: Array<{ passed: boolean; game: string | null; map: string | null }>
): boolean {
if (!servers.length) return false;
if (plan.scope === 'all') return servers.every(s => s.passed);
if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum;
const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map
: s.game && s.map ? `${s.game}/${s.map}` : null;
if (plan.scope === 'all') return servers.every((s) => s.passed);
if (plan.scope === 'quorum') return servers.filter((s) => s.passed).length >= plan.quorum;
const key = (s: (typeof servers)[number]) =>
plan.scope === 'game'
? s.game
: plan.scope === 'map'
? s.map
: s.game && s.map
? `${s.game}/${s.map}`
: null;
const groups = new Set(servers.map(key));
return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group));
return (
!groups.has(null) &&
[...groups].every((group) => servers.some((s) => s.passed && key(s) === group))
);
}
src/lib/server/fleet/beta-pages.ts
+39 -12
View File
@@ -8,13 +8,21 @@ const positive = (value: string | null, fallback: number) => {
};
export function betaListPage(params: URLSearchParams) {
const state = params.get('state'), view = params.get('view'), search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [], args: (string | number)[] = [];
if (state) { predicates.push('b.state=?'); args.push(state); }
else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
const state = params.get('state'),
view = params.get('view'),
search = (params.get('q') ?? '').slice(0, 200).trim();
const predicates: string[] = [],
args: (string | number)[] = [];
if (state) {
predicates.push('b.state=?');
args.push(state);
} else if (view === 'active') predicates.push("b.state IN ('running','ready-promote')");
else if (view === 'history') predicates.push("b.state IN ('promoted','rolled-back')");
const status = params.get('status');
if (status && ['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)) {
if (
status &&
['ready', 'testing', 'attention', 'paused', 'promoted', 'rolled-back'].includes(status)
) {
// Match betaStatus: closed lifecycle wins over the final test outcome.
predicates.push(`(CASE WHEN b.state IN ('promoted','rolled-back') THEN b.state
WHEN EXISTS (SELECT 1 FROM beta_servers m WHERE m.beta_id=b.id AND (m.server_deleted_at IS NOT NULL OR m.stage='failed'))
@@ -24,8 +32,15 @@ export function betaListPage(params: URLSearchParams) {
args.push(status);
}
if (search) {
const names = (db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[])
.filter(row => `${row.server} ${serverTitle(row.server)}`.toLowerCase().includes(search.toLowerCase())).map(row => row.server);
const names = (
db().prepare('SELECT DISTINCT server FROM beta_servers').all() as { server: string }[]
)
.filter((row) =>
`${row.server} ${serverTitle(row.server)}`
.toLowerCase()
.includes(search.toLowerCase())
)
.map((row) => row.server);
predicates.push(`(instr(lower('#'||b.id||' '||COALESCE(b.notes,'')),lower(?))>0 OR EXISTS (
SELECT 1 FROM beta_scripts bs JOIN scripts s ON s.id=bs.script_id WHERE bs.beta_id=b.id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
OR EXISTS (SELECT 1 FROM scripts s WHERE s.id=b.script_id AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.target_rel_path),lower(?))>0))
@@ -33,21 +48,33 @@ export function betaListPage(params: URLSearchParams) {
args.push(search, search, search, search, search, JSON.stringify(names));
}
const where = predicates.length ? ' WHERE ' + predicates.join(' AND ') : '';
const total = (db().prepare('SELECT COUNT(*) n FROM betas b' + where).get(...args) as { n: number }).n;
const total = (
db()
.prepare('SELECT COUNT(*) n FROM betas b' + where)
.get(...args) as { n: number }
).n;
const pageSize = Math.min(100, positive(params.get('limit'), 25));
const totalPages = Math.max(1, Math.ceil(total / pageSize));
const page = Math.min(totalPages, positive(params.get('page'), 1));
const betas = db().prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
const betas = db()
.prepare('SELECT b.* FROM betas b' + where + ' ORDER BY b.id DESC LIMIT ? OFFSET ?')
.all(...args, pageSize, (page - 1) * pageSize) as BetaRow[];
const counts = db().prepare(`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`).get() as { active: number; history: number };
const counts = db()
.prepare(
`SELECT COUNT(CASE WHEN state IN ('running','ready-promote') THEN 1 END) active,
COUNT(CASE WHEN state IN ('promoted','rolled-back') THEN 1 END) history FROM betas`
)
.get() as { active: number; history: number };
return { betas, page, pageSize, total, totalPages, counts };
}
export function betaActivityPage(id: number, params = new URLSearchParams()) {
const before = positive(params.get('before'), Number.MAX_SAFE_INTEGER);
const limit = Math.min(100, positive(params.get('limit'), 50));
const rows = db().prepare('SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?')
const rows = db()
.prepare(
'SELECT id,created_at,message FROM beta_events WHERE beta_id=? AND id<? ORDER BY id DESC LIMIT ?'
)
.all(id, before, limit + 1) as { id: number; created_at: string; message: string }[];
const more = rows.length > limit;
const events = rows.slice(0, limit).reverse();
src/lib/server/fleet/beta-revisions.ts
+51 -13
View File
@@ -2,29 +2,67 @@ import { db } from '../db/index.js';
import type { BetaRevision } from '../../beta-revision.js';
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
{ revision: number; started_at: string } | undefined
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
return (
(db()
.prepare(
`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`
)
.get(id) as { revision: number; started_at: string } | undefined) ?? {
revision: 1,
started_at: (
db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as {
opened_at: string;
}
).opened_at
}
);
}
/** Called in the revision transaction, before replacing the pinned bundle. */
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
export function archiveBetaRevision(
id: number,
revision: number,
startedAt: string,
notes: string | null
): void {
db()
.prepare(
'INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)'
)
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
db()
.prepare(
`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content_hash)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content_hash
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
WHERE bs.beta_id = ?`
)
.run(revision, id);
}
export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
const files = db().prepare(`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`).all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({ ...row,
const rows = db()
.prepare(
'SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC'
)
.all(id) as Array<{
revision: number;
started_at: string;
notes: string | null;
results_json: string | null;
}>;
const files = db()
.prepare(
`SELECT revision, script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? ORDER BY script_id`
)
.all(id) as Array<BetaRevision['files'][number] & { revision: number }>;
return rows.map(({ results_json, ...row }) => ({
...row,
results: results_json ? JSON.parse(results_json) : null,
files: files.filter(file => file.revision === row.revision).map(({ revision, ...file }) => file)
files: files
.filter((file) => file.revision === row.revision)
.map(({ revision, ...file }) => file)
}));
}
src/lib/server/fleet/beta-selection.ts
+98 -35
View File
@@ -7,7 +7,7 @@ import { observationReader } from './status.js';
import type { ScriptRow } from './scripts.js';
const games = ['t5', 't6'] as const;
type Game = typeof games[number];
type Game = (typeof games)[number];
type Counts = Partial<Record<Game, number>>;
export function parseBetaInstanceCounts(value: unknown): Counts {
@@ -16,7 +16,12 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
const counts: Counts = {};
for (const [game, count] of Object.entries(value)) {
if (!games.includes(game as Game) || !Number.isSafeInteger(count) || count < 1 || count > 1000) {
if (
!games.includes(game as Game) ||
!Number.isSafeInteger(count) ||
count < 1 ||
count > 1000
) {
throw new Error('Instance counts must be integers from 1 to 1000 for t5 or t6');
}
counts[game as Game] = count;
@@ -25,79 +30,137 @@ export function parseBetaInstanceCounts(value: unknown): Counts {
}
function matches(script: ScriptRow, server: ServerRow): boolean {
return !script.scope_type || !script.scope_value
|| (script.scope_type === 'game' ? server.game : server.map) === script.scope_value;
return (
!script.scope_type ||
!script.scope_value ||
(script.scope_type === 'game' ? server.game : server.map) === script.scope_value
);
}
/** Choose at submission time using the same fresh observations shown in Fleet. */
export function selectBetaServers(scripts: ScriptRow[], instancesPerGame: unknown): string[] {
const counts = parseBetaInstanceCounts(instancesPerGame), observe = observationReader(), now = Date.now();
const hosts = new Map(listHosts().map(host => [host.id, host]));
const busy = new Set((db().prepare(`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
const counts = parseBetaInstanceCounts(instancesPerGame),
observe = observationReader(),
now = Date.now();
const hosts = new Map(listHosts().map((host) => [host.id, host]));
const busy = new Set(
(
db()
.prepare(
`SELECT server FROM server_deployments WHERE server_deleted_at IS NULL
AND (state IN ('queued','waiting','running','awaiting-health','interrupted')
OR (action IN ('destroy','rebuild') AND state = 'failed'
AND stage IN ('Stopping and removing container','Removing server files and registry')))`)
.all() as { server: string }[]).map(row => row.server));
const candidates = listServers().flatMap(server => {
if (!counts[server.game as Game] || server.desired_state === 'stopped' || busy.has(server.name)
|| !scripts.some(script => matches(script, server)) || protectedWork(server.name)) return [];
AND stage IN ('Stopping and removing container','Removing server files and registry')))`
)
.all() as { server: string }[]
).map((row) => row.server)
);
const candidates = listServers()
.flatMap((server) => {
if (
!counts[server.game as Game] ||
server.desired_state === 'stopped' ||
busy.has(server.name) ||
!scripts.some((script) => matches(script, server)) ||
protectedWork(server.name)
)
return [];
const host = server.host_id ? hosts.get(server.host_id) : null;
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission)) return [];
if (server.host_id && (!host?.connected || host.mode !== 'active' || host.decommission))
return [];
const observation = observe(server);
if (observation.health !== 'online' || observation.last_players === null
|| !freshStatusTimestamp(observation.last_seen_at, now)) return [];
if (
observation.health !== 'online' ||
observation.last_players === null ||
!freshStatusTimestamp(observation.last_seen_at, now)
)
return [];
// An occupied RCON observation must not rank as empty just because a
// competing telemetry report claims zero players.
const rconPlayers = freshStatusTimestamp(server.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? server.last_players ?? 0 : 0;
const rconPlayers =
freshStatusTimestamp(server.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(server.last_seen_at!) > Date.parse(observation.telemetry_after))
? (server.last_players ?? 0)
: 0;
return [{ server, players: Math.max(observation.last_players, rconPlayers) }];
}).sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
})
.sort((a, b) => a.players - b.players || a.server.name.localeCompare(b.server.name));
for (const game of games) {
const available = candidates.filter(candidate => candidate.server.game === game).length;
const available = candidates.filter((candidate) => candidate.server.game === game).length;
if ((counts[game] ?? 0) > available) {
throw new Error(`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`);
throw new Error(
`${game.toUpperCase()} needs ${counts[game]} available instances, but only ${available} have fresh status and no pending work. Reduce the count or retry when more servers are available.`
);
}
}
for (const script of scripts) {
if (!candidates.some(candidate => matches(script, candidate.server))) {
throw new Error(`${script.name}: no available server in the requested games matches this file's scope.`);
if (!candidates.some((candidate) => matches(script, candidate.server))) {
throw new Error(
`${script.name}: no available server in the requested games matches this file's scope.`
);
}
}
// Map scopes need a representative even when the least populated servers
// all run the same map. Reserve enough remaining slots for every map.
const requiredMaps = new Set(scripts.filter(script => script.scope_type === 'map').map(script => script.scope_value!));
const requiredMaps = new Set(
scripts.filter((script) => script.scope_type === 'map').map((script) => script.scope_value!)
);
function canComplete(remaining: typeof candidates, slots: Counts, maps: Set<string>): boolean {
const options = new Map<string, Set<Game>>();
for (const game of games) {
if (remaining.filter(candidate => candidate.server.game === game).length < (slots[game] ?? 0)) return false;
if (
remaining.filter((candidate) => candidate.server.game === game).length <
(slots[game] ?? 0)
)
return false;
}
for (const map of maps) {
const choices = new Set(remaining.filter(candidate => candidate.server.map === map && (slots[candidate.server.game as Game] ?? 0) > 0)
.map(candidate => candidate.server.game as Game));
const choices = new Set(
remaining
.filter(
(candidate) =>
candidate.server.map === map &&
(slots[candidate.server.game as Game] ?? 0) > 0
)
.map((candidate) => candidate.server.game as Game)
);
if (!choices.size) return false;
options.set(map, choices);
}
// There are two supported games: single-game maps reserve that game's
// slots, and maps available in either game use any remaining slot.
return games.every(game => [...options.values()].filter(choices => choices.size === 1 && choices.has(game)).length <= (slots[game] ?? 0))
&& maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0);
return (
games.every(
(game) =>
[...options.values()].filter(
(choices) => choices.size === 1 && choices.has(game)
).length <= (slots[game] ?? 0)
) && maps.size <= games.reduce((total, game) => total + (slots[game] ?? 0), 0)
);
}
if (!canComplete(candidates, counts, requiredMaps)) {
throw new Error('Increase the instance counts to cover every selected file\'s map scope, or choose servers in advanced options.');
throw new Error(
"Increase the instance counts to cover every selected file's map scope, or choose servers in advanced options."
);
}
const selected: string[] = [], slots = { ...counts };
const selected: string[] = [],
slots = { ...counts };
let maps = requiredMaps;
for (let index = 0; index < candidates.length; index++) {
const { server } = candidates[index], game = server.game as Game;
const { server } = candidates[index],
game = server.game as Game;
if (!slots[game]) continue;
const nextSlots = { ...slots, [game]: slots[game]! - 1 }, nextMaps = new Set(maps);
const nextSlots = { ...slots, [game]: slots[game]! - 1 },
nextMaps = new Set(maps);
if (server.map) nextMaps.delete(server.map);
if (!canComplete(candidates.slice(index + 1), nextSlots, nextMaps)) continue;
selected.push(server.name); slots[game] = nextSlots[game]; maps = nextMaps;
if (games.every(game => !slots[game])) break;
selected.push(server.name);
slots[game] = nextSlots[game];
maps = nextMaps;
if (games.every((game) => !slots[game])) break;
}
return selected;
}

Some files were not shown because too many files have changed in this diff Show More