From a50eae9d21bd35433268411cbc614341ed4373b4 Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 17 Sep 2026 21:35:28 -0400 Subject: [PATCH] Add configurable beta test plans and staged rollout --- README.md | 5 + docs/beta-test-plans.md | 153 ++++++++++++ src/lib/beta-plan.ts | 90 +++++++ src/lib/operation-flow.ts | 3 +- src/lib/server/db/schema.sql | 48 ++++ src/lib/server/fleet/beta-evaluation.ts | 118 +++++++++ src/lib/server/fleet/beta-tests.ts | 182 ++++++++++++++ src/lib/server/fleet/beta.ts | 86 +++++-- src/lib/server/fleet/scripts.ts | 4 +- src/lib/server/fleet/servers.ts | 4 +- src/lib/server/pipeline/beta-plan-runner.ts | 167 +++++++++++++ src/lib/server/pipeline/beta-runner.ts | 5 +- src/lib/server/pipeline/bus.ts | 3 + src/lib/ui/components/BetaPlanBuilder.svelte | 119 +++++++++ src/lib/ui/components/BetaTestProgress.svelte | 82 ++++++ src/routes/api/betas/+server.ts | 9 + src/routes/api/betas/[id]/+server.ts | 3 +- src/routes/api/betas/[id]/retry/+server.ts | 2 + src/routes/api/betas/[id]/rollback/+server.ts | 10 + src/routes/api/betas/[id]/test/+server.ts | 55 ++++ src/routes/api/betas/presets/+server.ts | 18 ++ src/routes/api/telemetry/+server.ts | 25 +- src/routes/betas/+page.svelte | 23 +- src/routes/betas/[id]/+page.svelte | 19 +- tests/unit/beta-evaluation.test.ts | 95 +++++++ tests/unit/beta-plans.test.ts | 234 ++++++++++++++++++ 26 files changed, 1518 insertions(+), 44 deletions(-) create mode 100644 docs/beta-test-plans.md create mode 100644 src/lib/beta-plan.ts create mode 100644 src/lib/server/fleet/beta-evaluation.ts create mode 100644 src/lib/server/fleet/beta-tests.ts create mode 100644 src/lib/server/pipeline/beta-plan-runner.ts create mode 100644 src/lib/ui/components/BetaPlanBuilder.svelte create mode 100644 src/lib/ui/components/BetaTestProgress.svelte create mode 100644 src/routes/api/betas/[id]/test/+server.ts create mode 100644 src/routes/api/betas/presets/+server.ts create mode 100644 tests/unit/beta-evaluation.test.ts create mode 100644 tests/unit/beta-plans.test.ts diff --git a/README.md b/README.md index 7ff7e95..e9d8675 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,11 @@ without a Docker health check use their running state as the readiness signal. ## How restarts stay safe +Betas support configurable gameplay, participation, stability and manual-review +requirements, saved presets, coverage rules and staged rollout. See +[Beta test plans](docs/beta-test-plans.md) for usage and the session telemetry +contract used by round and custom-feature checks. + File changes are staged sequentially. Their activation uses rolling restarts: by default only **one server across the fleet** can restart or recover at a time. Promotions, rollbacks, drift reconciliation, and beta restarts share this diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md new file mode 100644 index 0000000..13df8db --- /dev/null +++ b/docs/beta-test-plans.md @@ -0,0 +1,153 @@ +# Beta test plans + +New betas use a saved plan rather than the two legacy wait flags. Existing +betas and API callers without `testPlan` retain their original workflow. +Schema additions are additive and run at startup; no active beta needs to be +closed to install this feature. + +## Requirements + +Use **Add requirement** in Betas. Requirements on a qualifying server are ANDed: + +| Requirement | Evidence | +| --- | --- | +| Reach round X | Session telemetry reports round >= X with the required player count | +| Complete qualifying matches | Normal completed matches, optionally qualified by round, players and duration | +| Populated minutes | Fresh RCON samples at both ends of a populated interval | +| Player-hours | Populated minutes × the lower sampled player count / 60 | +| Reach player count | Fresh healthy RCON sample | +| Sustained population | Consecutive fresh healthy samples above the player threshold | +| Continuous healthy minutes | Consecutive fresh container/RCON observations | +| Maximum health outages | Transitions from observed healthy to unhealthy | +| Maximum unexpected restarts | Distinct reported process boots within the same test run | +| Maximum script errors | New console lines matching script errors/exceptions, unhandled exceptions or fatal errors | +| Successful recovery cycles | Repeat the other requirements after each additional empty-server restart | +| Feature event count | Named, session-scoped game events | +| Tester checklist | Tester name and observation, entered on the beta detail page | + +There is one round requirement: **reach round X**. For repeated high-round runs, +use a qualifying-match count with a minimum round. The minimum player count must +be present at the qualifying round observation in the same match. A match with +many players and a different solo high-round match cannot combine to pass. + +Unqualified match counts use incremental game logs (`InitGame`, a player join, +then `ExitLevel: executed`). Round-qualified or multi-player-qualified matches +use session telemetry. Each requirement uses one source, preventing duplicate +counts. Match duration is measured from the observed join/session start. Log +polling can undercount duration; it does not infer unobserved elapsed time. + +Round, feature and unexpected-restart checks require the reporting protocol +below. Legacy round reports still update Fleet but cannot prove beta progress. +The game scripts themselves are uploaded artifacts, not source maintained in +this repository; update their reporter to emit the protocol before selecting +these requirements in production. + +## Coverage, rollout and outcomes + +Coverage can require all selected servers, a quorum, one per selected map, +one per selected game, or one per selected game/map combination. Coverage +metadata is pinned when the beta opens. Register game/map metadata first. + +Set wave size to 1 for a canary, a larger number for batches, or 0 for all +selected servers. Later waves reserve their artifact targets but receive no +files until every server in the prior waves has passed. All waves must start +testing before the final coverage rule can pass. Error-budget violations on +any tested server block promotion, including with quorum coverage. + +The test window starts when the beta opens, including time waiting for empty +servers. Missing evidence at the deadline produces **inconclusive**, not a +failure or an automatic rollback. Health and log monitoring continue after a +pass until promotion; a deteriorating result revokes readiness. + +For failures, select hold as failed, pause for investigation, or automatic +rollback. Rollback restores only files touched by the beta. It can queue rolling +restarts when servers empty, or leave restarts manual. Manual rollback uses the +same selected restart policy. No beta auto-promotes. + +Pause holds evidence evaluation and rollout. Resume retains evidence and the +original deadline. A fresh attempt retires old evidence and restarts deployed +servers when empty; it does not extend the deadline. Expired or incomplete-file +deployments should be rolled back and reopened. Recovery cycles similarly +require fresh milestones/checklists, while carrying error budgets forward. + +Time accounting excludes intervals spanning monitoring gaps longer than 60 +seconds. A missing first sample earns no time. A log that cannot be read cannot +pass a clean-log requirement. Log backlog above 2 MB makes the plan inconclusive +instead of silently dropping evidence. Partial lines, rotations, byte cursors, +attempts and accepted events persist in SQLite. The detail page shows the most +recent 200 evidence entries; older entries remain in the database. + +## Game reporter protocol + +On **game-process startup**, fetch: + +```http +GET /api/telemetry?server=t5_zombie_theater_1 +``` + +The response contains `runs: [{ runId, betaId }]`. Cache that list for the process +lifetime, generate a unique `bootId`, and send each event once to every returned +run. A run ID is an evidence correlation token, not an authentication credential. +Do not refresh it during a match or relabel queued events with a newer run ID. +Each beta-controlled restart creates a new run ID before requesting the restart. +An unexpected process restart fetches the current run ID and uses a new boot ID. + +POST JSON to `/api/telemetry`: + +```json +{ + "event": "round_change", + "server": "t5_zombie_theater_1", + "runId": "the-run-id-from-startup", + "eventId": "boot-uuid:session-uuid:12", + "bootId": "boot-uuid", + "sessionId": "session-uuid", + "sequence": 12, + "round": 25, + "players": 4 +} +``` + +Send events in order: + +1. `boot`: once per process, before all session events. `sessionId` can be + omitted. Send `sequence`, `round` and `players` as 0. +2. `session_start`: a new unique session ID, sequence 0, current players/round. + Send when actual gameplay begins, not when an idle map loads. +3. `round_change`: strictly increasing sequence within that session, reported + round and player count. Emit on entry to each round. +4. `feature`: same sequence ordering, current players/round, plus `name` such + as `perk.purchased`. Report one event per occurrence. +5. `match_end`: normal completion only, next sequence, final players/round. + Crashes, forced shutdowns and idle map rotations must not emit this event. + +IDs accept letters, numbers, `_`, `.`, `:`, and `-` (up to 128 characters). +Feature names are limited to 120 characters. Numbers must be nonnegative safe +integers. Retries reuse the same event ID and payload. Send sequentially: an +older sequence arriving late is rejected, not retroactively credited. + +Responses include `test: { accepted, reason? }`. Duplicates return HTTP 200 with +`accepted: false`; malformed events return 400. Events for retired runs, unknown +boots/sessions, already-ended sessions, or paused/closed tests are not credited. +The existing unauthenticated/trusted-network deployment model is unchanged. + +Legacy payloads without `runId` continue to support `round_change` for Fleet +display. They do not alter RCON restart safety data or test evidence. + +## API and storage + +`POST /api/betas` accepts `testPlan`, validated by `parseBetaPlan` in +`src/lib/beta-plan.ts`. New plans always verify restart recovery before testing. +`GET /api/betas/:id` includes the plan, outcome, per-server requirement progress +and recent evidence under `test`. + +`POST /api/betas/:id/test` accepts actions `pause`, `resume`, `retry`, or +`sign-off` (with `server`, `requirementId`, and `note`). +`GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are copied +into a beta; later preset edits never change an active test. + +`beta_test_plans`, `beta_test_coverage`, `beta_test_runs`, `beta_test_evidence`, +and `beta_test_presets` hold configuration and results. `beta_file_changes` +records which files were touched, including partially completed deployments. +Legacy beta lifecycle states remain compatible with artifact holds and history; +test outcomes are stored separately. Holds last until promotion or rollback. diff --git a/src/lib/beta-plan.ts b/src/lib/beta-plan.ts new file mode 100644 index 0000000..f7e9fd7 --- /dev/null +++ b/src/lib/beta-plan.ts @@ -0,0 +1,90 @@ +export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population', + 'sustained-population', 'healthy-minutes', 'failure-budget', 'restart-budget', 'clean-logs', 'recovery', 'feature', 'manual'] as const; +export type RequirementKind = typeof requirementKinds[number]; +export interface BetaRequirement { + id: string; + kind: RequirementKind; + target: number; + label: string; + minRound: number; + minPlayers: number; + minMinutes: number; + event: string; +} +export interface BetaPlan { + version: 1; + requirements: BetaRequirement[]; + scope: 'all' | 'quorum' | 'map' | 'game' | 'map-game'; + quorum: number; + timeoutHours: number; + failureAction: 'fail' | 'pause' | 'rollback'; + rollbackRestart: 'manual' | 'when-empty'; + waveSize: number; // 0 deploys all selected servers; otherwise pass each wave before expanding. +} +export const requirementLabels: Record = { + round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes', + 'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)', + 'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages', + 'restart-budget': 'Maximum unexpected restarts', + 'clean-logs': 'Maximum script errors', recovery: 'Successful recovery cycles', feature: 'Feature event count', manual: 'Tester checklist' +}; +export function requirement(kind: RequirementKind, id: string): BetaRequirement { + return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : ['failure-budget', 'restart-budget', 'clean-logs'].includes(kind) ? 0 : 1, + minRound: 0, minPlayers: 1, minMinutes: 0, event: '' }; +} +export function defaultPlan(): BetaPlan { + return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1, + timeoutHours: 24, failureAction: 'fail', rollbackRestart: 'when-empty', waveSize: 0 }; +} +export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [ + { name: 'Quick smoke test', plan: defaultPlan() }, + { name: 'Gameplay validation', plan: { ...defaultPlan(), requirements: [ + { ...requirement('matches', 'matches'), target: 2, minRound: 25 }, + { ...requirement('populated-minutes', 'playtime'), target: 90 }, requirement('failure-budget', 'outages') + ] } }, + { name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [ + { ...requirement('healthy-minutes', 'health'), target: 360 }, + { ...requirement('player-hours', 'participation'), target: 10 }, requirement('clean-logs', 'logs') + ] } } +]; + +/** Strict allow-list validation at every boundary (including stored presets). */ +export function parseBetaPlan(value: unknown): BetaPlan { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Test plan must be an object'); + const p = value as BetaPlan; + if (p.version !== 1 || !['all', 'quorum', 'map', 'game', 'map-game'].includes(p.scope)) throw new Error('Invalid test plan version or scope'); + if (!['fail', 'pause', 'rollback'].includes(p.failureAction) || !['manual', 'when-empty'].includes(p.rollbackRestart)) throw new Error('Invalid failure policy'); + for (const [key, min, max] of [['quorum', 1, 1000], ['timeoutHours', 0.01, 8760], ['waveSize', 0, 1000]] as const) { + if (typeof p[key] !== 'number' || !Number.isFinite(p[key]) || p[key] < min || p[key] > max) throw new Error(`Invalid ${key}`); + } + if (!Number.isInteger(p.quorum) || !Number.isInteger(p.waveSize)) throw new Error('Quorum and wave size must be integers'); + if (!Array.isArray(p.requirements) || !p.requirements.length || p.requirements.length > 30) throw new Error('Choose 1–30 requirements'); + const ids = new Set(); + const requirements = p.requirements.map(r => { + if (!r || typeof r !== 'object' || !requirementKinds.includes(r.kind)) throw new Error('Unknown requirement'); + if (typeof r.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(r.id) || r.id in Object.prototype || ids.has(r.id)) throw new Error('Requirement IDs must be unique, non-reserved identifiers'); + ids.add(r.id); + if (typeof r.label !== 'string' || !r.label.trim() || r.label.length > 200) throw new Error('Requirement needs a label (up to 200 characters)'); + for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) { + if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`); + } + if (!['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind) && r.target <= 0) throw new Error('Requirement target must be positive'); + if (['round', 'matches', 'population', 'failure-budget', 'restart-budget', 'clean-logs', 'recovery', 'feature', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers'); + if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers'); + if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item'); + if (typeof r.event !== 'string' || r.event.length > 120 || (r.kind === 'feature' && !/^[a-zA-Z0-9_.:-]+$/.test(r.event))) throw new Error('Feature requirements need an event name'); + return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound, + minPlayers: r.minPlayers, minMinutes: r.minMinutes, event: r.event }; + }); + return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours, + failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize }; +} + +export interface RequirementProgress { id: string; label: string; current: number; target: number; passed: boolean; failed: boolean; detail: string } +export interface BetaTestView { + plan: BetaPlan; + outcome: string; + reason: string | null; + servers: Array<{ server: string; runId: string; phase: number; passed: boolean; progress: RequirementProgress[] }>; + evidence: Array<{ id: number; server: string; received_at: string; kind: string; payload: string }>; +} diff --git a/src/lib/operation-flow.ts b/src/lib/operation-flow.ts index b832a12..87468af 100644 --- a/src/lib/operation-flow.ts +++ b/src/lib/operation-flow.ts @@ -6,11 +6,12 @@ interface BetaServer { healthy_at: string | null; match_seen_at: string | null; } -export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number }, servers: BetaServer[]): OperationStep[] { +export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] { const stages = ['deploying', 'awaiting-empty', 'restarting']; const labels = ['Deploy artifacts', 'Wait for empty servers', 'Restart servers']; if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); } if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); } + if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); } stages.push('ready'); labels.push('Review and promote'); const positions = servers.map(s => { if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage)); diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index e657553..5da0608 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -129,6 +129,54 @@ CREATE TABLE IF NOT EXISTS beta_servers ( ); CREATE INDEX IF NOT EXISTS idx_beta_servers_stage ON beta_servers(stage); +-- Immutable test configuration, durable attempts and append-only evidence. +-- Keeping test outcome separate preserves artifact holds until explicit closure. +CREATE TABLE IF NOT EXISTS beta_test_plans ( + beta_id INTEGER PRIMARY KEY REFERENCES betas(id) ON DELETE CASCADE, + plan TEXT NOT NULL, + outcome TEXT NOT NULL DEFAULT 'active', + reason TEXT +); +CREATE TABLE IF NOT EXISTS beta_test_runs ( + id TEXT PRIMARY KEY, + beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, + server TEXT NOT NULL REFERENCES servers(name), + phase INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + observing_at TEXT, + retired_at TEXT, + metrics TEXT NOT NULL DEFAULT '{}', + cursors TEXT NOT NULL DEFAULT '{}' +); +CREATE TABLE IF NOT EXISTS beta_test_coverage ( + beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, + server TEXT NOT NULL, + game TEXT, + map TEXT, + PRIMARY KEY (beta_id, server) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_beta_test_current_run ON beta_test_runs(beta_id, server) WHERE retired_at IS NULL; +CREATE TABLE IF NOT EXISTS beta_test_evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL REFERENCES beta_test_runs(id) ON DELETE CASCADE, + event_id TEXT NOT NULL, + received_at TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + UNIQUE(run_id, event_id) +); +CREATE INDEX IF NOT EXISTS idx_beta_evidence_run ON beta_test_evidence(run_id, id); +CREATE TABLE IF NOT EXISTS beta_file_changes ( + beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, + server TEXT NOT NULL, + rel_path TEXT NOT NULL, + PRIMARY KEY (beta_id, server, rel_path) +); +CREATE TABLE IF NOT EXISTS beta_test_presets ( + name TEXT PRIMARY KEY, + plan TEXT NOT NULL +); + -- Many-to-many: which scripts (and pinned versions) are bundled into a beta. -- Replaces the deprecated betas.script_id / betas.script_version_id columns. CREATE TABLE IF NOT EXISTS beta_scripts ( diff --git a/src/lib/server/fleet/beta-evaluation.ts b/src/lib/server/fleet/beta-evaluation.ts new file mode 100644 index 0000000..ab94827 --- /dev/null +++ b/src/lib/server/fleet/beta-evaluation.ts @@ -0,0 +1,118 @@ +import type { BetaPlan, RequirementProgress } from '../../beta-plan.js'; + +export interface SessionEvidence { + started: number; + sequence: number; + ended: boolean; + rounds: Record; + peakPlayers: number; +} +export interface TestMetrics { + values: Record; + sessions: Record; + manual: Record; + bootId?: string; + boots: string[]; + restarts: number; + lastSample?: number; + wasHealthy?: boolean; + lastPlayers?: number; + outages: number; + logErrors: number; + logAvailable?: boolean; + streaks: Record; +} +export const emptyMetrics = (): TestMetrics => ({ values: {}, sessions: {}, manual: {}, boots: [], restarts: 0, outages: 0, logErrors: 0, streaks: {} }); + +export interface TestSignal { + event: 'boot' | 'session_start' | 'round_change' | 'match_end' | 'feature'; + bootId: string; + sessionId: string; + sequence: number; + round: number; + players: number; + name: string; +} + +/** Events are accepted once, in session order; a match must start in this run. */ +export function applySignal(plan: BetaPlan, metrics: TestMetrics, signal: TestSignal, now: number): boolean { + if (signal.event === 'boot') { + if (metrics.boots.includes(signal.bootId)) return false; + if (metrics.bootId) metrics.restarts++; + metrics.boots.push(signal.bootId); + metrics.bootId = signal.bootId; + return true; + } + if (metrics.bootId !== signal.bootId) return false; + const key = `${signal.bootId}:${signal.sessionId}`; + if (signal.event === 'session_start') { + if (metrics.sessions[key]) return false; + metrics.sessions[key] = { started: now, sequence: signal.sequence, ended: false, rounds: {}, peakPlayers: signal.players }; + return true; + } + const session = metrics.sessions[key]; + if (!session || session.ended || signal.sequence <= session.sequence) return false; + session.sequence = signal.sequence; + session.peakPlayers = Math.max(session.peakPlayers, signal.players); + for (const r of plan.requirements) { + const current = metrics.values[r.id] ?? 0; + if (signal.event === 'round_change' && signal.players >= r.minPlayers) { + session.rounds[r.id] = Math.max(session.rounds[r.id] ?? 0, signal.round); + if (r.kind === 'round') metrics.values[r.id] = Math.max(current, signal.round); + } + if (signal.event === 'match_end' && r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1) + && (session.rounds[r.id] ?? 0) >= r.minRound && session.peakPlayers >= r.minPlayers + && now - session.started >= r.minMinutes * 60_000) metrics.values[r.id] = current + 1; + if (signal.event === 'feature' && r.kind === 'feature' && signal.name === r.event + && signal.players >= r.minPlayers && signal.round >= r.minRound) metrics.values[r.id] = current + 1; + } + if (signal.event === 'match_end') session.ended = true; + return true; +} + +/** Sample both ends of an interval. Monitoring gaps never earn playtime. */ +export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean, players: number): void { + if (m.lastSample !== undefined && at <= m.lastSample) return; + const gap = m.lastSample === undefined ? Infinity : at - m.lastSample; + const contiguous = gap <= 60_000; + const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0; + if (!healthy && m.wasHealthy === true) m.outages++; + for (const r of plan.requirements) { + const value = m.values[r.id] ?? 0; + const populated = players >= Math.max(1, r.minPlayers) && (m.lastPlayers ?? 0) >= Math.max(1, r.minPlayers); + if (r.kind === 'populated-minutes' && populated) m.values[r.id] = value + minutes; + if (r.kind === 'player-hours' && populated) m.values[r.id] = value + minutes * Math.min(players, m.lastPlayers ?? 0) / 60; + if (r.kind === 'population' && healthy) m.values[r.id] = Math.max(value, players); + if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? value + minutes : 0; + if (r.kind === 'sustained-population') { + m.streaks[r.id] = healthy && populated && contiguous ? (m.streaks[r.id] ?? 0) + minutes : 0; + m.values[r.id] = Math.max(value, m.streaks[r.id]); + } + } + m.lastSample = at; m.wasHealthy = healthy; m.lastPlayers = players; +} + +export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] { + return plan.requirements.map(r => { + const budget = ['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind); + const current = r.kind === 'failure-budget' ? metrics.outages : r.kind === 'clean-logs' ? metrics.logErrors + : r.kind === 'restart-budget' ? metrics.restarts + : r.kind === 'manual' ? Number(!!metrics.manual[r.id]) : r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0; + const available = (r.kind !== 'clean-logs' || metrics.logAvailable === true) && (r.kind !== 'restart-budget' || !!metrics.bootId); + return { id: r.id, label: r.label, current, target: r.target, passed: available && (budget ? current <= r.target : current >= r.target), + failed: budget && current > r.target, + detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off' + : !available ? r.kind === 'restart-budget' ? 'Waiting for boot telemetry' : 'Waiting for a readable console log' : budget ? `${current} / ${r.target} allowed; monitored until promotion` + : `${Math.round(current * 100) / 100} / ${r.target}` }; + }); +} + +export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean { + if (!servers.length) return false; + if (plan.scope === 'all') return servers.every(s => s.passed); + if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum; + const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map + : s.game && s.map ? `${s.game}/${s.map}` : null; + const groups = new Set(servers.map(key)); + return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group)); +} diff --git a/src/lib/server/fleet/beta-tests.ts b/src/lib/server/fleet/beta-tests.ts new file mode 100644 index 0000000..c978f77 --- /dev/null +++ b/src/lib/server/fleet/beta-tests.ts @@ -0,0 +1,182 @@ +import { randomUUID } from 'node:crypto'; +import { open, stat } from 'node:fs/promises'; +import { db } from '../db/index.js'; +import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js'; +import { applySample, applySignal, emptyMetrics, evaluateRequirements, type TestMetrics, type TestSignal } from './beta-evaluation.js'; +import { gameLogKey, resolveLogPath, type LogKey } from './logs.js'; + +export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string; cursors: string } +export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null } +interface Cursor { position: number; identity: string; partial: string; anchor?: string; generation?: string; session?: string; joined?: boolean; started?: number } +const nowIso = () => new Date().toISOString(); +export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined; +export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined; +export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); } +export function setOutcome(id: number, outcome: string, reason: string | null): void { + const previous = storedPlan(id); + if (!previous || (previous.outcome === outcome && previous.reason === reason)) return; + db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id); + activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`); +} +export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; } +export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); } +export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean { + return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)') + .run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0; +} + +/** Snapshot log offsets BEFORE restart: pre-beta log entries can never qualify. */ +export async function prepareRun(betaId: number, server: string, phase = 0): Promise { + const cursors: Record = {}; + for (const key of [gameLogKey(server), 'console'] as LogKey[]) { + const path = resolveLogPath(server, key); + const info = path ? await stat(path).catch(() => null) : null; + let anchor = ''; + if (path && info?.size) { + const file = await open(path, 'r'); + try { + const buffer = Buffer.alloc(Math.min(info.size, 128)); + await file.read(buffer, 0, buffer.length, info.size - buffer.length); + anchor = buffer.toString('base64'); + } finally { await file.close(); } + } + cursors[key] = { position: info?.size ?? 0, identity: info ? `${info.dev}:${info.ino}` : '', partial: '', anchor, generation: randomUUID() }; + } + const id = randomUUID(); + db().transaction(() => { + db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server); + db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, cursors) VALUES (?, ?, ?, ?, ?, ?, ?)') + .run(id, betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), JSON.stringify(cursors)); + })(); + return currentRun(betaId, server)!; +} + +export function acceptTestSignal(server: string, runId: string, eventId: string, signal: TestSignal): { accepted: boolean; reason?: string } { + return db().transaction(() => { + const run = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id + JOIN beta_test_plans p ON p.beta_id = b.id WHERE r.id = ? AND r.server = ? AND r.retired_at IS NULL + AND b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed')`).get(runId, server) as TestRun | undefined; + if (!run) return { accepted: false, reason: 'Run is inactive or superseded; obtain the current run at process startup' }; + if (db().prepare('SELECT 1 FROM beta_test_evidence WHERE run_id = ? AND event_id = ?').get(runId, `telemetry:${eventId}`)) return { accepted: false, reason: 'Duplicate event' }; + const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan)); + const metrics = readMetrics(run); + if (!applySignal(plan, metrics, signal, Date.now())) return { accepted: false, reason: 'Duplicate, out-of-order, closed, or unknown session/boot' }; + evidence(run, `telemetry:${eventId}`, signal.event, signal); + saveMetrics(run, metrics); + return { accepted: true }; + })(); +} + +export function parseTestSignal(body: Record): TestSignal { + if (!['boot', 'session_start', 'round_change', 'match_end', 'feature'].includes(String(body.event))) throw new Error('Unknown telemetry event'); + for (const key of ['runId', 'eventId', 'bootId'] as const) { + if (typeof body[key] !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body[key] as string)) throw new Error(`${key} is required (1–128 identifier characters)`); + } + if (body.event !== 'boot' && (typeof body.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body.sessionId))) throw new Error('sessionId is required'); + for (const key of ['sequence', 'round', 'players']) if (!Number.isSafeInteger(body[key]) || (body[key] as number) < 0) throw new Error(`${key} must be a nonnegative integer`); + if (body.event === 'feature' && (typeof body.name !== 'string' || !/^[a-zA-Z0-9_.:-]{1,120}$/.test(body.name))) throw new Error('Feature name is required'); + return { event: body.event as TestSignal['event'], bootId: body.bootId as string, sessionId: body.sessionId as string ?? '', + sequence: body.sequence as number, round: body.round as number, players: body.players as number, name: body.name as string ?? '' }; +} + +/** Bounded incremental reads, with persistent byte cursors and partial lines. */ +export async function observeRun(run: TestRun, plan: BetaPlan): Promise { + const cursors = JSON.parse(run.cursors) as Record; + const observed: Array<{ key: LogKey; line: string; id: string }> = []; + let logAvailable = false; + for (const key of [gameLogKey(run.server), 'console'] as LogKey[]) { + const path = resolveLogPath(run.server, key); + const file = path ? await open(path, 'r').catch(() => null) : null; + if (!file) continue; + try { + const info = await file.stat(); + if (key === 'console') logAvailable = true; + const identity = `${info.dev}:${info.ino}`; + let cursor = cursors[key] ?? { position: 0, identity, partial: '' }; + let anchorChanged = false; + if (cursor.anchor && info.size >= cursor.position) { + const expected = Buffer.from(cursor.anchor, 'base64'); + const actual = Buffer.alloc(expected.length); + await file.read(actual, 0, actual.length, cursor.position - actual.length); + anchorChanged = !actual.equals(expected); + } + if (cursor.identity !== identity || info.size < cursor.position || anchorChanged) { + cursor = { position: 0, identity, partial: '', generation: randomUUID() }; + observed.push({ key, line: '', id: `rotation:${key}:${randomUUID()}` }); + } + if (info.size - cursor.position > 2_000_000) { + setOutcome(run.beta_id, 'inconclusive', `${run.server}: log reader fell behind; evidence is incomplete`); + return; + } + const buffer = Buffer.alloc(info.size - cursor.position); + const { bytesRead } = await file.read(buffer, 0, buffer.length, cursor.position); + const combined = Buffer.concat([Buffer.from(cursor.partial, 'base64'), buffer.subarray(0, bytesRead)]); + let start = 0; + for (let i = 0; i < combined.length; i++) if (combined[i] === 10) { + observed.push({ key, line: combined.subarray(start, i).toString('utf8'), id: `${key}:${identity}:${cursor.generation ?? ''}:${cursor.position + i - Buffer.from(cursor.partial, 'base64').length}` }); + start = i + 1; + } + cursor.position += bytesRead; + const anchor = Buffer.alloc(Math.min(cursor.position, 128)); + await file.read(anchor, 0, anchor.length, cursor.position - anchor.length); + cursor.anchor = anchor.toString('base64'); + cursor.partial = combined.subarray(start).toString('base64'); + if (cursor.partial.length > 100_000) cursor.partial = ''; // malformed giant line cannot grow forever + cursor.identity = identity; + cursors[key] = cursor; + } finally { await file.close(); } + } + db().transaction(() => { + const fresh = currentRun(run.beta_id, run.server); + if (!fresh || fresh.id !== run.id) return; + const m = readMetrics(fresh); + m.logAvailable = logAvailable; + for (const entry of observed) { + const cursor = cursors[entry.key]; + if (!entry.line) { evidence(run, entry.id, 'log-rotation', { log: entry.key }); continue; } + if (entry.key === 'console') { + if (/script (?:runtime )?error|unhandled exception|script exception|fatal error/i.test(entry.line) && evidence(run, entry.id, 'script-error', { line: entry.line.slice(0, 1000) })) m.logErrors++; + continue; + } + if (/InitGame:/.test(entry.line)) { cursor.session = entry.id; cursor.joined = false; cursor.started = undefined; } + if (cursor.session && /\sJ;[^;]*;\d+;/.test(entry.line) && !cursor.joined) { cursor.joined = true; cursor.started = Date.now(); } + if (/ExitLevel:\s*executed/.test(entry.line) && cursor.session && cursor.joined) { + // Unqualified match counts use logs. Round/population-qualified counts use session telemetry exclusively. + if (evidence(run, `match:${cursor.session}`, 'completed-match', { session: cursor.session, line: entry.line, minutes: (Date.now() - (cursor.started ?? Date.now())) / 60_000 })) { + for (const r of plan.requirements) if (r.kind === 'matches' && r.minRound === 0 && r.minPlayers <= 1 + && Date.now() - (cursor.started ?? Date.now()) >= r.minMinutes * 60_000) m.values[r.id] = (m.values[r.id] ?? 0) + 1; + } + cursor.joined = false; + } + if (/ShutdownGame:/.test(entry.line)) cursor.joined = false; + } + const server = db().prepare('SELECT last_checked_at, last_seen_at, last_players, last_error, container_state FROM servers WHERE name = ?').get(run.server) as { + last_checked_at: string | null; last_seen_at: string | null; last_players: number | null; last_error: string | null; container_state: string | null }; + const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0; + const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000; + const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at; + const at = freshSample && server.container_state === 'running' ? sample : Date.now(); + // Startup requires fresh status; don't count an absent first observation as an outage. + const before = m.wasHealthy; + const lastSample = m.lastSample; + applySample(plan, m, at, healthy, healthy ? server.last_players ?? 0 : 0); + if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, players: server.last_players, checkedAt: server.last_checked_at }); + if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, players: server.last_players, reason: server.last_error }); + saveMetrics(run, m); + db().prepare('UPDATE beta_test_runs SET cursors = ? WHERE id = ?').run(JSON.stringify(cursors), run.id); + })(); +} + +export function testView(betaId: number): BetaTestView | null { + const stored = storedPlan(betaId); + if (!stored) return null; + const plan = parseBetaPlan(JSON.parse(stored.plan)); + const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL ORDER BY server').all(betaId) as TestRun[]; + return { plan, outcome: stored.outcome, reason: stored.reason, + servers: runs.map(run => { + const progress = evaluateRequirements(plan, readMetrics(run), run.phase); + return { server: run.server, runId: run.id, phase: run.phase, passed: progress.every(r => r.passed), progress }; + }), + evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e + JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] }; +} diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 5698d9a..c69ffc0 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -1,3 +1,14 @@ +import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; +import { storedPlan } from './beta-tests.js'; + +const operations = new Set(); +export const betaOperationInFlight = (id: number) => operations.has(id); +async function exclusive(id: number, action: () => Promise): Promise { + if (operations.has(id)) throw new BetaError('A beta operation is already running', 'invalid-state'); + operations.add(id); + try { return await action(); } finally { operations.delete(id); } +} + import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { dirname, join, posix } from 'node:path'; @@ -229,7 +240,7 @@ export function findRunningBetaForScript(scriptId: number): BetaRow | undefined .prepare( `SELECT b.* FROM betas b JOIN beta_scripts bs ON bs.beta_id = b.id - WHERE bs.script_id = ? AND b.state = 'running' + WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(scriptId) as BetaRow | undefined; @@ -245,6 +256,7 @@ export interface OpenBetaScript { export interface OpenBetaInput { scripts: OpenBetaScript[]; servers: string[]; + testPlan?: BetaPlan; waitForHealth: boolean; waitForMatch: boolean; notes?: string; @@ -257,6 +269,9 @@ export interface OpenBetaResult { } export async function openBeta(input: OpenBetaInput): Promise { + const plan = input.testPlan ? parseBetaPlan(input.testPlan) : null; + if (new Set(input.servers).size !== input.servers.length) throw new BetaError("Select each server only once", "invalid-state"); + if (plan?.scope === "quorum" && plan.quorum > input.servers.length) throw new BetaError("Quorum exceeds selected servers", "invalid-state"); if (!input.scripts || input.scripts.length === 0) { throw new BetaError('at least one script is required', 'no-scripts'); } @@ -319,6 +334,11 @@ export async function openBeta(input: OpenBetaInput): Promise { serverRows.push({ name, game: row.game, map: row.map }); } + if (plan && ['map', 'game', 'map-game'].includes(plan.scope)) { + const missing = serverRows.some(s => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map)); + if (missing) throw new BetaError('Coverage requires game/map metadata on every selected server', 'invalid-state'); + } + // Per-script scope validation. Aggregate all violations into one error. const outOfScope: Array<{ scriptId: number; scope: string; servers: string[] }> = []; for (const { script } of resolved) { @@ -387,11 +407,16 @@ export async function openBeta(input: OpenBetaInput): Promise { input.notes ?? null ); betaId = Number(res.lastInsertRowid); + if (plan) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(plan)); for (const r of resolved) { insertBetaScript.run(betaId, r.script.id, r.version.id, r.script.kind); } for (const name of input.servers) { insertBetaServer.run(betaId, name, nowIso()); + if (plan) { + const selected = serverRows.find(s => s.name === name)!; + db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map); + } for (const r of resolved) { const prior = getServerScript(name, r.script.target_rel_path); insertBetaServerScript.run( @@ -405,6 +430,30 @@ export async function openBeta(input: OpenBetaInput): Promise { } })(); + if (plan?.waveSize) db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId); + await deployBetaServers(betaId, plan?.waveSize ? input.servers.slice(0, plan.waveSize) : input.servers); + + emit({ + type: 'beta-opened', + betaId, + scriptIds: resolved.map((r) => r.script.id), + servers: input.servers + }); + + return { + betaId, + servers: input.servers, + scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id })) + }; +} + +/** Apply a reserved wave only when its predecessors have passed. */ +export async function deployBetaServers(betaId: number, names: string[]): Promise { + return exclusive(betaId, () => deployBetaServersInternal(betaId, names)); +} +async function deployBetaServersInternal(betaId: number, names: string[]): Promise { + const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! })); + const serverRows = names.map(name => getServer(name)!); // Split resolved artifacts into non-collapsing (each writes its own file) // and collapsing (config/baseline overlay onto one shared dedicated_sp.cfg). const nonCollapsing = resolved.filter((r) => !isCollapsingKind(r.script.kind)); @@ -434,7 +483,9 @@ export async function openBeta(input: OpenBetaInput): Promise { WHERE beta_id = ? AND server = ?` ); - for (const name of input.servers) { + for (const name of names) { + db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?") + .run(nowIso(), betaId, name); const serverRow = serverRows.find((r) => r.name === name)!; let firstErr: string | null = null; // 1. Non-collapsing artifacts: script/plugin — write directly to the @@ -456,7 +507,9 @@ export async function openBeta(input: OpenBetaInput): Promise { sha256Buffer(priorBytes), priorBytes.byteLength ); + db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id); } + db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path); await writeFile(targetPath, version.content); recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`); } catch (err) { @@ -494,6 +547,7 @@ export async function openBeta(input: OpenBetaInput): Promise { ); } const composedBuf = Buffer.from(composedResult.text, 'utf8'); + db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel); await writeFile(targetPath, composedBuf); recordScript( name, @@ -519,18 +573,6 @@ export async function openBeta(input: OpenBetaInput): Promise { } } - emit({ - type: 'beta-opened', - betaId, - scriptIds: resolved.map((r) => r.script.id), - servers: input.servers - }); - - return { - betaId, - servers: input.servers, - scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id })) - }; } // ---------- rollback ---------- @@ -543,6 +585,9 @@ export interface RollbackResult { } export async function rollbackBeta(betaId: number): Promise { + return exclusive(betaId, () => rollbackBetaInternal(betaId)); +} +async function rollbackBetaInternal(betaId: number): Promise { const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'running' && beta.state !== 'ready-promote') { @@ -570,8 +615,10 @@ export async function rollbackBeta(betaId: number): Promise { ); for (const bs of servers) { + if (!bs.deployed_at && bs.stage === 'queued') continue; for (const { script } of bundle) { if (isCollapsingKind(script.kind)) continue; // handled once per server below + if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue; const dest = destOfKind(bs.server, script.kind); const targetPath = join(dest, script.target_rel_path); @@ -601,7 +648,7 @@ export async function rollbackBeta(betaId: number): Promise { } // Collapsing config layer: single restore for the shared cfg file. - if (hasCollapsing) { + if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) { const dest = destOfKind(bs.server, 'config'); const targetRel = configFileOf(bs.server); const targetPath = join(dest, targetRel); @@ -681,6 +728,15 @@ function promoteDeploymentId(betaId: number): string { } export async function promoteBeta(betaId: number): Promise { + if (storedPlan(betaId)) { + const { plannedBetaTick } = await import('../pipeline/beta-plan-runner.js'); + await plannedBetaTick(); + } + return exclusive(betaId, () => promoteBetaInternal(betaId)); +} +async function promoteBetaInternal(betaId: number): Promise { + const test = storedPlan(betaId); + if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote'); const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'ready-promote') { diff --git a/src/lib/server/fleet/scripts.ts b/src/lib/server/fleet/scripts.ts index 5b5cb6c..f4755d0 100644 --- a/src/lib/server/fleet/scripts.ts +++ b/src/lib/server/fleet/scripts.ts @@ -316,7 +316,7 @@ export function deleteVersion(versionId: number): void { .prepare( `SELECT b.id FROM betas b JOIN beta_scripts bs ON bs.beta_id = b.id - WHERE bs.script_version_id = ? AND b.state = 'running' LIMIT 1` + WHERE bs.script_version_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(versionId) as { id: number } | undefined; if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta'); @@ -334,7 +334,7 @@ export function deleteScript(scriptId: number): void { .prepare( `SELECT b.id FROM betas b JOIN beta_scripts bs ON bs.beta_id = b.id - WHERE bs.script_id = ? AND b.state = 'running' LIMIT 1` + WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(scriptId) as { id: number } | undefined; if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta'); diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index 4a4ad42..ba152a3 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -204,7 +204,7 @@ export function heldByBeta(server: string): Map { JOIN betas b ON b.id = bsv.beta_id JOIN beta_scripts bs ON bs.beta_id = b.id JOIN scripts s ON s.id = bs.script_id - WHERE bsv.server = ? AND b.state = 'running'` + WHERE bsv.server = ? AND b.state IN ('running','ready-promote')` ) .all(server) as Array<{ target: string; beta_id: number }>; const map = new Map(); @@ -223,7 +223,7 @@ export function ownerOf(server: string, target: string): number | null { JOIN betas b ON b.id = bsv.beta_id JOIN beta_scripts bs ON bs.beta_id = b.id JOIN scripts s ON s.id = bs.script_id - WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state = 'running' + WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(server, target) as { beta_id: number } | undefined; diff --git a/src/lib/server/pipeline/beta-plan-runner.ts b/src/lib/server/pipeline/beta-plan-runner.ts new file mode 100644 index 0000000..03f3b47 --- /dev/null +++ b/src/lib/server/pipeline/beta-plan-runner.ts @@ -0,0 +1,167 @@ +import { db } from '../db/index.js'; +import { komodoConfigured, rconConfigured } from '../config.js'; +import { rconGetPlayerCount } from '../integrations/rcon.js'; +import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js'; +import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js'; +import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js'; +import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; +import { emit, type BetaStage } from './bus.js'; +import { restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js'; + +function stage(id: number, server: string, next: BetaStage, error: string | null = null): void { + db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?') + .run(next, error, new Date().toISOString(), id, server); + emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error }); +} + +export function betaState(id: number, ready: boolean): void { + const state = ready ? 'ready-promote' : 'running'; + const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state); + if (result.changes) emit({ type: 'beta-state', betaId: id, state }); + // A revoked pass must not leave this version labelled tested unless another beta still proves it. + db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?) + AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL) + AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id + WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`) + .run(ready ? 'tested' : 'untested', id, id); +} + +export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise { + const servers = listBetaServers(id).filter(s => s.deployed_at); + const result = await rollbackBeta(id); + if (plan.rollbackRestart === 'when-empty' && servers.length) { + const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`; + db().transaction(() => { + db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)") + .run(deploymentId, new Date().toISOString(), id); + for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server); + })(); + activity(id, 'Restored files; rolling restarts will wait for empty servers'); + } else activity(id, 'Restored files; manual restarts are required to load them'); + if (result.warnings.length) activity(id, result.warnings.join('; ')); +} + +async function fail(id: number, plan: BetaPlan, reason: string): Promise { + betaState(id, false); + setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason); + if (plan.failureAction === 'rollback') { + try { await rollbackTestPlan(id, plan); } + catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); } + } +} + +let inFlight: Promise | null = null; +export function plannedBetaTick(): Promise { + if (inFlight) return inFlight; + inFlight = tick().finally(() => { inFlight = null; }); + return inFlight; +} +async function tick(): Promise { + const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id + WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[]; + for (const stored of plans) { + if (betaOperationInFlight(stored.beta_id)) continue; + try { await advancePlan(stored); } + catch (error) { + const reason = `Test runner: ${(error as Error).message}`; + try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); } + catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); } + } + } +} + +async function advancePlan(stored: StoredPlan): Promise { + const id = stored.beta_id; + const plan = parseBetaPlan(JSON.parse(stored.plan)); + const beta = getBeta(id)!; + const servers = listBetaServers(id); + if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) { + betaState(id, false); + setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence'); + return; + } + if (!komodoConfigured() || !rconConfigured()) { + betaState(id, false); + setOutcome(id, 'active', 'Waiting for Komodo and RCON configuration'); + return; + } + for (const server of servers) { + const freshPlan = storedPlan(id)!; + if (!['active', 'passed'].includes(freshPlan.outcome)) return; + if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; } + if (server.stage === 'deploying' || server.stage === 'restarting') { + stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`); + await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return; + } + if (server.stage === 'queued') continue; + if (server.stage === 'awaiting-empty') { + const waitingRun = currentRun(id, server.server); + if (waitingRun?.observing_at) { + await observeRun(waitingRun, plan); + if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } + const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed); + if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } + } + if (rolloutWaitReason(server.server)) continue; + if (await rconGetPlayerCount(server.server) !== 0) continue; + const oldRun = currentRun(id, server.server); + const phase = oldRun ? oldRun.phase + 1 : 0; + // Fail closed after interrupted restart requests; never silently reuse earlier evidence. + stage(id, server.server, 'restarting'); + const run = await prepareRun(id, server.server, phase); + if (oldRun) { + const previous = readMetrics(oldRun); + const metrics = readMetrics(run); + metrics.outages = previous.outages; metrics.logErrors = previous.logErrors; metrics.restarts = previous.restarts; + saveMetrics(run, metrics); + } + const restart = await restartInRollout(server.server); + if (restart.kind === 'queued') { + db().prepare('DELETE FROM beta_test_runs WHERE id = ?').run(run.id); + if (oldRun) db().prepare('UPDATE beta_test_runs SET retired_at = NULL WHERE id = ?').run(oldRun.id); + stage(id, server.server, 'awaiting-empty', restart.message); continue; + } + if (restart.kind === 'failed') { stage(id, server.server, 'failed', restart.message); await fail(id, plan, restart.message); return; } + db().prepare('UPDATE beta_servers SET restarted_at = ?, healthy_at = NULL WHERE beta_id = ? AND server = ?') + .run(new Date().toISOString(), id, server.server); + stage(id, server.server, 'awaiting-health'); + continue; + } + if (server.stage === 'awaiting-health') { + const health = rolloutHealth(server.server); + if (!health.ok) continue; + const now = new Date().toISOString(); + db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server); + db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server); + stage(id, server.server, 'observing'); + } + const run = currentRun(id, server.server); + if (!run?.observing_at) continue; + await observeRun(run, plan); + if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } + const metrics = readMetrics(currentRun(id, server.server)!); + const progress = evaluateRequirements(plan, metrics, run.phase); + const broken = progress.find(p => p.failed); + if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; } + const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target)); + if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) { + stage(id, server.server, 'awaiting-empty'); + activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`); + } else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing'); + } + const updated = listBetaServers(id); + const queued = updated.filter(s => s.stage === 'queued'); + if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) { + const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server); + activity(id, `Previous wave passed; deploying ${wave.join(', ')}`); + await deployBetaServers(id, wave); + betaState(id, false); + return; + } + const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>; + // All waves must be activated, even when the final coverage rule is a quorum. + const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at) + && coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' }))); + betaState(id, ready); + setOutcome(id, ready ? 'passed' : 'active', null); +} diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index 087e79a..d67acc6 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -5,6 +5,7 @@ import { komodoConfigured, rconConfigured } from '../config.js'; import { rconGetPlayerCount } from '../integrations/rcon.js'; import { scanForMatchEnd } from '../fleet/match-log.js'; import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js'; +import { plannedBetaTick } from './beta-plan-runner.js'; interface BetaServerActiveRow { beta_id: number; @@ -93,8 +94,9 @@ export function betaTick(): Promise<{ processed: number }> { } async function runBetaTick(): Promise<{ processed: number }> { + if (integrationsOn()) await refreshRestartRollout(); + await plannedBetaTick(); if (!integrationsOn()) return { processed: 0 }; - await refreshRestartRollout(); const rows = db() .prepare( @@ -104,6 +106,7 @@ async function runBetaTick(): Promise<{ processed: number }> { FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE b.state = 'running' + AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id) AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match') ORDER BY b.id, bs.server` ) diff --git a/src/lib/server/pipeline/bus.ts b/src/lib/server/pipeline/bus.ts index b4f08b1..cd9bd88 100644 --- a/src/lib/server/pipeline/bus.ts +++ b/src/lib/server/pipeline/bus.ts @@ -26,6 +26,9 @@ export interface ServerStatusEntry { } export type BetaStage = + | 'queued' + | 'observing' + | 'inconclusive' | 'deploying' | 'awaiting-empty' | 'restarting' diff --git a/src/lib/ui/components/BetaPlanBuilder.svelte b/src/lib/ui/components/BetaPlanBuilder.svelte new file mode 100644 index 0000000..3914997 --- /dev/null +++ b/src/lib/ui/components/BetaPlanBuilder.svelte @@ -0,0 +1,119 @@ + + +
+

Test plan

+

All requirements must pass on each qualifying server. Milestones stay recorded; health and error budgets remain monitored until promotion.

+ + {#each plan.requirements as r, index (r.id)} +
+ {requirementLabels[r.kind]} +
+ + {#if r.kind !== 'manual'} + + {/if} + {#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population', 'feature'].includes(r.kind)} + + {/if} + {#if r.kind === 'matches' || r.kind === 'feature'} + + {/if} + {#if r.kind === 'matches'} + + {/if} + {#if r.kind === 'feature'} + + {/if} +
+ {#if r.kind === 'round' || r.kind === 'feature' || r.kind === 'restart-budget' || (r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1))} +

Requires session telemetry from the game reporter. Progress waits for events from this beta’s current run.

+ {:else if r.kind === 'recovery'} +

After the other requirements pass, restart when empty and repeat them. Every cycle requires fresh evidence and checklist sign-off.

+ {:else if r.kind === 'failure-budget'} +

Counts transitions from healthy to unhealthy. Missing or stale monitoring resets continuous health and does not earn playtime.

+ {:else if r.kind === 'clean-logs'} +

Checks new console lines for script errors, script exceptions, unhandled exceptions and fatal errors. An unreadable log cannot pass.

+ {:else if r.kind === 'manual'} +

A tester records their name and observation on the beta detail page.

+ {/if} + +
+ {/each} +
+ + +
+
+ Coverage and rollout +
+ + {#if plan.scope === 'quorum'}{/if} + +
+

Each wave must fully pass before files reach the next wave. Coverage rules apply after all selected servers have started testing.

+
+
+ Deadline and failures +
+ + + +
+

Missing evidence at the deadline is inconclusive. Promotion always requires a manual review.

+
+
+ + +
+ {#if message}

{message}

{/if} +
+ + diff --git a/src/lib/ui/components/BetaTestProgress.svelte b/src/lib/ui/components/BetaTestProgress.svelte new file mode 100644 index 0000000..1b37c06 --- /dev/null +++ b/src/lib/ui/components/BetaTestProgress.svelte @@ -0,0 +1,82 @@ + + +
+

Test plan

{test.outcome}
+ {#if test.reason}

{test.reason}

{/if} +

Coverage: {test.plan.scope === 'quorum' ? `${test.plan.quorum} servers` : test.plan.scope}. Window: {test.plan.timeoutHours} hours. + {test.plan.waveSize ? `${test.plan.waveSize} servers per wave.` : 'All servers deployed together.'} + Failure action: {test.plan.failureAction}. Rollback restart: {test.plan.rollbackRestart}.

+ {#if !closed} +
+ {#if test.outcome === 'paused'}{/if} + {#if ['active', 'passed'].includes(test.outcome)}{/if} + {#if ['paused', 'failed'].includes(test.outcome)}{/if} +
+ {/if} + {#if error}{/if} + {#if test.servers.length === 0}

Waiting for the first server restart. Evidence collection begins with that run.

{/if} + {#each test.servers as server (server.server)} +
+

{server.server} Recovery cycles: {server.phase}

+ {#each server.progress as progress (progress.id)} + {@const requirement = test.plan.requirements.find(r => r.id === progress.id)!} +
+
{progress.passed ? '✓' : progress.failed ? '✕' : '○'} {progress.label}{progress.detail}
+ {#if ['round', 'matches', 'feature', 'populated-minutes', 'player-hours', 'sustained-population'].includes(requirement.kind)} + Minimum {requirement.minPlayers} player(s){requirement.minRound ? `, round ${requirement.minRound} in the same session` : ''}{requirement.minMinutes ? `, ${requirement.minMinutes} minutes per match` : ''}{requirement.event ? ` · Event: ${requirement.event}` : ''} + {/if} + {#if !['manual', 'failure-budget', 'restart-budget', 'clean-logs'].includes(requirement.kind)} + + {/if} + {#if requirement.kind === 'manual' && !progress.passed && !closed} +
+ + +
+ {/if} +
+ {/each} +
+ {/each} +
+ Evidence ({test.evidence.length}{test.evidence.length === 200 ? ' most recent' : ''}) + {#each test.evidence as event (event.id)} +
{event.server} · {event.kind}
{event.payload}
+ {/each} + {#if !test.evidence.length}

No evidence recorded yet.

{/if} +
+
+ + diff --git a/src/routes/api/betas/+server.ts b/src/routes/api/betas/+server.ts index 712104c..8c54395 100644 --- a/src/routes/api/betas/+server.ts +++ b/src/routes/api/betas/+server.ts @@ -8,6 +8,8 @@ import { } from '$lib/server/fleet/beta.js'; import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; import { kick } from '$lib/server/pipeline/worker.js'; +import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js'; +import { storedPlan } from '$lib/server/fleet/beta-tests.js'; export async function GET({ url }: { url: URL }) { const stateFilter = url.searchParams.get('state') as @@ -32,6 +34,7 @@ export async function GET({ url }: { url: URL }) { }); return { ...b, + testOutcome: storedPlan(b.id)?.outcome ?? null, scripts, servers: listBetaServers(b.id) }; @@ -51,7 +54,9 @@ export async function POST({ request }: { request: Request }) { waitForHealth?: boolean; waitForMatch?: boolean; notes?: string; + testPlan?: BetaPlan; }; + if (!body || typeof body !== 'object' || Array.isArray(body)) return json({ ok: false, error: 'Expected a JSON object' }, { status: 400 }); // Normalize into the openBeta shape. Both fields feed the same list. const rawEntries: Array<{ scriptId?: number; scriptVersionId?: number }> = []; @@ -82,7 +87,11 @@ export async function POST({ request }: { request: Request }) { } try { + let testPlan: BetaPlan | undefined; + try { if (body.testPlan !== undefined) testPlan = parseBetaPlan(body.testPlan); } + catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); } const result = await openBeta({ + testPlan, scripts: normalized, servers: body.servers, waitForHealth: body.waitForHealth ?? true, diff --git a/src/routes/api/betas/[id]/+server.ts b/src/routes/api/betas/[id]/+server.ts index 859e302..c8f5c5b 100644 --- a/src/routes/api/betas/[id]/+server.ts +++ b/src/routes/api/betas/[id]/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js'; import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; +import { testView } from '$lib/server/fleet/beta-tests.js'; function parseId(raw: string): number | null { const n = Number.parseInt(raw, 10); @@ -30,5 +31,5 @@ export async function GET({ params }: { params: { id: string } }) { const servers = listBetaServers(id); const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id); - return json({ beta, scripts, servers, events }); + return json({ beta, scripts, servers, events, test: testView(id) }); } diff --git a/src/routes/api/betas/[id]/retry/+server.ts b/src/routes/api/betas/[id]/retry/+server.ts index 9ae975e..1938014 100644 --- a/src/routes/api/betas/[id]/retry/+server.ts +++ b/src/routes/api/betas/[id]/retry/+server.ts @@ -1,6 +1,7 @@ import { json } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; import { emit, type BetaStage } from '$lib/server/pipeline/bus.js'; +import { storedPlan } from '$lib/server/fleet/beta-tests.js'; function parseId(raw: string): number | null { const n = Number.parseInt(raw, 10); @@ -12,6 +13,7 @@ function parseId(raw: string): number | null { export async function POST({ params, request }: { params: { id: string }; request: Request }) { const betaId = parseId(params.id); if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 }); + if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 }); const body = (await request.json().catch(() => ({}))) as { server?: string }; if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 }); diff --git a/src/routes/api/betas/[id]/rollback/+server.ts b/src/routes/api/betas/[id]/rollback/+server.ts index e05b4a8..ba0b3f6 100644 --- a/src/routes/api/betas/[id]/rollback/+server.ts +++ b/src/routes/api/betas/[id]/rollback/+server.ts @@ -1,5 +1,8 @@ import { json } from '@sveltejs/kit'; import { rollbackBeta, BetaError } from '$lib/server/fleet/beta.js'; +import { plannedBetaTick, rollbackTestPlan } from '$lib/server/pipeline/beta-plan-runner.js'; +import { storedPlan, setOutcome } from '$lib/server/fleet/beta-tests.js'; +import { parseBetaPlan } from '$lib/beta-plan.js'; function parseId(raw: string): number | null { const n = Number.parseInt(raw, 10); @@ -10,6 +13,13 @@ export async function POST({ params }: { params: { id: string } }) { const id = parseId(params.id); if (id === null) return json({ ok: false, error: 'bad id' }, { status: 400 }); try { + await plannedBetaTick(); + const test = storedPlan(id); + if (test) { + setOutcome(id, 'paused', 'Rollback requested'); + await rollbackTestPlan(id, parseBetaPlan(JSON.parse(test.plan))); + return json({ ok: true }); + } const result = await rollbackBeta(id); return json({ ok: true, result }); } catch (err) { diff --git a/src/routes/api/betas/[id]/test/+server.ts b/src/routes/api/betas/[id]/test/+server.ts new file mode 100644 index 0000000..22f98c9 --- /dev/null +++ b/src/routes/api/betas/[id]/test/+server.ts @@ -0,0 +1,55 @@ +import { json } from '@sveltejs/kit'; +import { db } from '$lib/server/db/index.js'; +import { getBeta } from '$lib/server/fleet/beta.js'; +import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js'; +import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js'; +import { parseBetaPlan } from '$lib/beta-plan.js'; +import { kick } from '$lib/server/pipeline/worker.js'; + +export async function POST({ params, request }: { params: { id: string }; request: Request }) { + const id = Number(params.id); + if (!Number.isSafeInteger(id) || id < 1) return json({ error: 'Invalid beta ID' }, { status: 400 }); + await plannedBetaTick(); + const beta = getBeta(id); + const stored = storedPlan(id); + if (!beta || !stored) return json({ error: 'Test plan not found' }, { status: 404 }); + if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 }); + try { + const body = await request.json(); + const plan = parseBetaPlan(JSON.parse(stored.plan)); + if (body.action === 'sign-off') { + if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off'); + const requirement = plan.requirements.find(r => r.id === body.requirementId && r.kind === 'manual'); + const run = typeof body.server === 'string' ? currentRun(id, body.server) : undefined; + if (!requirement || !run?.observing_at) throw new Error('Checklist item or active server run not found'); + if (typeof body.note !== 'string' || !body.note.trim() || body.note.length > 1000) throw new Error('Add a tester name and observation (up to 1000 characters)'); + const metrics = readMetrics(run); + metrics.manual[requirement.id] = body.note.trim(); + saveMetrics(run, metrics); + evidence(run, `manual:${requirement.id}:${Date.now()}`, 'manual-sign-off', { requirement: requirement.label, note: body.note.trim() }); + activity(id, `${body.server}: ${requirement.label} signed off: ${body.note.trim()}`); + } else if (body.action === 'pause') { + setOutcome(id, 'paused', 'Paused by operator; deployment and evidence evaluation held'); + betaState(id, false); + } else if (body.action === 'resume') { + if (stored.outcome !== 'paused') throw new Error('Only paused plans can resume; failed or expired plans need a fresh attempt'); + setOutcome(id, 'active', null); + } else if (body.action === 'retry') { + // Fresh attempts keep their predecessors as audit history. Preserve the original deadline: + // operators open another beta if the overall testing window has expired. + if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline'); + if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta'); + db().transaction(() => { + db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(new Date().toISOString(), id); + db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error NOT LIKE 'deploy:%'").run(id); + // SQL NULL does not satisfy NOT LIKE. + db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error IS NULL").run(id); + betaState(id, false); + setOutcome(id, 'active', null); + activity(id, 'Fresh test attempt requested; deployed servers will restart when empty'); + })(); + } else throw new Error('Unknown test action'); + void kick(); + return json({ ok: true }); + } catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); } +} diff --git a/src/routes/api/betas/presets/+server.ts b/src/routes/api/betas/presets/+server.ts new file mode 100644 index 0000000..6a8da26 --- /dev/null +++ b/src/routes/api/betas/presets/+server.ts @@ -0,0 +1,18 @@ +import { json } from '@sveltejs/kit'; +import { db } from '$lib/server/db/index.js'; +import { builtInPlans, parseBetaPlan } from '$lib/beta-plan.js'; +export function GET() { + const saved = db().prepare('SELECT name, plan FROM beta_test_presets ORDER BY name').all() as Array<{ name: string; plan: string }>; + return json({ presets: [...builtInPlans, ...saved.map(p => ({ name: p.name, plan: JSON.parse(p.plan) }))] }); +} +export async function POST({ request }: { request: Request }) { + try { + const body = await request.json(); + if (typeof body.name !== 'string' || !body.name.trim() || body.name.length > 80) throw new Error('Preset name must be 1–80 characters'); + if (builtInPlans.some(p => p.name === body.name.trim())) throw new Error('Choose a name other than a built-in preset'); + const plan = parseBetaPlan(body.plan); + db().prepare('INSERT INTO beta_test_presets (name, plan) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET plan = excluded.plan') + .run(body.name.trim(), JSON.stringify(plan)); + return json({ ok: true }); + } catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); } +} diff --git a/src/routes/api/telemetry/+server.ts b/src/routes/api/telemetry/+server.ts index d102991..e0aa035 100644 --- a/src/routes/api/telemetry/+server.ts +++ b/src/routes/api/telemetry/+server.ts @@ -2,6 +2,19 @@ import { json } from '@sveltejs/kit'; import { getServer, listServers } from '$lib/server/fleet/servers.js'; import { saveTelemetry } from '$lib/server/fleet/telemetry.js'; import { emit } from '$lib/server/pipeline/bus.js'; +import { db } from '$lib/server/db/index.js'; +import { acceptTestSignal, parseTestSignal } from '$lib/server/fleet/beta-tests.js'; + +// Reporters fetch this once at process startup and keep the token for that boot. +export async function GET({ url }: { url: URL }) { + const server = url.searchParams.get('server'); + if (!server || !getServer(server)) return json({ error: 'Unknown server' }, { status: 404 }); + const runs = db().prepare(`SELECT r.id AS runId, r.beta_id AS betaId FROM beta_test_runs r + JOIN betas b ON b.id = r.beta_id JOIN beta_test_plans p ON p.beta_id = b.id + WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote') + AND p.outcome IN ('active','passed')`).all(server); + return json({ runs }, { headers: { 'Cache-Control': 'no-store' } }); +} export async function POST({ request }: { request: Request }) { console.log('[telemetry] received POST /api/telemetry'); @@ -30,7 +43,7 @@ export async function POST({ request }: { request: Request }) { return reject(400, 'Payload must be a JSON object'); } const issues: string[] = []; - if (body.event !== 'round_change') issues.push('event must equal "round_change"'); + if (body.runId === undefined && body.event !== 'round_change') issues.push('Session telemetry requires runId'); if (typeof body.server !== 'string' || !body.server.length || body.server.length > 255) { issues.push('server must be a registered server name (1–255 characters)'); } @@ -56,6 +69,14 @@ export async function POST({ request }: { request: Request }) { receivedServer: body.server, destroyedAt: server.destroyed_at }); } + let test: { accepted: boolean; reason?: string } | undefined; + if (body.runId !== undefined) { + try { + const signal = parseTestSignal(body); + test = acceptTestSignal(body.server, body.runId, body.eventId, signal); + } catch (error) { return reject(400, (error as Error).message); } + if (body.event !== 'round_change') return json({ ok: true, test }); + } const telemetry = { server: body.server as string, round: body.round as number, players: body.players as number, receivedAt: new Date().toISOString() @@ -63,5 +84,5 @@ export async function POST({ request }: { request: Request }) { saveTelemetry(telemetry); console.log(`[telemetry] accepted round_change ${JSON.stringify(telemetry)}`); emit({ type: 'server-telemetry', entries: [telemetry] }); - return json({ ok: true, telemetry }); + return json({ ok: true, telemetry, test }); } diff --git a/src/routes/betas/+page.svelte b/src/routes/betas/+page.svelte index 6337963..718c4dc 100644 --- a/src/routes/betas/+page.svelte +++ b/src/routes/betas/+page.svelte @@ -1,4 +1,6 @@