import { getServer } from './servers.js'; import { serverObservation } from './status.js'; import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js'; import type { GameTelemetry } from '../../telemetry.js'; import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { currentBetaRevision } from './beta-revisions.js'; import { invalidateBeta } from '../pipeline/bus.js'; export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string; } export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null; } const nowIso = () => new Date().toISOString(); const activeParticipant = (id: number, server: string) => !!db() .prepare( `SELECT 1 FROM beta_servers WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL` ) .get(id, server); export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined; export const currentRun = (id: number, server: string) => db() .prepare( 'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL' ) .get(id, server) as TestRun | undefined; export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); invalidateBeta(id); } export function setOutcome(id: number, outcome: string, reason: string | null): void { const previous = storedPlan(id); if (!previous || (previous.outcome === outcome && previous.reason === reason)) return; db() .prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?') .run(outcome, reason, id); activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`); } export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; } export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db() .prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?') .run(JSON.stringify(metrics), run.id); invalidateBeta(run.beta_id); } export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean { return ( db() .prepare( 'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)' ) .run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0 ); } /** Internal progress record; never sent to or required from the game script. */ export async function prepareRun(betaId: number, server: string, phase = 0): Promise { if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta'); db().transaction(() => { db() .prepare( 'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL' ) .run(nowIso(), betaId, server); db() .prepare( 'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)' ) .run( randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision ); })(); return currentRun(betaId, server)!; } /** Route ordinary server events to every deployed, active beta automatically. */ export function recordGameEvent(event: GameTelemetry): void { const runs = db() .prepare( `SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id JOIN beta_test_plans p ON p.beta_id = b.id JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote') AND s.server_deleted_at IS NULL AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready') AND s.restarted_at IS NOT NULL` ) .all(event.server) as TestRun[]; for (const run of runs) { const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan)); const metrics = readMetrics(run); applyGameEvent(plan, metrics, event, Date.parse(event.receivedAt)); evidence(run, randomUUID(), event.event, event); saveMetrics(run, metrics); } } export async function observeRun(run: TestRun, plan: BetaPlan): Promise { if (!activeParticipant(run.beta_id, run.server)) return; const m = readMetrics(run); const row = getServer(run.server); const server = row ? serverObservation(row) : null; let timestamp = server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at; let source = server?.status_source; // Telemetry owns the displayed roster until it expires, but a newer // successful RCON reply is already health evidence during the handoff. const rconSeen = server?.rcon_status?.last_seen_at; if ( server?.health === 'online' && server.status_source === 'telemetry' && rconSeen && Date.parse(rconSeen) > Date.parse(timestamp ?? '') && Date.parse(rconSeen) <= Date.now() ) { timestamp = rconSeen; source = 'rcon'; } const sample = Date.parse(timestamp ?? ''); const freshSample = Number.isFinite(sample) && sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample >= 0 && Date.now() - sample <= ONLINE_MAX_MS; const healthy = freshSample && server?.health === 'online'; const at = freshSample && server?.container_state === 'running' ? sample : Date.now(); const before = m.wasHealthy; const lastSample = m.lastSample; applySample(plan, m, at, healthy); if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: timestamp, source }); if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null }); saveMetrics(run, m); } export function testView(betaId: number): BetaTestView | null { const stored = storedPlan(betaId); if (!stored) return null; const plan = parseBetaPlan(JSON.parse(stored.plan)); const runs = db() .prepare( `SELECT r.*, s.server_deleted_at FROM beta_test_runs r JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server` ) .all(betaId) as Array; return { plan, outcome: stored.outcome, reason: stored.reason, servers: runs.map((run) => { const metrics = readMetrics(run); const progress = evaluateRequirements(plan, metrics, run.phase); const latest = db() .prepare( "SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1" ) .get(run.id) as { received_at: string; kind: string; payload: string } | undefined; const event = latest ? JSON.parse(latest.payload) : null; return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every((r) => r.passed), progress, observingAt: run.observing_at, observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null, players: metrics.match?.players ?? null, round: event?.round ?? null, matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false, healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } }; }), evidence: db() .prepare( `SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200` ) .all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] }; }