227 lines
9.2 KiB
TypeScript
227 lines
9.2 KiB
TypeScript
import { getServer } from './servers.js';
|
|
import { serverObservation } from './status.js';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { db } from '../db/index.js';
|
|
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
|
|
import type { GameTelemetry } from '../../telemetry.js';
|
|
import {
|
|
applySample,
|
|
applyGameEvent,
|
|
emptyMetrics,
|
|
evaluateRequirements,
|
|
type TestMetrics
|
|
} from './beta-evaluation.js';
|
|
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
|
import { currentBetaRevision } from './beta-revisions.js';
|
|
import { invalidateBeta } from '../pipeline/bus.js';
|
|
|
|
export interface TestRun {
|
|
id: string;
|
|
beta_id: number;
|
|
server: string;
|
|
phase: number;
|
|
created_at: string;
|
|
observing_at: string | null;
|
|
metrics: string;
|
|
}
|
|
export interface StoredPlan {
|
|
beta_id: number;
|
|
plan: string;
|
|
outcome: string;
|
|
reason: string | null;
|
|
}
|
|
const nowIso = () => new Date().toISOString();
|
|
const activeParticipant = (id: number, server: string) =>
|
|
!!db()
|
|
.prepare(
|
|
`SELECT 1 FROM beta_servers
|
|
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
|
|
)
|
|
.get(id, server);
|
|
export const storedPlan = (id: number) =>
|
|
db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as
|
|
StoredPlan | undefined;
|
|
export const currentRun = (id: number, server: string) =>
|
|
db()
|
|
.prepare(
|
|
'SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
|
|
)
|
|
.get(id, server) as TestRun | undefined;
|
|
export function activity(id: number, message: string) {
|
|
db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message);
|
|
invalidateBeta(id);
|
|
}
|
|
export function setOutcome(id: number, outcome: string, reason: string | null): void {
|
|
const previous = storedPlan(id);
|
|
if (!previous || (previous.outcome === outcome && previous.reason === reason)) return;
|
|
db()
|
|
.prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?')
|
|
.run(outcome, reason, id);
|
|
activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`);
|
|
}
|
|
export function readMetrics(run: TestRun): TestMetrics {
|
|
return { ...emptyMetrics(), ...JSON.parse(run.metrics) };
|
|
}
|
|
export function saveMetrics(run: TestRun, metrics: TestMetrics): void {
|
|
db()
|
|
.prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?')
|
|
.run(JSON.stringify(metrics), run.id);
|
|
invalidateBeta(run.beta_id);
|
|
}
|
|
export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean {
|
|
return (
|
|
db()
|
|
.prepare(
|
|
'INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)'
|
|
)
|
|
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0
|
|
);
|
|
}
|
|
|
|
/** Internal progress record; never sent to or required from the game script. */
|
|
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
|
|
if (!activeParticipant(betaId, server))
|
|
throw new Error('Original test server was deleted; start a new beta');
|
|
db().transaction(() => {
|
|
db()
|
|
.prepare(
|
|
'UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL'
|
|
)
|
|
.run(nowIso(), betaId, server);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
|
)
|
|
.run(
|
|
randomUUID(),
|
|
betaId,
|
|
server,
|
|
phase,
|
|
nowIso(),
|
|
JSON.stringify(emptyMetrics()),
|
|
currentBetaRevision(betaId).revision
|
|
);
|
|
})();
|
|
return currentRun(betaId, server)!;
|
|
}
|
|
|
|
/** Route ordinary server events to every deployed, active beta automatically. */
|
|
export function recordGameEvent(event: GameTelemetry): void {
|
|
const runs = db()
|
|
.prepare(
|
|
`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
|
|
JOIN beta_test_plans p ON p.beta_id = b.id
|
|
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
|
|
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
|
|
AND s.server_deleted_at IS NULL
|
|
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
|
|
AND s.restarted_at IS NOT NULL`
|
|
)
|
|
.all(event.server) as TestRun[];
|
|
for (const run of runs) {
|
|
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
|
|
const metrics = readMetrics(run);
|
|
applyGameEvent(plan, metrics, event, Date.parse(event.receivedAt));
|
|
evidence(run, randomUUID(), event.event, event);
|
|
saveMetrics(run, metrics);
|
|
}
|
|
}
|
|
|
|
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
|
if (!activeParticipant(run.beta_id, run.server)) return;
|
|
const m = readMetrics(run);
|
|
const row = getServer(run.server);
|
|
const server = row ? serverObservation(row) : null;
|
|
let timestamp =
|
|
server?.status_source === 'telemetry' ? server.last_seen_at : server?.last_checked_at;
|
|
let source = server?.status_source;
|
|
// Telemetry owns the displayed roster until it expires, but a newer
|
|
// successful RCON reply is already health evidence during the handoff.
|
|
const rconSeen = server?.rcon_status?.last_seen_at;
|
|
if (
|
|
server?.health === 'online' &&
|
|
server.status_source === 'telemetry' &&
|
|
rconSeen &&
|
|
Date.parse(rconSeen) > Date.parse(timestamp ?? '') &&
|
|
Date.parse(rconSeen) <= Date.now()
|
|
) {
|
|
timestamp = rconSeen;
|
|
source = 'rcon';
|
|
}
|
|
const sample = Date.parse(timestamp ?? '');
|
|
const freshSample =
|
|
Number.isFinite(sample) &&
|
|
sample >= Date.parse(run.observing_at ?? run.created_at) &&
|
|
Date.now() - sample >= 0 &&
|
|
Date.now() - sample <= ONLINE_MAX_MS;
|
|
const healthy = freshSample && server?.health === 'online';
|
|
const at = freshSample && server?.container_state === 'running' ? sample : Date.now();
|
|
const before = m.wasHealthy;
|
|
const lastSample = m.lastSample;
|
|
applySample(plan, m, at, healthy);
|
|
if (freshSample && sample !== lastSample)
|
|
evidence(run, `sample:${sample}`, 'status-sample', {
|
|
healthy,
|
|
checkedAt: timestamp,
|
|
source
|
|
});
|
|
if (before !== m.wasHealthy)
|
|
evidence(run, `health:${at}`, 'health', { healthy, reason: server?.last_error ?? null });
|
|
saveMetrics(run, m);
|
|
}
|
|
|
|
export function testView(betaId: number): BetaTestView | null {
|
|
const stored = storedPlan(betaId);
|
|
if (!stored) return null;
|
|
const plan = parseBetaPlan(JSON.parse(stored.plan));
|
|
const runs = db()
|
|
.prepare(
|
|
`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
|
|
JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
|
|
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`
|
|
)
|
|
.all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
|
|
return {
|
|
plan,
|
|
outcome: stored.outcome,
|
|
reason: stored.reason,
|
|
servers: runs.map((run) => {
|
|
const metrics = readMetrics(run);
|
|
const progress = evaluateRequirements(plan, metrics, run.phase);
|
|
const latest = db()
|
|
.prepare(
|
|
"SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1"
|
|
)
|
|
.get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
|
|
const event = latest ? JSON.parse(latest.payload) : null;
|
|
return {
|
|
server: run.server,
|
|
server_deleted_at: run.server_deleted_at,
|
|
phase: run.phase,
|
|
passed: progress.every((r) => r.passed),
|
|
progress,
|
|
observingAt: run.observing_at,
|
|
observation: {
|
|
lastGameplayAt: latest?.received_at ?? null,
|
|
lastEvent: latest?.kind ?? null,
|
|
players: metrics.match?.players ?? null,
|
|
round: event?.round ?? null,
|
|
matchStarted: metrics.match?.started !== undefined,
|
|
gameEnded: metrics.match?.ended ?? false,
|
|
healthy: metrics.wasHealthy ?? null,
|
|
healthCheckedAt: metrics.lastSample
|
|
? new Date(metrics.lastSample).toISOString()
|
|
: null
|
|
}
|
|
};
|
|
}),
|
|
evidence: db()
|
|
.prepare(
|
|
`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
|
|
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
|
|
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`
|
|
)
|
|
.all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence']
|
|
};
|
|
}
|