Simplify telemetry to three game events

This commit is contained in:
2026-09-18 15:35:21 -04:00
parent 6d903adb31
commit 5bb49a776e
29 changed files with 658 additions and 1314 deletions
+3 -6
View File
@@ -18,10 +18,7 @@ export function requirementSummary(r: BetaRequirement): string {
case 'sustained-population': return `Keep at least ${count(r.minPlayers, 'player')} online for ${duration(r.target)}`;
case 'healthy-minutes': return `Stay continuously healthy for ${duration(r.target)}`;
case 'failure-budget': return r.target === 0 ? 'Allow no health outages' : `Allow at most ${count(r.target, 'health outage')}`;
case 'restart-budget': return r.target === 0 ? 'Allow no unexpected restarts' : `Allow at most ${count(r.target, 'unexpected restart')}`;
case 'clean-logs': return r.target === 0 ? 'Allow no detected script errors' : `Allow at most ${count(r.target, 'detected script error')}`;
case 'recovery': return `Repeat the checks after ${count(r.target, 'recovery restart')}`;
case 'feature': return `Observe ${r.event || 'a named feature event'} ${count(r.target, 'time')}${players}${r.minRound ? ` at round ${r.minRound} or later` : ''}`;
case 'manual': return r.label;
}
}
@@ -29,9 +26,9 @@ export function coverageSummary(plan: BetaPlan): string {
return plan.scope === 'all' ? 'Every selected server' : plan.scope === 'quorum' ? `At least ${count(plan.quorum, 'selected server')}`
: plan.scope === 'map' ? 'One server per selected map' : plan.scope === 'game' ? 'One server per selected game' : 'One server per game / map';
}
export function needsSessionTelemetry(r: BetaRequirement): boolean {
return ['round', 'feature', 'restart-budget'].includes(r.kind) || (r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1));
export function needsGameTelemetry(r: BetaRequirement): boolean {
return ['round', 'matches', 'population', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind);
}
export function requirementCategory(r: BetaRequirement): string {
return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind) ? 'Continuous check' : 'Milestone';
return r.kind === 'manual' ? 'Tester sign-off' : ['healthy-minutes', 'failure-budget'].includes(r.kind) ? 'Continuous check' : 'Milestone';
}
+9 -12
View File
@@ -1,5 +1,5 @@
export const requirementKinds = ['round', 'matches', 'populated-minutes', 'player-hours', 'population',
'sustained-population', 'healthy-minutes', 'failure-budget', 'restart-budget', 'clean-logs', 'recovery', 'feature', 'manual'] as const;
'sustained-population', 'healthy-minutes', 'failure-budget', 'recovery', 'manual'] as const;
export type RequirementKind = typeof requirementKinds[number];
export interface BetaRequirement {
id: string;
@@ -9,7 +9,6 @@ export interface BetaRequirement {
minRound: number;
minPlayers: number;
minMinutes: number;
event: string;
}
export interface BetaPlan {
version: 1;
@@ -25,12 +24,11 @@ export const requirementLabels: Record<RequirementKind, string> = {
round: 'Reach round', matches: 'Complete qualifying matches', 'populated-minutes': 'Populated minutes',
'player-hours': 'Player-hours', population: 'Reach player count', 'sustained-population': 'Sustained population (minutes)',
'healthy-minutes': 'Continuous healthy minutes', 'failure-budget': 'Maximum health outages',
'restart-budget': 'Maximum unexpected restarts',
'clean-logs': 'Maximum script errors', recovery: 'Successful recovery cycles', feature: 'Feature event count', manual: 'Tester checklist'
recovery: 'Successful recovery cycles', manual: 'Tester checklist'
};
export function requirement(kind: RequirementKind, id: string): BetaRequirement {
return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : ['failure-budget', 'restart-budget', 'clean-logs'].includes(kind) ? 0 : 1,
minRound: 0, minPlayers: 1, minMinutes: 0, event: '' };
return { id, kind, label: requirementLabels[kind], target: kind === 'round' ? 25 : kind === 'failure-budget' ? 0 : 1,
minRound: 0, minPlayers: 1, minMinutes: 0 };
}
export function defaultPlan(): BetaPlan {
return { version: 1, requirements: [requirement('healthy-minutes', 'health')], scope: 'all', quorum: 1,
@@ -44,7 +42,7 @@ export const builtInPlans: Array<{ name: string; plan: BetaPlan }> = [
] } },
{ name: 'Long soak', plan: { ...defaultPlan(), timeoutHours: 72, requirements: [
{ ...requirement('healthy-minutes', 'health'), target: 360 },
{ ...requirement('player-hours', 'participation'), target: 10 }, requirement('clean-logs', 'logs')
{ ...requirement('player-hours', 'participation'), target: 10 }, requirement('failure-budget', 'outages')
] } }
];
@@ -68,13 +66,12 @@ export function parseBetaPlan(value: unknown): BetaPlan {
for (const key of ['target', 'minRound', 'minPlayers', 'minMinutes'] as const) {
if (typeof r[key] !== 'number' || !Number.isFinite(r[key]) || r[key] < 0 || r[key] > 1_000_000) throw new Error(`Invalid requirement ${key}`);
}
if (!['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind) && r.target <= 0) throw new Error('Requirement target must be positive');
if (['round', 'matches', 'population', 'failure-budget', 'restart-budget', 'clean-logs', 'recovery', 'feature', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers');
if (r.kind !== 'failure-budget' && r.target <= 0) throw new Error('Requirement target must be positive');
if (['round', 'matches', 'population', 'failure-budget', 'recovery', 'manual'].includes(r.kind) && !Number.isInteger(r.target)) throw new Error('Count targets must be integers');
if (!Number.isInteger(r.minRound) || !Number.isInteger(r.minPlayers)) throw new Error('Round and player qualifiers must be integers');
if (r.kind === 'manual' && r.target !== 1) throw new Error('Add a separate requirement for each checklist item');
if (typeof r.event !== 'string' || r.event.length > 120 || (r.kind === 'feature' && !/^[a-zA-Z0-9_.:-]+$/.test(r.event))) throw new Error('Feature requirements need an event name');
return { id: r.id, kind: r.kind, label: r.label.trim(), target: r.target, minRound: r.minRound,
minPlayers: r.minPlayers, minMinutes: r.minMinutes, event: r.event };
minPlayers: r.minPlayers, minMinutes: r.minMinutes };
});
return { version: 1, requirements, scope: p.scope, quorum: p.quorum, timeoutHours: p.timeoutHours,
failureAction: p.failureAction, rollbackRestart: p.rollbackRestart, waveSize: p.waveSize };
@@ -85,6 +82,6 @@ export interface BetaTestView {
plan: BetaPlan;
outcome: string;
reason: string | null;
servers: Array<{ server: string; runId: string; phase: number; passed: boolean; progress: RequirementProgress[] }>;
servers: Array<{ server: string; phase: number; passed: boolean; progress: RequirementProgress[] }>;
evidence: Array<{ id: number; server: string; received_at: string; kind: string; payload: string }>;
}
+3 -3
View File
@@ -1,7 +1,7 @@
import { parseServerName, zombieMapBySlug } from './map-aliases.js';
import { freshStatusTimestamp } from './server-freshness.js';
import type { ZombieGame } from './map-aliases.js';
import type { RoundTelemetry } from './telemetry.js';
import type { GameTelemetry } from './telemetry.js';
import type { Health, RosterPlayer } from './ui/stores/servers.js';
import { mapTitle, serverTitle } from './ui/server-labels.js';
@@ -63,8 +63,8 @@ export function fleetIdentity(server: FleetServer) {
};
}
export function currentRound(server: FleetServer, report: RoundTelemetry | undefined, now: number): number | null {
if (!report || !freshServer(server, now) || !server.active_since || !server.game_started_at
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
if (!report || report.event !== 'round_change' || !freshServer(server, now) || !server.active_since || !server.game_started_at
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
const received = Date.parse(report.receivedAt);
const age = now - received;
+38
View File
@@ -3,6 +3,8 @@ import { mkdirSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { config } from '../config.js';
import schemaSql from './schema.sql?raw';
import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-plan.js';
import { emptyMetrics } from '../fleet/beta-evaluation.js';
let _db: Database.Database | null = null;
@@ -190,6 +192,40 @@ function assertSnapshotSchemeSafe(handle: Database.Database): void {
handle.prepare("INSERT INTO settings (key, value) VALUES ('snapshots_v1', '1')").run();
}
// Preserve old plans as reviewable checklists instead of silently passing checks
// that the three-event protocol can no longer perform. Existing evidence stays
// in history; active progress starts fresh under the new gameplay contract.
function migrateGameTelemetry(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'game_telemetry_v2'").get()) return;
handle.transaction(() => {
const convert = (raw: string): string => {
const plan = JSON.parse(raw) as BetaPlan;
plan.requirements = plan.requirements.map(r => {
const { event: _event, ...rest } = r as BetaRequirement & { event?: string };
return ['feature', 'restart-budget', 'clean-logs'].includes(r.kind)
? { ...requirement('manual', r.id), label: `Review: ${r.label}`.slice(0, 200) } : rest;
});
return JSON.stringify(plan);
};
const plans = handle.prepare('SELECT beta_id, plan FROM beta_test_plans').all() as Array<{ beta_id: number; plan: string }>;
for (const row of plans) handle.prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?').run(convert(row.plan), row.beta_id);
const presets = handle.prepare('SELECT name, plan FROM beta_test_presets').all() as Array<{ name: string; plan: string }>;
for (const row of presets) handle.prepare('UPDATE beta_test_presets SET plan = ? WHERE name = ?').run(convert(row.plan), row.name);
handle.prepare(`UPDATE beta_test_runs SET metrics = ?, cursors = '{}' WHERE retired_at IS NULL
AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'))`).run(JSON.stringify(emptyMetrics()));
handle.exec(`UPDATE beta_test_plans SET outcome = 'active', reason = 'Gameplay telemetry updated; collecting fresh script events'
WHERE outcome IN ('active','passed') AND beta_id IN (SELECT id FROM betas WHERE state IN ('running','ready-promote'));
UPDATE script_versions SET state = 'untested' WHERE state = 'tested'
AND id IN (SELECT bs.script_version_id FROM beta_scripts bs JOIN beta_test_plans p ON p.beta_id = bs.beta_id WHERE p.outcome = 'active')
AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL)
AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.script_version_id = script_versions.id AND b.state = 'promoted');
UPDATE betas SET state = 'running' WHERE state = 'ready-promote' AND id IN (SELECT beta_id FROM beta_test_plans WHERE outcome = 'active');
UPDATE beta_servers SET stage = 'observing' WHERE stage = 'ready' AND beta_id IN (SELECT beta_id FROM beta_test_plans WHERE outcome = 'active');
INSERT INTO settings (key, value) VALUES ('game_telemetry_v2', '1');`);
})();
}
export function db(): Database.Database {
if (_db) return _db;
@@ -212,6 +248,7 @@ export function db(): Database.Database {
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]);
// Kind-referencing indexes: safe here because migrateTable above just
// ALTER'd the column into existence if it was missing.
handle.exec(
@@ -221,6 +258,7 @@ export function db(): Database.Database {
handle.exec(`CREATE INDEX IF NOT EXISTS idx_scripts_kind ON scripts(kind)`);
backfillMultiScriptJoins(handle);
assertSnapshotSchemeSafe(handle);
migrateGameTelemetry(handle);
} catch (err) {
handle.close();
throw err;
+4 -3
View File
@@ -39,8 +39,8 @@ CREATE TABLE IF NOT EXISTS servers (
-- and resume from the same anchor when the server comes back.
active_since TEXT, -- ISO ts of last 0 -> >0 player transition
inactive_since TEXT, -- ISO ts of last >0 -> 0 player transition
game_started_at TEXT, -- wall-clock ISO when we first saw a J; line for the current match cycle
game_ended_at TEXT, -- wall-clock ISO when we first saw ExitLevel/ShutdownGame end it
game_started_at TEXT, -- receipt time of the script's game_start
game_ended_at TEXT, -- receipt time of the script's game_end
game_cycle_key TEXT, -- opaque marker for the currently-tracked cycle (the first J; line after the latest InitGame); lets us detect same-players game restarts
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
);
@@ -314,7 +314,8 @@ CREATE TABLE IF NOT EXISTS server_telemetry (
server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE,
round INTEGER NOT NULL,
players INTEGER NOT NULL,
received_at TEXT NOT NULL
received_at TEXT NOT NULL,
payload TEXT
);
-- Lifecycle history deliberately survives removal of the server registry row.
CREATE TABLE IF NOT EXISTS server_deployments (
+46 -71
View File
@@ -1,108 +1,83 @@
import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
import type { GameEvent } from '../../telemetry.js';
export interface SessionEvidence {
started: number;
sequence: number;
export interface MatchEvidence {
map: string;
started?: number;
lastEvent: number;
players: number;
ended: boolean;
rounds: Record<string, number>;
peakPlayers: number;
}
export interface TestMetrics {
values: Record<string, number>;
sessions: Record<string, SessionEvidence>;
match?: MatchEvidence;
manual: Record<string, string>;
bootId?: string;
boots: string[];
restarts: number;
lastSample?: number;
wasHealthy?: boolean;
lastPlayers?: number;
outages: number;
logErrors: number;
logAvailable?: boolean;
streaks: Record<string, number>;
}
export const emptyMetrics = (): TestMetrics => ({ values: {}, sessions: {}, manual: {}, boots: [], restarts: 0, outages: 0, logErrors: 0, streaks: {} });
export const emptyMetrics = (): TestMetrics => ({ values: {}, manual: {}, outages: 0, streaks: {} });
export interface TestSignal {
event: 'boot' | 'session_start' | 'round_change' | 'match_end' | 'feature';
bootId: string;
sessionId: string;
sequence: number;
round: number;
players: number;
name: string;
}
/** Events are accepted once, in session order; a match must start in this run. */
export function applySignal(plan: BetaPlan, metrics: TestMetrics, signal: TestSignal, now: number): boolean {
if (signal.event === 'boot') {
if (metrics.boots.includes(signal.bootId)) return false;
if (metrics.bootId) metrics.restarts++;
metrics.boots.push(signal.bootId);
metrics.bootId = signal.bootId;
return true;
/** Gameplay comes only from the script. No reporter IDs or beta handshake. */
export function applyGameEvent(plan: BetaPlan, metrics: TestMetrics, event: GameEvent, now: number): void {
const players = event.players.length;
if (event.event === 'game_start' || !metrics.match || metrics.match.map !== event.map) {
metrics.match = { map: event.map, started: event.event === 'game_start' ? now : undefined,
lastEvent: now, players, ended: false, rounds: {} };
metrics.streaks = {};
}
if (metrics.bootId !== signal.bootId) return false;
const key = `${signal.bootId}:${signal.sessionId}`;
if (signal.event === 'session_start') {
if (metrics.sessions[key]) return false;
metrics.sessions[key] = { started: now, sequence: signal.sequence, ended: false, rounds: {}, peakPlayers: signal.players };
return true;
}
const session = metrics.sessions[key];
if (!session || session.ended || signal.sequence <= session.sequence) return false;
session.sequence = signal.sequence;
session.peakPlayers = Math.max(session.peakPlayers, signal.players);
const match = metrics.match;
if (match.ended) return;
const minutes = Math.max(0, now - match.lastEvent) / 60_000;
for (const r of plan.requirements) {
const current = metrics.values[r.id] ?? 0;
if (signal.event === 'round_change' && signal.players >= r.minPlayers) {
session.rounds[r.id] = Math.max(session.rounds[r.id] ?? 0, signal.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, signal.round);
const populated = players >= Math.max(1, r.minPlayers) && match.players >= Math.max(1, r.minPlayers);
if (r.kind === 'population') metrics.values[r.id] = Math.max(current, players);
if (r.kind === 'populated-minutes' && populated) metrics.values[r.id] = current + minutes;
if (r.kind === 'player-hours' && populated) metrics.values[r.id] = current + minutes * Math.min(players, match.players) / 60;
if (r.kind === 'sustained-population') {
metrics.streaks[r.id] = populated ? (metrics.streaks[r.id] ?? 0) + minutes : 0;
metrics.values[r.id] = Math.max(current, metrics.streaks[r.id]);
}
if (event.event !== 'game_start' && players >= Math.max(1, r.minPlayers)) {
match.rounds[r.id] = Math.max(match.rounds[r.id] ?? 0, event.round);
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, event.round);
}
if (event.event === 'game_end' && r.kind === 'matches'
&& (match.rounds[r.id] ?? -1) >= r.minRound
&& (r.minMinutes === 0 || (match.started !== undefined && now - match.started >= r.minMinutes * 60_000))) {
metrics.values[r.id] = current + 1;
}
if (signal.event === 'match_end' && r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1)
&& (session.rounds[r.id] ?? 0) >= r.minRound && session.peakPlayers >= r.minPlayers
&& now - session.started >= r.minMinutes * 60_000) metrics.values[r.id] = current + 1;
if (signal.event === 'feature' && r.kind === 'feature' && signal.name === r.event
&& signal.players >= r.minPlayers && signal.round >= r.minRound) metrics.values[r.id] = current + 1;
}
if (signal.event === 'match_end') session.ended = true;
return true;
match.lastEvent = now;
match.players = players;
match.ended = event.event === 'game_end';
}
/** Sample both ends of an interval. Monitoring gaps never earn playtime. */
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean, players: number): void {
/** Infrastructure health is separate from script-reported gameplay. */
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean): void {
if (m.lastSample !== undefined && at <= m.lastSample) return;
const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
const contiguous = gap <= 60_000;
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
if (!healthy && m.wasHealthy === true) m.outages++;
for (const r of plan.requirements) {
const value = m.values[r.id] ?? 0;
const populated = players >= Math.max(1, r.minPlayers) && (m.lastPlayers ?? 0) >= Math.max(1, r.minPlayers);
if (r.kind === 'populated-minutes' && populated) m.values[r.id] = value + minutes;
if (r.kind === 'player-hours' && populated) m.values[r.id] = value + minutes * Math.min(players, m.lastPlayers ?? 0) / 60;
if (r.kind === 'population' && healthy) m.values[r.id] = Math.max(value, players);
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? value + minutes : 0;
if (r.kind === 'sustained-population') {
m.streaks[r.id] = healthy && populated && contiguous ? (m.streaks[r.id] ?? 0) + minutes : 0;
m.values[r.id] = Math.max(value, m.streaks[r.id]);
}
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? (m.values[r.id] ?? 0) + minutes : 0;
}
m.lastSample = at; m.wasHealthy = healthy; m.lastPlayers = players;
m.lastSample = at; m.wasHealthy = healthy;
}
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
return plan.requirements.map(r => {
const budget = ['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind);
const current = r.kind === 'failure-budget' ? metrics.outages : r.kind === 'clean-logs' ? metrics.logErrors
: r.kind === 'restart-budget' ? metrics.restarts
: r.kind === 'manual' ? Number(!!metrics.manual[r.id]) : r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
const available = (r.kind !== 'clean-logs' || metrics.logAvailable === true) && (r.kind !== 'restart-budget' || !!metrics.bootId);
return { id: r.id, label: r.label, current, target: r.target, passed: available && (budget ? current <= r.target : current >= r.target),
const budget = r.kind === 'failure-budget';
const current = budget ? metrics.outages : r.kind === 'manual' ? Number(!!metrics.manual[r.id])
: r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
return { id: r.id, label: r.label, current, target: r.target, passed: budget ? current <= r.target : current >= r.target,
failed: budget && current > r.target,
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
: !available ? r.kind === 'restart-budget' ? 'Waiting for boot telemetry' : 'Waiting for a readable console log' : budget ? `${current} / ${r.target} allowed; monitored until promotion`
: budget ? `${current} / ${r.target} allowed; monitored until promotion`
: `${Math.round(current * 100) / 100} / ${r.target}` };
});
}
+31 -132
View File
@@ -1,13 +1,11 @@
import { randomUUID } from 'node:crypto';
import { open, stat } from 'node:fs/promises';
import { db } from '../db/index.js';
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
import { applySample, applySignal, emptyMetrics, evaluateRequirements, type TestMetrics, type TestSignal } from './beta-evaluation.js';
import { gameLogKey, resolveLogPath, type LogKey } from './logs.js';
import type { GameTelemetry } from '../../telemetry.js';
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string; cursors: string }
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
interface Cursor { position: number; identity: string; partial: string; anchor?: string; generation?: string; session?: string; joined?: boolean; started?: number }
const nowIso = () => new Date().toISOString();
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
@@ -25,146 +23,47 @@ export function evidence(run: TestRun, eventId: string, kind: string, payload: u
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0;
}
/** Snapshot log offsets BEFORE restart: pre-beta log entries can never qualify. */
/** Internal progress record; never sent to or required from the game script. */
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
const cursors: Record<string, Cursor> = {};
for (const key of [gameLogKey(server), 'console'] as LogKey[]) {
const path = resolveLogPath(server, key);
const info = path ? await stat(path).catch(() => null) : null;
let anchor = '';
if (path && info?.size) {
const file = await open(path, 'r');
try {
const buffer = Buffer.alloc(Math.min(info.size, 128));
await file.read(buffer, 0, buffer.length, info.size - buffer.length);
anchor = buffer.toString('base64');
} finally { await file.close(); }
}
cursors[key] = { position: info?.size ?? 0, identity: info ? `${info.dev}:${info.ino}` : '', partial: '', anchor, generation: randomUUID() };
}
const id = randomUUID();
db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, cursors) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), JSON.stringify(cursors));
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)')
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()));
})();
return currentRun(betaId, server)!;
}
export function acceptTestSignal(server: string, runId: string, eventId: string, signal: TestSignal): { accepted: boolean; reason?: string } {
return db().transaction(() => {
const run = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
JOIN beta_test_plans p ON p.beta_id = b.id WHERE r.id = ? AND r.server = ? AND r.retired_at IS NULL
AND b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed')`).get(runId, server) as TestRun | undefined;
if (!run) return { accepted: false, reason: 'Run is inactive or superseded; obtain the current run at process startup' };
if (db().prepare('SELECT 1 FROM beta_test_evidence WHERE run_id = ? AND event_id = ?').get(runId, `telemetry:${eventId}`)) return { accepted: false, reason: 'Duplicate event' };
/** Route ordinary server events to every deployed, active beta automatically. */
export function recordGameEvent(event: GameTelemetry): void {
const runs = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
JOIN beta_test_plans p ON p.beta_id = b.id
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
for (const run of runs) {
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
const metrics = readMetrics(run);
if (!applySignal(plan, metrics, signal, Date.now())) return { accepted: false, reason: 'Duplicate, out-of-order, closed, or unknown session/boot' };
evidence(run, `telemetry:${eventId}`, signal.event, signal);
applyGameEvent(plan, metrics, event, Date.parse(event.receivedAt));
evidence(run, randomUUID(), event.event, event);
saveMetrics(run, metrics);
return { accepted: true };
})();
}
export function parseTestSignal(body: Record<string, unknown>): TestSignal {
if (!['boot', 'session_start', 'round_change', 'match_end', 'feature'].includes(String(body.event))) throw new Error('Unknown telemetry event');
for (const key of ['runId', 'eventId', 'bootId'] as const) {
if (typeof body[key] !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body[key] as string)) throw new Error(`${key} is required (1–128 identifier characters)`);
}
if (body.event !== 'boot' && (typeof body.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body.sessionId))) throw new Error('sessionId is required');
for (const key of ['sequence', 'round', 'players']) if (!Number.isSafeInteger(body[key]) || (body[key] as number) < 0) throw new Error(`${key} must be a nonnegative integer`);
if (body.event === 'feature' && (typeof body.name !== 'string' || !/^[a-zA-Z0-9_.:-]{1,120}$/.test(body.name))) throw new Error('Feature name is required');
return { event: body.event as TestSignal['event'], bootId: body.bootId as string, sessionId: body.sessionId as string ?? '',
sequence: body.sequence as number, round: body.round as number, players: body.players as number, name: body.name as string ?? '' };
}
/** Bounded incremental reads, with persistent byte cursors and partial lines. */
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
const cursors = JSON.parse(run.cursors) as Record<string, Cursor>;
const observed: Array<{ key: LogKey; line: string; id: string }> = [];
let logAvailable = false;
for (const key of [gameLogKey(run.server), 'console'] as LogKey[]) {
const path = resolveLogPath(run.server, key);
const file = path ? await open(path, 'r').catch(() => null) : null;
if (!file) continue;
try {
const info = await file.stat();
if (key === 'console') logAvailable = true;
const identity = `${info.dev}:${info.ino}`;
let cursor = cursors[key] ?? { position: 0, identity, partial: '' };
let anchorChanged = false;
if (cursor.anchor && info.size >= cursor.position) {
const expected = Buffer.from(cursor.anchor, 'base64');
const actual = Buffer.alloc(expected.length);
await file.read(actual, 0, actual.length, cursor.position - actual.length);
anchorChanged = !actual.equals(expected);
}
if (cursor.identity !== identity || info.size < cursor.position || anchorChanged) {
cursor = { position: 0, identity, partial: '', generation: randomUUID() };
observed.push({ key, line: '', id: `rotation:${key}:${randomUUID()}` });
}
if (info.size - cursor.position > 2_000_000) {
setOutcome(run.beta_id, 'inconclusive', `${run.server}: log reader fell behind; evidence is incomplete`);
return;
}
const buffer = Buffer.alloc(info.size - cursor.position);
const { bytesRead } = await file.read(buffer, 0, buffer.length, cursor.position);
const combined = Buffer.concat([Buffer.from(cursor.partial, 'base64'), buffer.subarray(0, bytesRead)]);
let start = 0;
for (let i = 0; i < combined.length; i++) if (combined[i] === 10) {
observed.push({ key, line: combined.subarray(start, i).toString('utf8'), id: `${key}:${identity}:${cursor.generation ?? ''}:${cursor.position + i - Buffer.from(cursor.partial, 'base64').length}` });
start = i + 1;
}
cursor.position += bytesRead;
const anchor = Buffer.alloc(Math.min(cursor.position, 128));
await file.read(anchor, 0, anchor.length, cursor.position - anchor.length);
cursor.anchor = anchor.toString('base64');
cursor.partial = combined.subarray(start).toString('base64');
if (cursor.partial.length > 100_000) cursor.partial = ''; // malformed giant line cannot grow forever
cursor.identity = identity;
cursors[key] = cursor;
} finally { await file.close(); }
}
db().transaction(() => {
const fresh = currentRun(run.beta_id, run.server);
if (!fresh || fresh.id !== run.id) return;
const m = readMetrics(fresh);
m.logAvailable = logAvailable;
for (const entry of observed) {
const cursor = cursors[entry.key];
if (!entry.line) { evidence(run, entry.id, 'log-rotation', { log: entry.key }); continue; }
if (entry.key === 'console') {
if (/script (?:runtime )?error|unhandled exception|script exception|fatal error/i.test(entry.line) && evidence(run, entry.id, 'script-error', { line: entry.line.slice(0, 1000) })) m.logErrors++;
continue;
}
if (/InitGame:/.test(entry.line)) { cursor.session = entry.id; cursor.joined = false; cursor.started = undefined; }
if (cursor.session && /\sJ;[^;]*;\d+;/.test(entry.line) && !cursor.joined) { cursor.joined = true; cursor.started = Date.now(); }
if (/ExitLevel:\s*executed/.test(entry.line) && cursor.session && cursor.joined) {
// Unqualified match counts use logs. Round/population-qualified counts use session telemetry exclusively.
if (evidence(run, `match:${cursor.session}`, 'completed-match', { session: cursor.session, line: entry.line, minutes: (Date.now() - (cursor.started ?? Date.now())) / 60_000 })) {
for (const r of plan.requirements) if (r.kind === 'matches' && r.minRound === 0 && r.minPlayers <= 1
&& Date.now() - (cursor.started ?? Date.now()) >= r.minMinutes * 60_000) m.values[r.id] = (m.values[r.id] ?? 0) + 1;
}
cursor.joined = false;
}
if (/ShutdownGame:/.test(entry.line)) cursor.joined = false;
}
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_players, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
last_checked_at: string | null; last_seen_at: string | null; last_players: number | null; last_error: string | null; container_state: string | null };
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
// Startup requires fresh status; don't count an absent first observation as an outage.
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy, healthy ? server.last_players ?? 0 : 0);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, players: server.last_players, checkedAt: server.last_checked_at });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, players: server.last_players, reason: server.last_error });
saveMetrics(run, m);
db().prepare('UPDATE beta_test_runs SET cursors = ? WHERE id = ?').run(JSON.stringify(cursors), run.id);
})();
const m = readMetrics(run);
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
last_checked_at: string | null; last_seen_at: string | null; last_error: string | null; container_state: string | null };
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
const before = m.wasHealthy;
const lastSample = m.lastSample;
applySample(plan, m, at, healthy);
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, checkedAt: server.last_checked_at });
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, reason: server.last_error });
saveMetrics(run, m);
}
export function testView(betaId: number): BetaTestView | null {
@@ -175,7 +74,7 @@ export function testView(betaId: number): BetaTestView | null {
return { plan, outcome: stored.outcome, reason: stored.reason,
servers: runs.map(run => {
const progress = evaluateRequirements(plan, readMetrics(run), run.phase);
return { server: run.server, runId: run.id, phase: run.phase, passed: progress.every(r => r.passed), progress };
return { server: run.server, phase: run.phase, passed: progress.every(r => r.passed), progress };
}),
evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
-110
View File
@@ -1,110 +0,0 @@
import { resolveLogPath, tailLog, gameLogKey } from './logs.js';
// Match markers for T5 Plutonium's `logs/games_sp.log`. Verified against a live
// sample: each game session is bracketed by `InitGame:` and `ShutdownGame:`.
// A REAL completed match always emits `ExitLevel: executed` right before
// `ShutdownGame:`. Idle map cycles print `InitGame:` / `ShutdownGame:` at
// 0:00 with no ExitLevel — those don't count as a played match.
//
// So the reliable discriminator is `ExitLevel: executed`: seeing that line
// after `healthy_at` means a game actually ended normally (no crash — a crash
// would kill the container before ExitLevel fires and take RCON with it).
export const MATCH_START = /InitGame:/;
export const MATCH_END = /ExitLevel:\s*executed/;
export const MATCH_SHUTDOWN = /ShutdownGame:/;
// Player join line: `MM:SS J;<guid>;<slot>;<name>`. games_sp.log accumulates
// across restarts and contains many old J; lines; only the ones AFTER the
// latest InitGame belong to the current match cycle.
export const MATCH_JOIN = /\sJ;[^;]*;\d+;/;
/**
* Poll a server's games_sp.log tail for match markers. Returns whether a
* completed match end has been observed since `sinceIso` (the server's
* `healthy_at` timestamp).
*/
export async function scanForMatchEnd(
serverName: string,
sinceIso: string
): Promise<{ ended: boolean; started: boolean; reason?: string }> {
const path = resolveLogPath(serverName, gameLogKey(serverName));
if (!path) return { ended: false, started: false, reason: 'log path resolution failed' };
const tail = await tailLog(path, 2000);
if (!tail) return { ended: false, started: false, reason: 'log not written yet' };
if (tail.mtime < sinceIso) return { ended: false, started: false, reason: 'no log activity since healthy' };
let started = false;
let ended = false;
for (const line of tail.lines) {
if (MATCH_START.test(line)) started = true;
if (MATCH_END.test(line)) ended = true;
}
return { ended, started };
}
export type MatchScanKind = 'no-signal' | 'idle' | 'in-progress' | 'ended';
export interface MatchScanResult {
kind: MatchScanKind;
// Opaque identifier for the currently-observed match cycle. When kind is
// 'in-progress' or 'ended', this is the first J; line after the latest
// InitGame. Compare against persisted state to detect a fresh cycle
// (e.g. game auto-restart with same players — new J; line = new key).
cycleKey?: string;
}
/**
* Read the games_sp.log tail and classify the current match state. We do NOT
* infer any wall-clock timestamps from log content — the caller stamps
* game_started_at/game_ended_at with the observation time.
*
* - `in-progress`: latest InitGame has at least one J; join line after it
* and no ExitLevel/ShutdownGame between that J; and end-of-tail. `cycleKey`
* is set to the first J; line after the latest InitGame.
* - `ended`: latest InitGame has a J; join line followed by an ExitLevel
* or ShutdownGame. `cycleKey` identifies which cycle ended.
* - `idle`: no J; line after the latest InitGame — empty map loaded, no
* match happening.
* - `no-signal`: log missing / unreadable / no InitGame in tail.
*/
export async function scanMatchState(serverName: string): Promise<MatchScanResult> {
const path = resolveLogPath(serverName, gameLogKey(serverName));
if (!path) return { kind: 'no-signal' };
const tail = await tailLog(path, 2000);
if (!tail || tail.lines.length === 0) return { kind: 'no-signal' };
const lines = tail.lines;
// Find the LATEST InitGame — everything before it is prior sessions that
// this log accumulated. Only what comes after is the current cycle.
let lastInit = -1;
for (let i = lines.length - 1; i >= 0; i--) {
if (MATCH_START.test(lines[i])) {
lastInit = i;
break;
}
}
if (lastInit === -1) return { kind: 'no-signal' };
// Scan only the current cycle (everything after the latest InitGame).
let firstJoinIdx = -1;
let endIdx = -1;
for (let i = lastInit + 1; i < lines.length; i++) {
if (firstJoinIdx === -1 && MATCH_JOIN.test(lines[i])) {
firstJoinIdx = i;
}
if (MATCH_END.test(lines[i]) || MATCH_SHUTDOWN.test(lines[i])) {
endIdx = i;
break;
}
}
// No join in the current cycle -> nobody entered this match yet.
if (firstJoinIdx === -1) return { kind: 'idle' };
const cycleKey = lines[firstJoinIdx];
if (endIdx !== -1 && endIdx > firstJoinIdx) {
return { kind: 'ended', cycleKey };
}
return { kind: 'in-progress', cycleKey };
}
+19 -10
View File
@@ -1,15 +1,24 @@
import { db } from '../db/index.js';
import type { RoundTelemetry } from '../../telemetry.js';
import type { GameTelemetry } from '../../telemetry.js';
export function listTelemetry(): RoundTelemetry[] {
return db().prepare(`SELECT t.server, t.round, t.players, t.received_at AS receivedAt
FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL`).all() as RoundTelemetry[];
export function listTelemetry(): GameTelemetry[] {
const rows = db().prepare(`SELECT t.payload FROM server_telemetry t JOIN servers s ON s.name = t.server
WHERE s.destroyed_at IS NULL AND t.payload IS NOT NULL`).all() as Array<{ payload: string }>;
return rows.map(row => JSON.parse(row.payload) as GameTelemetry);
}
export function saveTelemetry(report: RoundTelemetry): void {
db().prepare(`INSERT INTO server_telemetry (server, round, players, received_at)
VALUES (?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
round = excluded.round, players = excluded.players, received_at = excluded.received_at`)
.run(report.server, report.round, report.players, report.receivedAt);
export function saveTelemetry(report: GameTelemetry): void {
db().prepare(`INSERT INTO server_telemetry (server, round, players, received_at, payload)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET
round = excluded.round, players = excluded.players, received_at = excluded.received_at, payload = excluded.payload`)
.run(report.server, report.event === 'game_start' ? 0 : report.round, report.players.length, report.receivedAt, JSON.stringify(report));
if (report.event === 'game_start') {
db().prepare('UPDATE servers SET game_started_at = ?, game_ended_at = NULL WHERE name = ?').run(report.receivedAt, report.server);
} else if (report.event === 'game_end') {
db().prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
// Legacy betas also take match completion directly from the script.
db().prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
.run(report.receivedAt, report.server, report.receivedAt);
}
}
+1 -1
View File
@@ -112,7 +112,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
if (oldRun) {
const previous = readMetrics(oldRun);
const metrics = readMetrics(run);
metrics.outages = previous.outages; metrics.logErrors = previous.logErrors; metrics.restarts = previous.restarts;
metrics.outages = previous.outages;
saveMetrics(run, metrics);
}
const restart = await restartInRollout(server.server);
+3 -5
View File
@@ -3,7 +3,6 @@ import { emit } from './bus.js';
import type { BetaStage } from './bus.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { scanForMatchEnd } from '../fleet/match-log.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
import { plannedBetaTick } from './beta-plan-runner.js';
@@ -12,6 +11,7 @@ interface BetaServerActiveRow {
server: string;
stage: BetaStage;
healthy_at: string | null;
match_seen_at: string | null;
wait_for_health: number;
wait_for_match: number;
}
@@ -101,7 +101,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
const rows = db()
.prepare(
`SELECT bs.beta_id AS beta_id, bs.server AS server, bs.stage AS stage,
bs.healthy_at AS healthy_at,
bs.healthy_at AS healthy_at, bs.match_seen_at AS match_seen_at,
b.wait_for_health AS wait_for_health, b.wait_for_match AS wait_for_match
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
@@ -181,9 +181,7 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
return;
}
case 'awaiting-match': {
const since = row.healthy_at ?? nowIso();
const result = await scanForMatchEnd(row.server, since);
if (!result.ended) return;
if (!row.match_seen_at) return;
transition(row.beta_id, row.server, 'ready');
return;
}
+2 -2
View File
@@ -1,5 +1,5 @@
import { EventEmitter } from 'node:events';
import type { RoundTelemetry } from '../../telemetry.js';
import type { GameTelemetry } from '../../telemetry.js';
import type { RconEndpoint, RconFailure, RconPlayer } from '../integrations/rcon.js';
import type { Health } from '../fleet/health.js';
@@ -40,7 +40,7 @@ export type BetaStage =
export type BetaState = 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
export type PipelineEvent =
| { type: 'server-telemetry'; entries: RoundTelemetry[] }
| { type: 'server-telemetry'; entries: GameTelemetry[] }
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
| { type: 'server-changed'; deploymentId: string; server: string; files: number }
| {
+2 -1
View File
@@ -56,7 +56,8 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
+2 -66
View File
@@ -10,7 +10,6 @@ import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
import { scanMatchState } from '../fleet/match-log.js';
const TICK_MS = 15_000;
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
@@ -44,61 +43,6 @@ async function runTick(): Promise<void> {
} catch (err) {
console.error('[worker] beta-runner error:', err);
}
try {
await scanMatches();
} catch (err) {
console.error('[worker] match-scan error:', err);
}
}
/**
* Read every server's games_sp.log tail to advance game_started_at /
* game_ended_at. Concurrency-capped so 15 servers don't hammer disk at once.
*/
async function scanMatches(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
let cursor = 0;
async function pump(): Promise<void> {
while (true) {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
const result = await scanMatchState(srv.name).catch((err) => {
console.error(`[worker] scanMatchState(${srv.name}) failed:`, err);
return { kind: 'no-signal' as const };
});
const persistedInProgress = !!srv.game_started_at && !srv.game_ended_at;
const persistedKey = srv.game_cycle_key;
const now = new Date().toISOString();
if (result.kind === 'in-progress' && result.cycleKey) {
// New cycle when persisted key differs (fresh start, or a game
// restart with the same players — new J; line = new key).
if (result.cycleKey !== persistedKey) {
setServerField(srv.name, 'game_started_at', now);
setServerField(srv.name, 'game_ended_at', null);
setServerField(srv.name, 'game_cycle_key', result.cycleKey);
} else if (!persistedInProgress) {
// Same key but previous cycle was ended — treat as new run
// (shouldn't happen in practice since key would differ,
// but be defensive).
setServerField(srv.name, 'game_started_at', now);
setServerField(srv.name, 'game_ended_at', null);
}
} else if (result.kind === 'ended' && result.cycleKey) {
// Only stamp the end if it's the cycle we've been tracking and
// we haven't already stamped it.
if (result.cycleKey === persistedKey && persistedInProgress) {
setServerField(srv.name, 'game_ended_at', now);
}
}
// 'idle' / 'no-signal': leave state alone. UI hides the timer when
// the server has no players (see ServerCard/detail page).
}
}
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
}
/**
@@ -169,8 +113,8 @@ async function refreshStatus(): Promise<void> {
const nextPlayers = result.status.players;
let activeSince = srv.active_since;
let inactiveSince = srv.inactive_since;
let gameStartedAt = srv.game_started_at;
let gameEndedAt = srv.game_ended_at;
const gameStartedAt = srv.game_started_at;
const gameEndedAt = srv.game_ended_at;
if (nextPlayers > 0 && (activeSince === null || prevPlayers === 0 || prevPlayers === null)) {
activeSince = now;
inactiveSince = null;
@@ -181,14 +125,6 @@ async function refreshStatus(): Promise<void> {
activeSince = null;
setServerField(srv.name, 'inactive_since', inactiveSince);
setServerField(srv.name, 'active_since', null);
// Game timer cannot run while inactive. Drop any tracked
// match state so the next active session anchors fresh
// when a new J; line fires.
gameStartedAt = null;
gameEndedAt = null;
setServerField(srv.name, 'game_started_at', null);
setServerField(srv.name, 'game_ended_at', null);
setServerField(srv.name, 'game_cycle_key', null);
}
// Persist last-observed state. last_seen_at is the health anchor.
setServerField(srv.name, 'last_players', nextPlayers);
+48 -5
View File
@@ -1,6 +1,49 @@
export interface RoundTelemetry {
server: string;
round: number;
players: number;
receivedAt: string;
export interface TelemetryPlayer {
name: string;
guid: string;
}
export interface TelemetryPlayerStats extends TelemetryPlayer {
kills: number;
headshots: number;
downs: number;
revives: number;
score: number;
}
export type GameEvent = { server: string; map: string } & (
| { event: 'game_start'; players: TelemetryPlayer[] }
| { event: 'round_change' | 'game_end'; round: number; players: TelemetryPlayerStats[] }
);
export type GameTelemetry = GameEvent & { receivedAt: string };
/** Validate the three script payloads; beta state is never part of the protocol. */
export function parseGameEvent(value: unknown): GameEvent {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Payload must be a JSON object');
const body = value as Record<string, unknown>;
if (!['game_start', 'round_change', 'game_end'].includes(String(body.event))) throw new Error('event must be game_start, round_change, or game_end');
for (const key of ['server', 'map'] as const) {
if (typeof body[key] !== 'string' || !body[key].trim() || body[key].length > 255) throw new Error(`${key} must be a nonempty string (up to 255 characters)`);
}
if (!Array.isArray(body.players)) throw new Error('players must be an array');
const players = body.players.map((value, index) => {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`players[${index}] must be an object`);
const player = value as Record<string, unknown>;
for (const key of ['name', 'guid'] as const) {
if (typeof player[key] !== 'string' || !player[key].trim()) throw new Error(`players[${index}].${key} must be a nonempty string`);
}
const identity = { name: player.name as string, guid: player.guid as string };
if (body.event === 'game_start') return identity;
const stats = {} as Omit<TelemetryPlayerStats, keyof TelemetryPlayer>;
for (const key of ['kills', 'headshots', 'downs', 'revives', 'score'] as const) {
if (!Number.isSafeInteger(player[key]) || (player[key] as number) < 0) throw new Error(`players[${index}].${key} must be a nonnegative integer`);
stats[key] = player[key] as number;
}
return { ...identity, ...stats };
});
const common = { server: body.server as string, map: body.map as string };
if (body.event === 'game_start') return { ...common, event: 'game_start', players };
if (!Number.isSafeInteger(body.round) || (body.round as number) < 0) throw new Error('round must be a nonnegative integer');
return { ...common, event: body.event as 'round_change' | 'game_end', round: body.round as number, players: players as TelemetryPlayerStats[] };
}
+11 -16
View File
@@ -2,7 +2,7 @@
import { onMount } from 'svelte';
import Button from '$ui/components/primitives/Button.svelte';
import { builtInPlans, requirement, requirementKinds, requirementLabels, parseBetaPlan, type BetaPlan, type RequirementKind } from '$lib/beta-plan.js';
import { coverageSummary, needsSessionTelemetry, presetDescriptions, requirementCategory, requirementSummary } from '$lib/beta-plan-summary.js';
import { coverageSummary, needsGameTelemetry, presetDescriptions, requirementCategory, requirementSummary } from '$lib/beta-plan-summary.js';
let { plan = $bindable<BetaPlan>() }: { plan: BetaPlan } = $props();
let presets = $state(builtInPlans);
let presetName = $state('');
@@ -102,8 +102,8 @@
<div><dt>If a check fails</dt><dd>{failureSummary}</dd></div>
</dl>
<p class="plan-caption">Every requirement below must pass on each qualifying server. You can edit any check.</p>
{#if plan.requirements.some(needsSessionTelemetry)}
<p class="telemetry-note"><span aria-hidden="true">ⓘ</span> This plan needs session telemetry from the game reporter.</p>
{#if plan.requirements.some(needsGameTelemetry)}
<p class="telemetry-note"><span aria-hidden="true">ⓘ</span> Gameplay progress comes from game start, round change, and game end reports.</p>
{/if}
<div class="requirements">
@@ -120,31 +120,26 @@
<div class="fields">
<label>Description<input bind:value={r.label} maxlength="200" required /></label>
{#if r.kind !== 'manual'}
<label>{['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind) ? 'Maximum allowed' : 'Target'}
<input type="number" min={['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind) ? 0 : ['healthy-minutes', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind) ? 0.01 : 1} step={['healthy-minutes', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind) ? 'any' : 1} bind:value={r.target} required />
<label>{r.kind === 'failure-budget' ? 'Maximum allowed' : 'Target'}
<input type="number" min={r.kind === 'failure-budget' ? 0 : ['healthy-minutes', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind) ? 0.01 : 1} step={['healthy-minutes', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind) ? 'any' : 1} bind:value={r.target} required />
</label>
{/if}
{#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population', 'feature'].includes(r.kind)}
{#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(r.kind)}
<label>Minimum players<input type="number" min="1" step="1" bind:value={r.minPlayers} required /></label>
{/if}
{#if r.kind === 'matches' || r.kind === 'feature'}
<label>Minimum round in the same session<input type="number" min="0" step="1" bind:value={r.minRound} required /></label>
{#if r.kind === 'matches'}
<label>Minimum round in the same game<input type="number" min="0" step="1" bind:value={r.minRound} required /></label>
{/if}
{#if r.kind === 'matches'}
<label>Minimum match duration (minutes)<input type="number" min="0" step="any" bind:value={r.minMinutes} required /></label>
{/if}
{#if r.kind === 'feature'}
<label>Reported event name<input bind:value={r.event} placeholder="perk.purchased" maxlength="120" required /></label>
{/if}
</div>
{#if r.kind === 'round' || r.kind === 'feature' || r.kind === 'restart-budget' || (r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1))}
<p>Requires session telemetry from the game reporter. Progress waits for events from this beta’s current run.</p>
{#if needsGameTelemetry(r)}
<p>Uses game events reported by this server automatically.</p>
{:else if r.kind === 'recovery'}
<p>After the other requirements pass, restart when empty and repeat them. Every cycle requires fresh evidence and checklist sign-off.</p>
{:else if r.kind === 'failure-budget'}
<p>Counts transitions from healthy to unhealthy. Missing or stale monitoring resets continuous health and does not earn playtime.</p>
{:else if r.kind === 'clean-logs'}
<p>Checks new console lines for script errors, script exceptions, unhandled exceptions and fatal errors. An unreadable log cannot pass.</p>
<p>Counts transitions from healthy to unhealthy. Missing or stale monitoring resets continuous healthy time.</p>
{:else if r.kind === 'manual'}
<p>A tester records their name and observation on the beta detail page.</p>
{/if}
@@ -44,10 +44,10 @@
{@const requirement = test.plan.requirements.find(r => r.id === progress.id)!}
<div class="requirement" class:failed={progress.failed}>
<div><strong>{progress.passed ? '✓' : progress.failed ? '✕' : '○'} {progress.label}</strong><span>{progress.detail}</span></div>
{#if ['round', 'matches', 'feature', 'populated-minutes', 'player-hours', 'sustained-population'].includes(requirement.kind)}
<span>Minimum {requirement.minPlayers} player(s){requirement.minRound ? `, round ${requirement.minRound} in the same session` : ''}{requirement.minMinutes ? `, ${requirement.minMinutes} minutes per match` : ''}{requirement.event ? ` · Event: ${requirement.event}` : ''}</span>
{#if ['round', 'matches', 'populated-minutes', 'player-hours', 'sustained-population'].includes(requirement.kind)}
<span>Minimum {requirement.minPlayers} player(s){requirement.minRound ? `, round ${requirement.minRound} in the same game` : ''}{requirement.minMinutes ? `, ${requirement.minMinutes} minutes per match` : ''}</span>
{/if}
{#if !['manual', 'failure-budget', 'restart-budget', 'clean-logs'].includes(requirement.kind)}
{#if !['manual', 'failure-budget'].includes(requirement.kind)}
<progress aria-label={progress.label} value={Math.min(progress.current, progress.target)} max={progress.target}></progress>
{/if}
{#if requirement.kind === 'manual' && !progress.passed && !closed}
+2 -2
View File
@@ -8,10 +8,10 @@
&& (!since || Date.parse(report.receivedAt) >= Date.parse(since)));
</script>
{#if report && fresh}
{#if report && report.event === 'round_change' && fresh}
<div class="round" aria-live="polite">
<strong>Round {report.round}</strong>
<span title={report.receivedAt}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago · {report.players} players</span>
<span title={report.receivedAt}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago · {report.players.length} players</span>
</div>
{/if}
+4 -4
View File
@@ -1,8 +1,8 @@
import { writable, readonly } from 'svelte/store';
import { browser } from '$app/environment';
import type { RoundTelemetry } from '$lib/telemetry.js';
import type { GameTelemetry } from '$lib/telemetry.js';
export const telemetryByServer = writable<Map<string, RoundTelemetry>>(new Map());
export const telemetryByServer = writable<Map<string, GameTelemetry>>(new Map());
export interface RosterPlayer {
num: number | null;
@@ -54,9 +54,9 @@ export function startServerStream(): (() => void) | undefined {
if (!browser || source) return;
source = new EventSource('/api/events/servers');
source.addEventListener('server-telemetry', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { entries: RoundTelemetry[]; snapshot?: boolean };
const data = JSON.parse((evt as MessageEvent).data) as { entries: GameTelemetry[]; snapshot?: boolean };
telemetryByServer.update((previous) => {
const next = data.snapshot ? new Map<string, RoundTelemetry>() : new Map(previous);
const next = data.snapshot ? new Map<string, GameTelemetry>() : new Map(previous);
for (const entry of data.entries) next.set(entry.server, entry);
return next;
});