Add heartbeat telemetry support

This commit is contained in:
2026-09-19 21:52:05 -04:00
parent 8ac420fe61
commit ce0321a6e9
14 changed files with 63 additions and 21 deletions
+2 -1
View File
@@ -66,7 +66,7 @@ See [Plugin releases](docs/plugin-releases.md) for the import and beta workflow.
Betas support configurable gameplay, participation, stability and manual-review
requirements, saved presets, coverage rules and staged rollout. See
[Beta test plans](docs/beta-test-plans.md) for usage and the three-event game telemetry contract.
[Beta test plans](docs/beta-test-plans.md) for usage and the four-event game telemetry contract.
File changes are staged sequentially. Their activation uses rolling restarts:
by default only **one server across the fleet** can restart or recover at a
@@ -247,6 +247,7 @@ The server script sends JSON to `POST /api/telemetry`:
- `game_start`: server, map, and all players (name and GUID).
- `round_change`: server, map, round, and all players with kills, headshots, downs, revives, and score.
- `heartbeat`: the same fields as round change, sent periodically during a round to refresh player stats.
- `game_end`: the same fields as round change, with final stats.
These reports work with or without an active beta. Electric Cherry records
+4 -3
View File
@@ -7,12 +7,13 @@ beta on that server.
## Game reporter protocol
Send JSON to `POST /api/telemetry` with `Content-Type: application/json`.
There are exactly three events. Send them whether or not a beta is active.
There are four events. Send them whether or not a beta is active.
| Event | Fields |
| --- | --- |
| `game_start` | `server`, `map`, `players` with each player's `name` and `guid` |
| `round_change` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
| `heartbeat` | Same fields as `round_change`; sent periodically during a round to refresh the current round and player stats |
| `game_end` | `server`, `map`, `round`, `players` with each player's `name`, `guid`, `kills`, `headshots`, `downs`, `revives`, and `score` |
`server` is the exact registered Electric Cherry server name. `map` is the
@@ -92,7 +93,7 @@ of beta activity. That page documents cumulative stat accounting, round credit,
and the limits of identifying matches from this protocol.
The game scripts are uploaded artifacts, not source maintained in this
repository. Their reporter should emit the three payloads above.
repository. Their reporter should emit the four payloads above.
### Telemetry diagnostics
@@ -138,7 +139,7 @@ Requirements on a qualifying server are ANDed:
| Requirement | Evidence |
| --- | --- |
| Reach round X | A round change or game end reports round >= X with the required player count |
| Reach round X | A round change, heartbeat, or game end reports round >= X with the required player count |
| Complete qualifying matches | Game end, optionally qualified by round, player count, and duration |
| Populated minutes | Time between consecutive game reports with enough players at both ends |
| Player-hours | Populated minutes multiplied by the lower reported player count, divided by 60 |
+3 -3
View File
@@ -17,7 +17,7 @@ export function betaWaitingCondition(requirement: BetaRequirement, observation?:
if (requirement.kind === 'matches') return requirement.minMinutes && !observation.matchStarted
? 'This match has no game-start report, so its duration cannot qualify. Waiting for a new game start and game end.'
: `Waiting for game end${requirement.minRound ? ` after reaching round ${requirement.minRound}` : ''} with ${players}${requirement.minMinutes ? ` and at least ${requirement.minMinutes} minutes since game start` : ''}.`;
if (requirement.kind === 'round') return `Waiting for a round change or game end reporting round ${requirement.target} or higher with ${players}.`;
if (requirement.kind === 'round') return `Waiting for a round change, heartbeat, or game end reporting round ${requirement.target} or higher with ${players}.`;
return `Time is credited when the next report arrives, with ${players} at both ends of the interval. It does not advance continuously between reports.`;
}
return 'Waiting for qualifying evidence.';
@@ -27,8 +27,8 @@ export function betaEvidenceSummary(event: BetaTestView['evidence'][number]): st
let payload: Record<string, unknown>;
try { payload = JSON.parse(event.payload); } catch { return event.kind.replaceAll('-', ' ').replaceAll('_', ' '); }
if (!payload || typeof payload !== 'object') return event.kind;
if (['game_start', 'round_change', 'game_end'].includes(event.kind)) {
const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : 'Round changed';
if (['game_start', 'round_change', 'heartbeat', 'game_end'].includes(event.kind)) {
const label = event.kind === 'game_start' ? 'Game started' : event.kind === 'game_end' ? 'Game ended' : event.kind === 'heartbeat' ? 'Heartbeat' : 'Round changed';
return `${label}${typeof payload.round === 'number' ? ` at round ${payload.round}` : ''}${Array.isArray(payload.players) ? ` with ${payload.players.length} players` : ''}${typeof payload.map === 'string' ? ` on ${payload.map}` : ''}.`;
}
if (event.kind === 'manual-sign-off') return `${payload.requirement ?? 'Checklist'}: ${payload.note ?? 'Signed off'}`;
+1 -1
View File
@@ -64,7 +64,7 @@ export function fleetIdentity(server: FleetServer) {
}
export function currentRound(server: FleetServer, report: GameTelemetry | undefined, now: number): number | null {
if (!report || report.event !== 'round_change' || !freshServer(server, now) || !server.active_since || !server.game_started_at
if (!report || (report.event !== 'round_change' && report.event !== 'heartbeat') || !freshServer(server, now) || !server.active_since || !server.game_started_at
|| server.game_ended_at || (server.last_players ?? 0) === 0) return null;
const received = Date.parse(report.receivedAt);
const age = now - received;
+1 -1
View File
@@ -75,7 +75,7 @@ export function testView(betaId: number): BetaTestView | null {
servers: runs.map(run => {
const metrics = readMetrics(run);
const progress = evaluateRequirements(plan, metrics, run.phase);
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const event = latest ? JSON.parse(latest.payload) : null;
return { server: run.server, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
+1 -1
View File
@@ -60,7 +60,7 @@ export function recordLeaderboardTelemetry(handle: Database.Database, report: Ga
const newMatch = !match || match.map_key !== identity.key
|| (isStart && (match.status !== 'active' || match.last_event !== 'game_start'))
|| (!isStart && round < match.highest_round)
|| (report.event === 'round_change' && match.status !== 'active');
|| ((report.event === 'round_change' || report.event === 'heartbeat') && match.status !== 'active');
if (newMatch) {
if (match?.status === 'active') {
+4 -4
View File
@@ -13,16 +13,16 @@ export interface TelemetryPlayerStats extends TelemetryPlayer {
export type GameEvent = { server: string; map: string } & (
| { event: 'game_start'; players: TelemetryPlayer[] }
| { event: 'round_change' | 'game_end'; round: number; players: TelemetryPlayerStats[] }
| { event: 'round_change' | 'heartbeat' | 'game_end'; round: number; players: TelemetryPlayerStats[] }
);
export type GameTelemetry = GameEvent & { receivedAt: string };
/** Validate the three script payloads; beta state is never part of the protocol. */
/** Validate the four script payloads; beta state is never part of the protocol. */
export function parseGameEvent(value: unknown): GameEvent {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Payload must be a JSON object');
const body = value as Record<string, unknown>;
if (!['game_start', 'round_change', 'game_end'].includes(String(body.event))) throw new Error('event must be game_start, round_change, or game_end');
if (!['game_start', 'round_change', 'heartbeat', 'game_end'].includes(String(body.event))) throw new Error('event must be game_start, round_change, heartbeat, or game_end');
for (const key of ['server', 'map'] as const) {
if (typeof body[key] !== 'string' || !body[key].trim() || body[key].length > 255) throw new Error(`${key} must be a nonempty string (up to 255 characters)`);
}
@@ -48,5 +48,5 @@ export function parseGameEvent(value: unknown): GameEvent {
const common = { server: body.server as string, map: body.map as string };
if (body.event === 'game_start') return { ...common, event: 'game_start', players };
if (!Number.isSafeInteger(body.round) || (body.round as number) < 0) throw new Error('round must be a nonnegative integer');
return { ...common, event: body.event as 'round_change' | 'game_end', round: body.round as number, players: players as TelemetryPlayerStats[] };
return { ...common, event: body.event as 'round_change' | 'heartbeat' | 'game_end', round: body.round as number, players: players as TelemetryPlayerStats[] };
}
@@ -17,7 +17,7 @@
const expired = $derived(test.outcome === 'inconclusive' || (!!deadline && test.outcome !== 'passed' && Date.parse(deadline) <= Date.now()));
const ranked = $derived([...test.servers].sort((a,b) => (a.progress.some(r => r.failed) ? 0 : a.passed ? 2 : 1) - (b.progress.some(r => r.failed) ? 0 : b.passed ? 2 : 1) || a.server.localeCompare(b.server)));
const filteredEvidence = $derived(test.evidence.filter(event => (!evidenceServer || event.server === evidenceServer)
&& (!evidenceKind || (evidenceKind === 'gameplay' ? ['game_start','round_change','game_end'].includes(event.kind) : evidenceKind === 'health' ? ['health','status-sample'].includes(event.kind) : event.kind === evidenceKind))
&& (!evidenceKind || (evidenceKind === 'gameplay' ? ['game_start','round_change','heartbeat','game_end'].includes(event.kind) : evidenceKind === 'health' ? ['health','status-sample'].includes(event.kind) : event.kind === evidenceKind))
&& (!evidenceQuery || `${betaEvidenceSummary(event)} ${event.payload}`.toLowerCase().includes(evidenceQuery.toLowerCase()))));
$effect(() => { evidenceServer; evidenceKind; evidenceQuery; evidenceLimit = 25; });
function reportAge(at:string) { const minutes=Math.max(0,Math.floor((Date.now()-Date.parse(at))/60000));return minutes<1?'less than a minute ago':minutes<60?`${minutes}m ago`:`${Math.floor(minutes/60)}h ${minutes%60}m ago`; }
@@ -94,7 +94,7 @@
</div>
{/if}
<div class="scoreboard-footer">
<p class="note">{report.event === 'game_start' ? 'Player stats arrive with the next round report.' : report.event === 'game_end' ? 'Final stats from this game. The scoreboard resets when the next game starts.' : 'Stats update at each round change and game end.'}</p>
<p class="note">{report.event === 'game_start' ? 'Player stats arrive with the next round report or heartbeat.' : report.event === 'game_end' ? 'Final stats from this game. The scoreboard resets when the next game starts.' : 'Stats update at each round change, heartbeat, and game end.'}</p>
<a href="/leaderboard">View lifetime leaderboard →</a>
</div>
{/if}
+1 -1
View File
@@ -8,7 +8,7 @@
&& (!since || Date.parse(report.receivedAt) >= Date.parse(since)));
</script>
{#if report && report.event === 'round_change' && fresh}
{#if report && (report.event === 'round_change' || report.event === 'heartbeat') && fresh}
<div class="round" aria-live="polite">
<strong>Round {report.round}</strong>
<span title={report.receivedAt}>Last reported {elapsedSince(report.receivedAt, $uiNow)} ago · {report.players.length} players</span>
+12
View File
@@ -6,6 +6,18 @@ import { gameEvent } from '../helpers/telemetry.js';
const planWith = (...kinds: Parameters<typeof requirement>[0][]): BetaPlan => ({ ...defaultPlan(), requirements: kinds.map((k, i) => requirement(k, `r${i}`)) });
describe('beta requirement evaluation', () => {
it('credits heartbeat participation during a long round without completing a match', () => {
const plan = planWith('populated-minutes', 'round', 'matches');
const m = emptyMetrics();
applyGameEvent(plan, m, gameEvent('game_start'), 0);
applyGameEvent(plan, m, gameEvent('round_change', 30), 60_000);
applyGameEvent(plan, m, gameEvent('heartbeat', 30), 120_000);
applyGameEvent(plan, m, gameEvent('heartbeat', 30), 180_000);
expect(m.values).toMatchObject({ r0: 3, r1: 30 });
expect(m.values.r2 ?? 0).toBe(0);
expect(m.match).toMatchObject({ started: 0, lastEvent: 180_000, ended: false });
});
it('copies reactive presets without mutating saved requirements', () => {
const source = defaultPlan();
const selected = parseBetaPlan(new Proxy(source, {}));
+2 -2
View File
@@ -42,8 +42,8 @@ describe('Fleet discovery and match freshness', () => {
expect(matchesFleet(server, '', 'attention', '', [], [{ ...operation, state: 'attention' }], now)).toBe(true);
expect(matchesFleet({ ...server, health: 'starting' }, '', 'attention', '', [], [], now)).toBe(false);
});
it('only spotlights fresh telemetry from the current occupied match', () => {
const report = { ...gameEvent('round_change', 38, 4, server.name), receivedAt: iso(-1000) };
it.each(['round_change', 'heartbeat'] as const)('only spotlights fresh %s telemetry from the current occupied match', (event) => {
const report = { ...gameEvent(event, 38, 4, server.name), receivedAt: iso(-1000) };
expect(currentRound(server, report, now)).toBe(38);
for (const change of [{ receivedAt: iso(-121_000) }, { receivedAt: iso(-61_000) }, { receivedAt: iso(60_000) }]) {
expect(currentRound(server, { ...report, ...change }, now)).toBeNull();
+12
View File
@@ -43,6 +43,18 @@ async function records(guid = '2741626') {
}>;
}
it('refreshes stats during a round without adding rounds or completing the match', async () => {
await report('game_start');
await report('round_change', 30, [player(undefined, { kills: 100 })]);
await report('heartbeat', 30, [player(undefined, { kills: 150 })]);
await report('heartbeat', 30, [player(undefined, { kills: 150 })]);
expect(await matches()).toEqual([expect.objectContaining({ status: 'active', highest_round: 30 })]);
expect(await records()).toEqual([expect.objectContaining({ kills: 150, rounds_survived: 29, completed: 0 })]);
await report('game_end', 30);
await report('heartbeat', 30);
expect((await matches()).map(match => match.status)).toEqual(['completed', 'active']);
});
it('persists independent stat high-water marks without adding cumulative snapshots or retries', async () => {
await report('game_start');
await report('round_change', 1);
+18 -2
View File
@@ -38,7 +38,8 @@ it('timestamps initial telemetry and subsequent status and telemetry events with
}
});
it('persists full player stats, broadcasts updates, and leaves RCON safety data untouched', async () => {
it.each(['round_change', 'heartbeat'] as const)('persists %s stats, broadcasts updates, and leaves RCON safety data untouched', async (event) => {
const payload = gameEvent(event, 17);
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const listener = vi.fn();
const unsubscribe = subscribe(listener);
@@ -62,7 +63,7 @@ it('persists full player stats, broadcasts updates, and leaves RCON safety data
await reader.cancel();
});
it('accepts all three events without an active beta and updates match lifecycle', async () => {
it('accepts all four events without an active beta and updates match lifecycle', async () => {
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
const start = gameEvent('game_start');
@@ -71,6 +72,12 @@ it('accepts all three events without an active beta and updates match lifecycle'
expect(getServer(start.server)?.game_started_at).toBeTruthy();
expect(getServer(start.server)?.game_ended_at).toBeNull();
expect((await post(payload)).status).toBe(200);
const startedAt = getServer(start.server)?.game_started_at;
const heartbeat = gameEvent('heartbeat', 17);
expect((await post(heartbeat)).status).toBe(200);
expect(listTelemetry()[0]).toMatchObject(heartbeat);
expect(getServer(start.server)?.game_started_at).toBe(startedAt);
expect(getServer(start.server)?.game_ended_at).toBeNull();
const end = gameEvent('game_end', 18);
expect((await post(end)).status).toBe(200);
expect(getServer(start.server)?.game_ended_at).toBeTruthy();
@@ -93,6 +100,15 @@ it.each([null, [], {}, ...['boot', 'session_start', 'match_end', 'feature', 'oth
expect((await post(body)).status).toBe(400);
});
it.each([
{ ...gameEvent('heartbeat'), round: undefined },
{ ...gameEvent('heartbeat'), round: -1 },
{ ...gameEvent('heartbeat'), players: [{ name: 'Player', guid: '123' }] },
{ ...gameEvent('heartbeat'), players: [{ ...roster(1)[0], kills: -1 }] }
])('requires round and full stats for heartbeats: %j', async (body) => {
expect((await post(body)).status).toBe(400);
});
it('rejects unknown and destroyed servers and non-JSON requests', async () => {
expect((await post({ ...payload, server: 'unknown' })).status).toBe(404);
expect((await post(payload, 'text/plain')).status).toBe(415);