Add per-server telemetry request diagnostics

This commit is contained in:
2026-09-19 18:59:30 -04:00
parent a182aeb950
commit 41891b4209
10 changed files with 614 additions and 14 deletions
+8
View File
@@ -244,6 +244,14 @@ See [Game reporter protocol](docs/beta-test-plans.md#game-reporter-protocol) for
complete payload examples. Telemetry does not replace RCON's empty-server
check before an automated restart.
For a missing scoreboard, open the server's **Diagnostics → Telemetry API requests**.
The log shows received requests for that exact server name, including accepted
reports, rejected payloads, HTTP status, errors, and expandable submitted JSON.
It refreshes every five seconds while visible and survives app restarts. Logging
starts with this version; older requests cannot be reconstructed. See
[Telemetry diagnostics](docs/beta-test-plans.md#telemetry-diagnostics) for retention
and how to interpret missing or rejected reports.
The **Leaderboard** records persistent player totals and map records from these
same reports, including highest round, cumulative kills, rounds survived,
headshots, revives, downs, and games played. See [Leaderboard](docs/leaderboard.md)
+38
View File
@@ -94,6 +94,44 @@ and the limits of identifying matches from this protocol.
The game scripts are uploaded artifacts, not source maintained in this
repository. Their reporter should emit the three payloads above.
### Telemetry diagnostics
Open a server's **Diagnostics → Telemetry API requests** to inspect incoming
`POST /api/telemetry` calls whose top-level `server` exactly matches its registered
name. Each entry includes the Electric Cherry receipt time, HTTP result, error
reason, event, map, round, player count, content type, and submitted JSON. Rejected
reports are included, even when the content type is wrong but the body is readable
JSON. Processing failures return 500 and record the internal error when diagnostic
storage is available; failed database transactions do not update the scoreboard.
The panel refreshes every five seconds while visible and has a manual refresh
button. History is stored in SQLite starting with this version, with up to 200
requests per server from the last seven days and a fleet-wide cap of 10,000
requests. Old entries are pruned when new requests arrive, and expired entries
are excluded from reads. Payload previews are limited to 16,384 characters and
marked when truncated. Request headers other than content type are not stored.
`GET /api/servers/{name}/telemetry-requests` returns the same recent history,
newest first, with caching disabled.
- **No entries:** no attributable requests are retained for this exact name.
Check the reporter's destination and server field. Calls that never reach
Electric Cherry, malformed JSON, and bodies without a usable server name
cannot appear in this server's log. Earlier requests are not backfilled.
- **400 / 415:** inspect the error and payload. One invalid player's stats or a
duplicate GUID rejects the entire report. These requests do not update the
scoreboard.
- **404:** the reported server was not registered or had been destroyed at the
time of the request.
- **200:** Electric Cherry processed the report. An empty `players` array
produces an empty scoreboard, including when sent with `game_end`. If a
populated accepted report is missing from the scoreboard, investigate the
live update/browser path.
- **500:** Electric Cherry failed to process the request. Inspect the recorded
error and the application log.
RCON's online status and player count are separate from gameplay telemetry, so
an occupied server can still have no scoreboard report.
## Requirements
Requirements on a qualifying server are ANDed:
+18
View File
@@ -367,6 +367,24 @@ CREATE TABLE IF NOT EXISTS server_telemetry (
received_at TEXT NOT NULL,
payload TEXT
);
-- Request diagnostics survive registry removal and also capture unknown names.
-- Bounded independently of the latest scoreboard and permanent leaderboard.
CREATE TABLE IF NOT EXISTS telemetry_requests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server TEXT NOT NULL,
received_at TEXT NOT NULL,
status INTEGER NOT NULL,
error TEXT,
event TEXT,
map TEXT,
round INTEGER,
player_count INTEGER,
content_type TEXT,
payload TEXT NOT NULL,
payload_truncated INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_server ON telemetry_requests(server, id DESC);
CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests(received_at);
-- Lifecycle history deliberately survives removal of the server registry row.
CREATE TABLE IF NOT EXISTS server_deployments (
id TEXT PRIMARY KEY,
@@ -0,0 +1,50 @@
import { db } from '../db/index.js';
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '../../telemetry-diagnostics.js';
const MAX_PAYLOAD_LENGTH = 16 * 1024;
// Bound storage even when requests claim many different, unregistered names.
const MAX_REQUESTS = 10_000;
const cutoff = () => new Date(Date.now() - TELEMETRY_REQUEST_RETENTION.maxAgeDays * 86_400_000).toISOString();
const textField = (value: unknown) => typeof value === 'string' ? value.slice(0, 255) : null;
export function recordTelemetryRequest(input: {
value: unknown;
receivedAt: string;
status: number;
error: string | null;
contentType: string | null;
}): void {
// Use only the top-level JSON field, never a player's name or a substring.
// Malformed JSON cannot be reliably attributed to an individual server.
if (!input.value || typeof input.value !== 'object' || Array.isArray(input.value)) return;
const body = input.value as Record<string, unknown>;
if (typeof body.server !== 'string' || !body.server.trim() || body.server.length > 255) return;
const payload = JSON.stringify(body);
const handle = db();
handle.transaction(() => {
handle.prepare(`INSERT INTO telemetry_requests
(server, received_at, status, error, event, map, round, player_count, content_type, payload, payload_truncated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
body.server, input.receivedAt, input.status, input.error?.slice(0, 2000) ?? null,
textField(body.event), textField(body.map),
Number.isSafeInteger(body.round) ? body.round as number : null,
Array.isArray(body.players) ? body.players.length : null,
textField(input.contentType), payload.slice(0, MAX_PAYLOAD_LENGTH), Number(payload.length > MAX_PAYLOAD_LENGTH)
);
handle.prepare('DELETE FROM telemetry_requests WHERE received_at < ?').run(cutoff());
handle.prepare(`DELETE FROM telemetry_requests WHERE server = ? AND id <= (
SELECT id FROM telemetry_requests WHERE server = ? ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(body.server, body.server, TELEMETRY_REQUEST_RETENTION.maxPerServer);
handle.prepare(`DELETE FROM telemetry_requests WHERE id <= (
SELECT id FROM telemetry_requests ORDER BY id DESC LIMIT 1 OFFSET ?
)`).run(MAX_REQUESTS);
})();
}
export function listTelemetryRequests(server: string): TelemetryRequestLogEntry[] {
const rows = db().prepare(`SELECT id, server, received_at AS receivedAt, status, error, event, map, round,
player_count AS playerCount, content_type AS contentType, payload, payload_truncated AS payloadTruncated
FROM telemetry_requests WHERE server = ? AND received_at >= ? ORDER BY id DESC LIMIT ?`)
.all(server, cutoff(), TELEMETRY_REQUEST_RETENTION.maxPerServer) as Array<Omit<TelemetryRequestLogEntry, 'payloadTruncated'> & { payloadTruncated: number }>;
return rows.map(row => ({ ...row, payloadTruncated: !!row.payloadTruncated }));
}
+17
View File
@@ -0,0 +1,17 @@
/** A received POST /api/telemetry request, including reports rejected by validation. */
export interface TelemetryRequestLogEntry {
id: number;
server: string;
receivedAt: string;
status: number;
error: string | null;
event: string | null;
map: string | null;
round: number | null;
playerCount: number | null;
contentType: string | null;
payload: string;
payloadTruncated: boolean;
}
export const TELEMETRY_REQUEST_RETENTION = { maxPerServer: 200, maxAgeDays: 7 } as const;
@@ -0,0 +1,198 @@
<script lang="ts">
import { onMount } from 'svelte';
import { TELEMETRY_REQUEST_RETENTION, type TelemetryRequestLogEntry } from '$lib/telemetry-diagnostics.js';
import { pollVisible } from '$ui/poll.js';
import Card from './primitives/Card.svelte';
import Badge from './primitives/Badge.svelte';
import Button from './primitives/Button.svelte';
import EmptyState from './primitives/EmptyState.svelte';
import ListFooter from './primitives/ListFooter.svelte';
let { server }: { server: string } = $props();
let requests = $state<TelemetryRequestLogEntry[]>([]);
let retention = $state<{ maxPerServer: number; maxAgeDays: number }>({ ...TELEMETRY_REQUEST_RETENTION });
let refreshing = $state(false);
let loaded = $state(false);
let error = $state('');
let refreshedAt = $state<string | null>(null);
let limit = $state(25);
let active = false;
let controller: AbortController | null = null;
async function refresh(): Promise<void> {
if (!active || refreshing) return;
const currentServer = server;
const requestController = new AbortController();
controller = requestController;
refreshing = true;
try {
const response = await fetch(`/api/servers/${encodeURIComponent(currentServer)}/telemetry-requests`, {
cache: 'no-store',
signal: requestController.signal
});
const body = await response.json().catch(() => null);
if (!response.ok || !body?.ok || !Array.isArray(body.requests)) {
throw new Error(body?.error || `Could not load telemetry requests (HTTP ${response.status}).`);
}
if (!active || server !== currentServer) return;
requests = body.requests;
retention = body.retention;
loaded = true;
error = '';
refreshedAt = new Date().toISOString();
} catch (cause) {
if (active && server === currentServer && !requestController.signal.aborted) {
error = cause instanceof Error ? cause.message : 'Could not load telemetry requests.';
}
} finally {
if (active && server === currentServer) refreshing = false;
if (controller === requestController) controller = null;
}
}
onMount(() => {
active = true;
const stopPolling = pollVisible(refresh, 5000);
return () => {
active = false;
stopPolling();
controller?.abort();
};
});
function timestamp(value: string): string {
return new Date(value).toLocaleString(undefined, {
year: 'numeric', month: 'short', day: 'numeric',
hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'short'
});
}
function outcome(status: number): string {
return status >= 500 ? 'Failed' : status >= 400 ? 'Rejected' : 'Accepted';
}
</script>
<Card padding="lg" as="section" ariaLabel="Telemetry API requests">
<div class="panel-head">
<div>
<h2>Telemetry API requests</h2>
<p class="server-name"><code>{server}</code></p>
</div>
<Button size="sm" onclick={refresh} disabled={refreshing} loading={refreshing}>Refresh</Button>
</div>
<p class="intro">
Incoming <code>POST /api/telemetry</code> requests with this exact server name.
The scoreboard uses telemetry; the fleet player count comes from RCON separately.
</p>
<p class="retention">
Newest first · up to {retention.maxPerServer} requests from the last {retention.maxAgeDays} days.
Refreshes every 5 seconds while visible.
{#if refreshedAt}<span>Last refreshed <time datetime={refreshedAt}>{timestamp(refreshedAt)}</time>.</span>{/if}
</p>
{#if error}
<p class="load-error" role="alert">
{error}{#if loaded} Previously loaded requests are shown below.{/if}
</p>
{/if}
{#if !loaded && !error}
<EmptyState title="Loading telemetry requests…" />
{:else if loaded && requests.length === 0}
<EmptyState
title="No telemetry requests recorded"
description="No matching requests are retained for this server. An empty log alone does not establish a network problem."
/>
{:else if requests.length > 0}
<ol class="request-list" aria-label="Recent telemetry requests">
{#each requests.slice(0, limit) as request (request.id)}
<li class="request">
<div class="request-head">
<div class="request-result">
<Badge variant={request.status >= 500 ? 'danger' : request.status >= 400 ? 'warning' : 'online'}>
{outcome(request.status)} · HTTP {request.status}
</Badge>
<code class="event">{request.event ?? 'Event not provided'}</code>
</div>
<time class="received" datetime={request.receivedAt}>{timestamp(request.receivedAt)}</time>
</div>
<dl class="request-meta">
<div><dt>Map</dt><dd><code>{request.map ?? 'Not provided'}</code></dd></div>
<div><dt>Round</dt><dd>{request.round ?? 'Not provided'}</dd></div>
<div><dt>Players</dt><dd>{request.playerCount ?? 'Not provided'}</dd></div>
</dl>
{#if request.error}<p class="request-error">{request.error}</p>{/if}
<details class="payload">
<summary>Request payload{#if request.payloadTruncated} (truncated){/if}</summary>
<p class="content-type">Content type: <code>{request.contentType ?? 'Not provided'}</code></p>
<pre>{request.payload}</pre>
{#if request.payloadTruncated}<p class="truncation">Only the beginning of this payload was retained.</p>{/if}
</details>
</li>
{/each}
</ol>
<ListFooter total={requests.length} bind:limit label="requests" />
{/if}
<p class="attribution">
Logging starts after this update is installed.
Requests can be linked here only when readable JSON contains a matching top-level <code>server</code> field.
Calls that never reach Electric Cherry, malformed JSON, and different server names will not appear here.
</p>
</Card>
<style>
.panel-head, .request-head, .request-result {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--space-3);
flex-wrap: wrap;
}
.panel-head { margin-bottom: var(--space-4); }
h2 { margin: 0; font-size: var(--text-md); font-weight: var(--weight-semibold); }
p { margin: 0; }
code, pre { font-family: var(--font-mono); }
.server-name { margin-top: var(--space-2); overflow-wrap: anywhere; color: var(--text-secondary); font-size: var(--text-xs); }
.intro { font-size: var(--text-sm); color: var(--text-secondary); line-height: 1.6; }
.retention, .attribution, .content-type, .truncation {
font-size: var(--text-xs);
color: var(--text-tertiary);
line-height: 1.6;
overflow-wrap: anywhere;
}
.retention { margin-top: var(--space-2); }
.retention span { display: block; }
.attribution { margin-top: var(--space-4); }
.request-list { list-style: none; padding: 0; margin: var(--space-5) 0 0; }
.request { padding: var(--space-4) 0; border-top: 1px solid var(--border-subtle); }
.request-result { justify-content: flex-start; min-width: 0; }
.event { font-size: var(--text-sm); overflow-wrap: anywhere; }
.received { color: var(--text-tertiary); font-size: var(--text-xs); font-variant-numeric: tabular-nums; }
.request-meta { display: flex; flex-wrap: wrap; gap: var(--space-3) var(--space-6); margin: var(--space-3) 0; font-size: var(--text-xs); }
.request-meta div { display: flex; gap: var(--space-2); min-width: 0; }
dt { color: var(--text-tertiary); }
dd { margin: 0; color: var(--text-secondary); overflow-wrap: anywhere; }
.load-error, .request-error { color: var(--state-danger); font-size: var(--text-sm); overflow-wrap: anywhere; }
.load-error { margin-top: var(--space-4); }
.request-error { margin-bottom: var(--space-3); }
.payload { font-size: var(--text-xs); }
summary { color: var(--text-secondary); cursor: pointer; width: fit-content; }
summary:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-sm); }
.content-type { margin-top: var(--space-3); }
pre {
margin: var(--space-2) 0 0;
padding: var(--space-3);
max-height: 24rem;
overflow: auto;
white-space: pre-wrap;
overflow-wrap: anywhere;
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-sm);
color: var(--text-secondary);
line-height: 1.6;
}
.truncation { margin-top: var(--space-2); }
</style>
@@ -0,0 +1,11 @@
import { json } from '@sveltejs/kit';
import { getServer } from '$lib/server/fleet/servers.js';
import { listTelemetryRequests } from '$lib/server/fleet/telemetry-requests.js';
import { TELEMETRY_REQUEST_RETENTION } from '$lib/telemetry-diagnostics.js';
export async function GET({ params }: { params: { name: string } }) {
if (!getServer(params.name)) return json({ ok: false, error: 'no such server' }, { status: 404 });
return json({ ok: true, requests: listTelemetryRequests(params.name), retention: TELEMETRY_REQUEST_RETENTION }, {
headers: { 'Cache-Control': 'no-store' }
});
}
+32 -7
View File
@@ -2,26 +2,51 @@ import { json } from '@sveltejs/kit';
import { parseGameEvent, type GameTelemetry } from '$lib/telemetry.js';
import { getServer } from '$lib/server/fleet/servers.js';
import { saveTelemetry } from '$lib/server/fleet/telemetry.js';
import { recordTelemetryRequest } from '$lib/server/fleet/telemetry-requests.js';
import { recordGameEvent } from '$lib/server/fleet/beta-tests.js';
import { db } from '$lib/server/db/index.js';
import { emit } from '$lib/server/pipeline/bus.js';
export async function POST({ request }: { request: Request }) {
if (request.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !== 'application/json') {
return json({ ok: false, error: 'Content-Type must be application/json' }, { status: 415 });
}
const receivedAt = new Date().toISOString();
const contentType = request.headers.get('content-type');
let value: unknown;
let malformed = false;
try { value = await request.json(); }
catch { return json({ ok: false, error: 'Malformed JSON' }, { status: 400 }); }
catch { malformed = true; }
function log(status: number, error: string | null): void {
try { recordTelemetryRequest({ value, receivedAt, status, error, contentType }); }
catch (error) {
// Diagnostics must not turn a saved report into a failed request.
console.error('Could not record telemetry request diagnostics', error);
}
}
function reject(status: number, error: string): Response {
log(status, error);
return json({ ok: false, error }, { status });
}
// Even a wrong content type may carry readable JSON with a server name.
if (contentType?.split(';')[0].trim().toLowerCase() !== 'application/json') {
return reject(415, 'Content-Type must be application/json');
}
if (malformed) return reject(400, 'Malformed JSON');
let telemetry: GameTelemetry;
try { telemetry = { ...parseGameEvent(value), receivedAt: new Date().toISOString() }; }
catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
try { telemetry = { ...parseGameEvent(value), receivedAt }; }
catch (error) { return reject(400, (error as Error).message); }
try {
const server = getServer(telemetry.server);
if (!server || server.destroyed_at) return json({ ok: false, error: 'Server is not registered or has been destroyed' }, { status: 404 });
if (!server || server.destroyed_at) return reject(404, 'Server is not registered or has been destroyed');
db().transaction(() => {
saveTelemetry(telemetry);
recordGameEvent(telemetry);
})();
emit({ type: 'server-telemetry', entries: [telemetry] });
} catch (error) {
console.error('Could not process telemetry', error);
log(500, `Could not process telemetry: ${error instanceof Error ? error.message : String(error)}`);
return json({ ok: false, error: 'Could not process telemetry' }, { status: 500 });
}
log(200, null);
return json({ ok: true, telemetry });
}
+9 -1
View File
@@ -1,5 +1,6 @@
<script lang="ts">
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
import { pollVisible } from '$ui/poll.js';
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
@@ -284,7 +285,7 @@
<Tabs {tabs} bind:value={activeTab} ariaLabel="Server sections" idPrefix="server-tabs" panelId="server-panel" />
</div>
<div class="tab-panel" role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
<div class="tab-panel" class:diagnostics={activeTab === 'diagnostics'} role="tabpanel" id="server-panel" aria-labelledby={"server-tabs-" + activeTab}>
{#if tabError}<div class="ui-error" role="alert">{tabError}<Button size="sm" onclick={() => loadTab(name, activeTab)}>Retry</Button></div>{/if}
{#if activeTab === 'scoreboard'}
<Card padding="lg">
@@ -328,6 +329,9 @@
</Card>
<ListFooter total={inventory.length} bind:limit={inventoryLimit} label="files" />
{:else if activeTab === 'diagnostics'}
{#key name}
<TelemetryDiagnostics server={name} />
{/key}
<Card padding="lg">
<div class="panel-head">
<h2>RCON diagnostic</h2>
@@ -575,6 +579,10 @@
}
/* Tabs row */
.tab-panel.diagnostics {
display: grid;
gap: var(--space-6);
}
.tabs-row {
display: flex;
justify-content: center;
+227
View File
@@ -0,0 +1,227 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { gameEvent, postTelemetry as post } from '../helpers/telemetry.js';
const server = 't5_zombie_theater_1';
const otherServer = 't5_zombie_theater_2';
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([server, otherServer]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => {
vi.restoreAllMocks();
await fixture.cleanup();
});
async function diagnostics(name = server) {
const { GET } = await import('../../src/routes/api/servers/[name]/telemetry-requests/+server.js');
return GET({ params: { name } });
}
it('returns uncached, empty diagnostics for a registered server before any requests arrive', async () => {
const response = await diagnostics();
expect(response.status).toBe(200);
expect(response.headers.get('cache-control')).toContain('no-store');
expect(await response.json()).toEqual({
ok: true, requests: [], retention: { maxPerServer: 200, maxAgeDays: 7 }
});
expect((await diagnostics('unknown')).status).toBe(404);
});
it('persists the received payload and response, isolates each server, and lists newest requests first', async () => {
const first = gameEvent('round_change', 1);
const latest = { ...gameEvent('round_change', 3), scriptVersion: 'diagnostic-version' };
expect((await post(first)).status).toBe(200);
expect((await post(gameEvent('round_change', 2, 1, otherServer))).status).toBe(200);
expect((await post(latest)).status).toBe(200);
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
const body = await (await diagnostics()).json();
expect(body.requests).toHaveLength(2);
expect(body.requests[0]).toEqual({
id: expect.any(Number), server, receivedAt: expect.any(String), status: 200, error: null,
event: 'round_change', map: 'zombie_theater', round: 3, playerCount: 4,
contentType: 'application/json', payload: JSON.stringify(latest), payloadTruncated: false
});
expect(body.requests[0].id).toBeGreaterThan(body.requests[1].id);
expect(body.requests[1].payload).toBe(JSON.stringify(first));
expect((await (await diagnostics(otherServer)).json()).requests).toEqual([
expect.objectContaining({ server: otherServer, round: 2, playerCount: 1 })
]);
});
it('records rejected player data with the same validation reason returned to the telemetry client', async () => {
const invalid = { ...gameEvent('round_change', 5), players: [{ name: 'Player', guid: '123' }] };
const response = await post(invalid);
expect(response.status).toBe(400);
const result = await response.json();
expect(result.error).toContain('players[0].kills');
const { requests } = await (await diagnostics()).json();
expect(requests).toEqual([expect.objectContaining({
server, status: 400, error: result.error, event: 'round_change', round: 5,
playerCount: 1, payload: JSON.stringify(invalid), payloadTruncated: false
})]);
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
expect(listTelemetry()).toEqual([]);
});
it('records identifiable JSON sent with an unsupported or missing Content-Type', async () => {
const payload = gameEvent('round_change', 2);
expect((await post(payload, 'text/plain')).status).toBe(415);
const { POST } = await import('../../src/routes/api/telemetry/+server.js');
// A byte body avoids Request automatically assigning text/plain to a string body.
const request = new Request('http://localhost/api/telemetry', {
method: 'POST', body: new TextEncoder().encode(JSON.stringify(payload))
});
expect(request.headers.get('content-type')).toBeNull();
expect((await POST({ request })).status).toBe(415);
const { requests } = await (await diagnostics()).json();
expect(requests).toEqual([
expect.objectContaining({ status: 415, contentType: null, error: 'Content-Type must be application/json' }),
expect.objectContaining({ status: 415, contentType: 'text/plain', payload: JSON.stringify(payload) })
]);
});
it('distinguishes a zero round and an empty scoreboard from absent or invalid fields', async () => {
await post(gameEvent('round_change', 0, 0));
await post(gameEvent('game_start', 0, 0));
await post({ server, players: 4 });
const { requests } = await (await diagnostics()).json();
expect(requests).toEqual([
expect.objectContaining({ status: 400, event: null, map: null, round: null, playerCount: null }),
expect.objectContaining({ status: 200, event: 'game_start', round: null, playerCount: 0 }),
expect.objectContaining({ status: 200, event: 'round_change', round: 0, playerCount: 0 })
]);
});
it('keeps unknown and destroyed server rejections and matches the submitted server name exactly', async () => {
const unknown = 't5_zombie_theater_99';
const spaced = ` ${server} `;
expect((await post(gameEvent('game_start', 0, 0, unknown))).status).toBe(404);
expect((await post(gameEvent('game_start', 0, 0, spaced))).status).toBe(404);
await post(gameEvent('game_start'));
const { setServerField } = await import('../../src/lib/server/fleet/servers.js');
setServerField(server, 'destroyed_at', new Date().toISOString());
expect((await post(gameEvent('game_end', 1))).status).toBe(404);
const { listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js');
expect(listTelemetryRequests(unknown)).toEqual([expect.objectContaining({ server: unknown, status: 404 })]);
expect(listTelemetryRequests(spaced)).toEqual([expect.objectContaining({ server: spaced, status: 404 })]);
expect((await diagnostics(unknown)).status).toBe(404);
const response = await diagnostics();
expect(response.status).toBe(200);
expect((await response.json()).requests).toEqual([
expect.objectContaining({ server, status: 404, error: 'Server is not registered or has been destroyed' }),
expect.objectContaining({ server, status: 200 })
]);
});
it('does not attribute malformed JSON or unusable server names to a server', async () => {
const { POST } = await import('../../src/routes/api/telemetry/+server.js');
const malformed = new Request('http://localhost/api/telemetry', {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: `{"server":"${server}",`
});
expect((await POST({ request: malformed })).status).toBe(400);
for (const value of [undefined, null, '', ' ', 123, 's'.repeat(256)]) {
expect((await post({ ...gameEvent('game_start'), server: value })).status).toBe(400);
}
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 0 });
});
it('bounds stored payloads and marks truncation without rejecting a valid event', async () => {
const payload = { ...gameEvent('round_change', 4), diagnostic: 'x'.repeat(20_000) };
expect((await post(payload)).status).toBe(200);
const { requests } = await (await diagnostics()).json();
expect(requests[0]).toMatchObject({ status: 200, round: 4, playerCount: 4, payloadTruncated: true });
expect(requests[0].payload).toBe(JSON.stringify(payload).slice(0, 16 * 1024));
});
it('records processing failures after rollback without changing the scoreboard, match state, or broadcasts', async () => {
const first = gameEvent('round_change', 3);
await post(first);
const { db } = await import('../../src/lib/server/db/index.js');
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const beta = await import('../../src/lib/server/fleet/beta-tests.js');
const snapshot = listTelemetry();
const matches = db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all();
const listener = vi.fn();
const unsubscribe = subscribe(listener);
vi.spyOn(beta, 'recordGameEvent').mockImplementationOnce(() => { throw new Error('Injected beta processing failure'); });
const response = await post(gameEvent('game_end', 99));
unsubscribe();
expect(response.status).toBe(500);
expect(await response.json()).toEqual({ ok: false, error: 'Could not process telemetry' });
expect(listTelemetry()).toEqual(snapshot);
expect(getServer(server)?.game_ended_at).toBeNull();
expect(db().prepare('SELECT * FROM leaderboard_matches ORDER BY id').all()).toEqual(matches);
expect(listener).not.toHaveBeenCalled();
expect((await (await diagnostics()).json()).requests).toEqual([
expect.objectContaining({ status: 500, event: 'game_end', round: 99, error: 'Could not process telemetry: Injected beta processing failure' }),
expect.objectContaining({ status: 200, round: 3 })
]);
});
it('still accepts and broadcasts telemetry if storing diagnostics fails', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
db().exec(`CREATE TRIGGER reject_diagnostics BEFORE INSERT ON telemetry_requests
BEGIN SELECT RAISE(FAIL, 'Injected diagnostic storage failure'); END`);
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const listener = vi.fn();
const unsubscribe = subscribe(listener);
const payload = gameEvent('round_change', 7);
const response = await post(payload);
unsubscribe();
expect(response.status).toBe(200);
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
expect(listTelemetry()).toEqual([{ ...payload, receivedAt: expect.any(String) }]);
expect(listener).toHaveBeenCalledWith({
type: 'server-telemetry', entries: [{ ...payload, receivedAt: expect.any(String) }]
});
});
it('retains only the latest 200 requests for a server and removes requests older than seven days', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { recordTelemetryRequest, listTelemetryRequests } = await import('../../src/lib/server/fleet/telemetry-requests.js');
const now = new Date().toISOString();
const old = new Date(Date.now() - 8 * 86_400_000).toISOString();
const insert = db().prepare(`INSERT INTO telemetry_requests
(server, received_at, status, payload, payload_truncated) VALUES (?, ?, 200, '{}', 0)`);
db().transaction(() => {
insert.run(otherServer, old);
insert.run(otherServer, now);
for (let i = 0; i < 205; i++) insert.run(server, now);
})();
recordTelemetryRequest({ value: gameEvent('round_change', 9), receivedAt: now, status: 200, error: null, contentType: 'application/json' });
const requests = listTelemetryRequests(server);
expect(requests).toHaveLength(200);
expect(requests[0]).toMatchObject({ round: 9 });
expect(requests.map(entry => entry.id)).toEqual([...requests.map(entry => entry.id)].sort((a, b) => b - a));
expect(listTelemetryRequests(otherServer)).toHaveLength(1);
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests WHERE received_at = ?').get(old)).toEqual({ count: 0 });
});
it('caps global diagnostic history even when rejected requests name many unknown servers', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { recordTelemetryRequest } = await import('../../src/lib/server/fleet/telemetry-requests.js');
const now = new Date().toISOString();
const insert = db().prepare(`INSERT INTO telemetry_requests
(server, received_at, status, payload, payload_truncated) VALUES (?, ?, 404, '{}', 0)`);
db().transaction(() => {
for (let i = 0; i < 10_005; i++) insert.run(`unknown-${i}`, now);
})();
recordTelemetryRequest({ value: gameEvent('game_start'), receivedAt: now, status: 200, error: null, contentType: 'application/json' });
expect(db().prepare('SELECT COUNT(*) AS count FROM telemetry_requests').get()).toEqual({ count: 10_000 });
expect(db().prepare('SELECT server FROM telemetry_requests ORDER BY id DESC LIMIT 1').get()).toEqual({ server });
expect(db().prepare("SELECT id FROM telemetry_requests WHERE server = 'unknown-0'").get()).toBeUndefined();
});